You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/07/03 18:10:19 UTC
[01/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Repository: spark-website
Updated Branches:
refs/heads/asf-site 775127770 -> 6bbac4966
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/searchindex.js
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/searchindex.js b/site/docs/2.1.2/api/python/searchindex.js
index 6dd6c4c..3aaa727 100644
--- a/site/docs/2.1.2/api/python/searchindex.js
+++ b/site/docs/2.1.2/api/python/searchindex.js
@@ -1 +1 @@
-Search.setIndex({envversion:49,filenames:["index","pyspark","pyspark.ml","pyspark.mllib","pyspark.sql","pyspark.streaming"],objects:{"":{pyspark:[1,0,0,"-"]},"pyspark.Accumulator":{add:[1,2,1,""],value:[1,3,1,""]},"pyspark.AccumulatorParam":{addInPlace:[1,2,1,""],zero:[1,2,1,""]},"pyspark.BasicProfiler":{profile:[1,2,1,""],stats:[1,2,1,""]},"pyspark.Broadcast":{destroy:[1,2,1,""],dump:[1,2,1,""],load:[1,2,1,""],unpersist:[1,2,1,""],value:[1,3,1,""]},"pyspark.MarshalSerializer":{dumps:[1,2,1,""],loads:[1,2,1,""]},"pyspark.PickleSerializer":{dumps:[1,2,1,""],loads:[1,2,1,""]},"pyspark.Profiler":{dump:[1,2,1,""],profile:[1,2,1,""],show:[1,2,1,""],stats:[1,2,1,""]},"pyspark.RDD":{aggregate:[1,2,1,""],aggregateByKey:[1,2,1,""],cache:[1,2,1,""],cartesian:[1,2,1,""],checkpoint:[1,2,1,""],coalesce:[1,2,1,""],cogroup:[1,2,1,""],collect:[1,2,1,""],collectAsMap:[1,2,1,""],combineByKey:[1,2,1,""],context:[1,3,1,""],count:[1,2,1,""],countApprox:[1,2,1,""],countApproxDistinct:[1,2,1,""],countByKe
y:[1,2,1,""],countByValue:[1,2,1,""],distinct:[1,2,1,""],filter:[1,2,1,""],first:[1,2,1,""],flatMap:[1,2,1,""],flatMapValues:[1,2,1,""],fold:[1,2,1,""],foldByKey:[1,2,1,""],foreach:[1,2,1,""],foreachPartition:[1,2,1,""],fullOuterJoin:[1,2,1,""],getCheckpointFile:[1,2,1,""],getNumPartitions:[1,2,1,""],getStorageLevel:[1,2,1,""],glom:[1,2,1,""],groupBy:[1,2,1,""],groupByKey:[1,2,1,""],groupWith:[1,2,1,""],histogram:[1,2,1,""],id:[1,2,1,""],intersection:[1,2,1,""],isCheckpointed:[1,2,1,""],isEmpty:[1,2,1,""],isLocallyCheckpointed:[1,2,1,""],join:[1,2,1,""],keyBy:[1,2,1,""],keys:[1,2,1,""],leftOuterJoin:[1,2,1,""],localCheckpoint:[1,2,1,""],lookup:[1,2,1,""],map:[1,2,1,""],mapPartitions:[1,2,1,""],mapPartitionsWithIndex:[1,2,1,""],mapPartitionsWithSplit:[1,2,1,""],mapValues:[1,2,1,""],max:[1,2,1,""],mean:[1,2,1,""],meanApprox:[1,2,1,""],min:[1,2,1,""],name:[1,2,1,""],partitionBy:[1,2,1,""],persist:[1,2,1,""],pipe:[1,2,1,""],randomSplit:[1,2,1,""],reduce:[1,2,1,""],reduceByKey:[1,2,1,""]
,reduceByKeyLocally:[1,2,1,""],repartition:[1,2,1,""],repartitionAndSortWithinPartitions:[1,2,1,""],rightOuterJoin:[1,2,1,""],sample:[1,2,1,""],sampleByKey:[1,2,1,""],sampleStdev:[1,2,1,""],sampleVariance:[1,2,1,""],saveAsHadoopDataset:[1,2,1,""],saveAsHadoopFile:[1,2,1,""],saveAsNewAPIHadoopDataset:[1,2,1,""],saveAsNewAPIHadoopFile:[1,2,1,""],saveAsPickleFile:[1,2,1,""],saveAsSequenceFile:[1,2,1,""],saveAsTextFile:[1,2,1,""],setName:[1,2,1,""],sortBy:[1,2,1,""],sortByKey:[1,2,1,""],stats:[1,2,1,""],stdev:[1,2,1,""],subtract:[1,2,1,""],subtractByKey:[1,2,1,""],sum:[1,2,1,""],sumApprox:[1,2,1,""],take:[1,2,1,""],takeOrdered:[1,2,1,""],takeSample:[1,2,1,""],toDebugString:[1,2,1,""],toLocalIterator:[1,2,1,""],top:[1,2,1,""],treeAggregate:[1,2,1,""],treeReduce:[1,2,1,""],union:[1,2,1,""],unpersist:[1,2,1,""],values:[1,2,1,""],variance:[1,2,1,""],zip:[1,2,1,""],zipWithIndex:[1,2,1,""],zipWithUniqueId:[1,2,1,""]},"pyspark.SparkConf":{contains:[1,2,1,""],get:[1,2,1,""],getAll:[1,2,1,""],se
t:[1,2,1,""],setAll:[1,2,1,""],setAppName:[1,2,1,""],setExecutorEnv:[1,2,1,""],setIfMissing:[1,2,1,""],setMaster:[1,2,1,""],setSparkHome:[1,2,1,""],toDebugString:[1,2,1,""]},"pyspark.SparkContext":{PACKAGE_EXTENSIONS:[1,3,1,""],accumulator:[1,2,1,""],addFile:[1,2,1,""],addPyFile:[1,2,1,""],applicationId:[1,3,1,""],binaryFiles:[1,2,1,""],binaryRecords:[1,2,1,""],broadcast:[1,2,1,""],cancelAllJobs:[1,2,1,""],cancelJobGroup:[1,2,1,""],defaultMinPartitions:[1,3,1,""],defaultParallelism:[1,3,1,""],dump_profiles:[1,2,1,""],emptyRDD:[1,2,1,""],getConf:[1,2,1,""],getLocalProperty:[1,2,1,""],getOrCreate:[1,4,1,""],hadoopFile:[1,2,1,""],hadoopRDD:[1,2,1,""],newAPIHadoopFile:[1,2,1,""],newAPIHadoopRDD:[1,2,1,""],parallelize:[1,2,1,""],pickleFile:[1,2,1,""],range:[1,2,1,""],runJob:[1,2,1,""],sequenceFile:[1,2,1,""],setCheckpointDir:[1,2,1,""],setJobGroup:[1,2,1,""],setLocalProperty:[1,2,1,""],setLogLevel:[1,2,1,""],setSystemProperty:[1,4,1,""],show_profiles:[1,2,1,""],sparkUser:[1,2,1,""],start
Time:[1,3,1,""],statusTracker:[1,2,1,""],stop:[1,2,1,""],textFile:[1,2,1,""],uiWebUrl:[1,3,1,""],union:[1,2,1,""],version:[1,3,1,""],wholeTextFiles:[1,2,1,""]},"pyspark.SparkFiles":{get:[1,4,1,""],getRootDirectory:[1,4,1,""]},"pyspark.StatusTracker":{getActiveJobsIds:[1,2,1,""],getActiveStageIds:[1,2,1,""],getJobIdsForGroup:[1,2,1,""],getJobInfo:[1,2,1,""],getStageInfo:[1,2,1,""]},"pyspark.StorageLevel":{DISK_ONLY:[1,3,1,""],DISK_ONLY_2:[1,3,1,""],MEMORY_AND_DISK:[1,3,1,""],MEMORY_AND_DISK_2:[1,3,1,""],MEMORY_AND_DISK_SER:[1,3,1,""],MEMORY_AND_DISK_SER_2:[1,3,1,""],MEMORY_ONLY:[1,3,1,""],MEMORY_ONLY_2:[1,3,1,""],MEMORY_ONLY_SER:[1,3,1,""],MEMORY_ONLY_SER_2:[1,3,1,""],OFF_HEAP:[1,3,1,""]},"pyspark.ml":{Estimator:[2,1,1,""],Model:[2,1,1,""],Pipeline:[2,1,1,""],PipelineModel:[2,1,1,""],Transformer:[2,1,1,""],classification:[2,0,0,"-"],clustering:[2,0,0,"-"],evaluation:[2,0,0,"-"],feature:[2,0,0,"-"],linalg:[2,0,0,"-"],param:[2,0,0,"-"],recommendation:[2,0,0,"-"],regression:[2,0,0,"-"],
tuning:[2,0,0,"-"]},"pyspark.ml.Estimator":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],params:[2,3,1,""]},"pyspark.ml.Model":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],params:[2,3,1,""],transform:[2,2,1,""]},"pyspark.ml.Pipeline":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getStages:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,4,1,""],save:[2,2,1,""],setParams:[2,2,1,""],setStages:[2,2,1,""],stages:[2,3,1,""],write:[2,2,1,""]},"pyspark
.ml.PipelineModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,4,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.Transformer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],params:[2,3,1,""],transform:[2,2,1,""]},"pyspark.ml.classification":{BinaryLogisticRegressionSummary:[2,1,1,""],BinaryLogisticRegressionTrainingSummary:[2,1,1,""],DecisionTreeClassificationModel:[2,1,1,""],DecisionTreeClassifier:[2,1,1,""],GBTClassificationModel:[2,1,1,""],GBTClassifier:[2,1,1,""],LogisticRegression:[2,1,1,""],LogisticRegressionModel:[2,1,1,""],LogisticRegressionSummary:[2,1,1,""],LogisticRegre
ssionTrainingSummary:[2,1,1,""],MultilayerPerceptronClassificationModel:[2,1,1,""],MultilayerPerceptronClassifier:[2,1,1,""],NaiveBayes:[2,1,1,""],NaiveBayesModel:[2,1,1,""],OneVsRest:[2,1,1,""],OneVsRestModel:[2,1,1,""],RandomForestClassificationModel:[2,1,1,""],RandomForestClassifier:[2,1,1,""]},"pyspark.ml.classification.BinaryLogisticRegressionSummary":{areaUnderROC:[2,3,1,""],fMeasureByThreshold:[2,3,1,""],featuresCol:[2,3,1,""],labelCol:[2,3,1,""],pr:[2,3,1,""],precisionByThreshold:[2,3,1,""],predictions:[2,3,1,""],probabilityCol:[2,3,1,""],recallByThreshold:[2,3,1,""],roc:[2,3,1,""]},"pyspark.ml.classification.BinaryLogisticRegressionTrainingSummary":{areaUnderROC:[2,3,1,""],fMeasureByThreshold:[2,3,1,""],featuresCol:[2,3,1,""],labelCol:[2,3,1,""],objectiveHistory:[2,3,1,""],pr:[2,3,1,""],precisionByThreshold:[2,3,1,""],predictions:[2,3,1,""],probabilityCol:[2,3,1,""],recallByThreshold:[2,3,1,""],roc:[2,3,1,""],totalIterations:[2,3,1,""]},"pyspark.ml.classification.DecisionTr
eeClassificationModel":{copy:[2,2,1,""],depth:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureImportances:[2,3,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numClasses:[2,3,1,""],numFeatures:[2,3,1,""],numNodes:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],toDebugString:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.classification.DecisionTreeClassifier":{cacheNodeIds:[2,3,1,""],checkpointInterval:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getCacheNodeIds:[2,2,1,""],getCheckpointInterval:[2,2,1,""],getFeaturesCol:[2,2,1,""],getImpurity:[2,2,1,""],getLabelCol:[2,2,1,""],getMaxBins:[2,2,1,""],getMaxDepth:[2,2,1,""],getMaxMemoryInMB:[2,2,1,""],getMinInfoGain:[2,2,1,""],getMinInstancesPerNode:[2,2,1,""],getOrDefault:[
2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getProbabilityCol:[2,2,1,""],getRawPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],impurity:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],maxBins:[2,3,1,""],maxDepth:[2,3,1,""],maxMemoryInMB:[2,3,1,""],minInfoGain:[2,3,1,""],minInstancesPerNode:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],probabilityCol:[2,3,1,""],rawPredictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setCacheNodeIds:[2,2,1,""],setCheckpointInterval:[2,2,1,""],setFeaturesCol:[2,2,1,""],setImpurity:[2,2,1,""],setLabelCol:[2,2,1,""],setMaxBins:[2,2,1,""],setMaxDepth:[2,2,1,""],setMaxMemoryInMB:[2,2,1,""],setMinInfoGain:[2,2,1,""],setMinInstancesPerNode:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setProbabilityCol:[2,2,1,""],setRawPredictionCol:[2,2,1,""],setSeed:[2,2,1,""],supportedImpurities:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classif
ication.GBTClassificationModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureImportances:[2,3,1,""],getNumTrees:[2,3,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],toDebugString:[2,3,1,""],totalNumNodes:[2,3,1,""],transform:[2,2,1,""],treeWeights:[2,3,1,""],trees:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classification.GBTClassifier":{cacheNodeIds:[2,3,1,""],checkpointInterval:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getCacheNodeIds:[2,2,1,""],getCheckpointInterval:[2,2,1,""],getFeaturesCol:[2,2,1,""],getLabelCol:[2,2,1,""],getLossType:[2,2,1,""],getMaxBins:[2,2,1,""],getMaxDepth:[2,2,1,""],getMaxIter:[2,2,1,""],getMaxMemoryInMB:[2,2,1,""],getMinInfoGain:[2,2,1,"
"],getMinInstancesPerNode:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],getStepSize:[2,2,1,""],getSubsamplingRate:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],lossType:[2,3,1,""],maxBins:[2,3,1,""],maxDepth:[2,3,1,""],maxIter:[2,3,1,""],maxMemoryInMB:[2,3,1,""],minInfoGain:[2,3,1,""],minInstancesPerNode:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setCacheNodeIds:[2,2,1,""],setCheckpointInterval:[2,2,1,""],setFeaturesCol:[2,2,1,""],setLabelCol:[2,2,1,""],setLossType:[2,2,1,""],setMaxBins:[2,2,1,""],setMaxDepth:[2,2,1,""],setMaxIter:[2,2,1,""],setMaxMemoryInMB:[2,2,1,""],setMinInfoGain:[2,2,1,""],setMinInstancesPerNode:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setSeed:[2,2,1,""],setStepSize:[2,2,1,""],setSubsamplingRate:[2,2,1,""],stepSize:[2,3,1,""],subsamplingRate:[2,3,1,"
"],supportedLossTypes:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classification.LogisticRegression":{aggregationDepth:[2,3,1,""],copy:[2,2,1,""],elasticNetParam:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],family:[2,3,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],fitIntercept:[2,3,1,""],getAggregationDepth:[2,2,1,""],getElasticNetParam:[2,2,1,""],getFamily:[2,2,1,""],getFeaturesCol:[2,2,1,""],getFitIntercept:[2,2,1,""],getLabelCol:[2,2,1,""],getMaxIter:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getProbabilityCol:[2,2,1,""],getRawPredictionCol:[2,2,1,""],getRegParam:[2,2,1,""],getStandardization:[2,2,1,""],getThreshold:[2,2,1,""],getThresholds:[2,2,1,""],getTol:[2,2,1,""],getWeightCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],probabilityCol:[2,3,1,""],rawPredicti
onCol:[2,3,1,""],read:[2,2,1,""],regParam:[2,3,1,""],save:[2,2,1,""],setAggregationDepth:[2,2,1,""],setElasticNetParam:[2,2,1,""],setFamily:[2,2,1,""],setFeaturesCol:[2,2,1,""],setFitIntercept:[2,2,1,""],setLabelCol:[2,2,1,""],setMaxIter:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setProbabilityCol:[2,2,1,""],setRawPredictionCol:[2,2,1,""],setRegParam:[2,2,1,""],setStandardization:[2,2,1,""],setThreshold:[2,2,1,""],setThresholds:[2,2,1,""],setTol:[2,2,1,""],setWeightCol:[2,2,1,""],standardization:[2,3,1,""],threshold:[2,3,1,""],thresholds:[2,3,1,""],tol:[2,3,1,""],weightCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classification.LogisticRegressionModel":{coefficientMatrix:[2,3,1,""],coefficients:[2,3,1,""],copy:[2,2,1,""],evaluate:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],hasSummary:[2,3,1,""],intercept:[2,3,1,""],interceptVector:[2,3,1,"
"],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numClasses:[2,3,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],summary:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.classification.LogisticRegressionSummary":{featuresCol:[2,3,1,""],labelCol:[2,3,1,""],predictions:[2,3,1,""],probabilityCol:[2,3,1,""]},"pyspark.ml.classification.LogisticRegressionTrainingSummary":{featuresCol:[2,3,1,""],labelCol:[2,3,1,""],objectiveHistory:[2,3,1,""],predictions:[2,3,1,""],probabilityCol:[2,3,1,""],totalIterations:[2,3,1,""]},"pyspark.ml.classification.MultilayerPerceptronClassificationModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],layers:[2,3,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],weights:[2,3,1,
""],write:[2,2,1,""]},"pyspark.ml.classification.MultilayerPerceptronClassifier":{blockSize:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getBlockSize:[2,2,1,""],getFeaturesCol:[2,2,1,""],getInitialWeights:[2,2,1,""],getLabelCol:[2,2,1,""],getLayers:[2,2,1,""],getMaxIter:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],getSolver:[2,2,1,""],getStepSize:[2,2,1,""],getTol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],initialWeights:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],layers:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setBlockSize:[2,2,1,""],setFeaturesCol:[2,2,1,""],setInitialWeights:[2,2,1,""],setLabelCol:[2,2,1,""],setLayers:[2,2,1,""],setMaxIter:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],se
tSeed:[2,2,1,""],setSolver:[2,2,1,""],setStepSize:[2,2,1,""],setTol:[2,2,1,""],solver:[2,3,1,""],stepSize:[2,3,1,""],tol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classification.NaiveBayes":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getFeaturesCol:[2,2,1,""],getLabelCol:[2,2,1,""],getModelType:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getProbabilityCol:[2,2,1,""],getRawPredictionCol:[2,2,1,""],getSmoothing:[2,2,1,""],getThresholds:[2,2,1,""],getWeightCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],modelType:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],probabilityCol:[2,3,1,""],rawPredictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setFeaturesCol:[2,2,1,""],setLabelCol:[2,2,1,""],setModelType:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setProbabil
ityCol:[2,2,1,""],setRawPredictionCol:[2,2,1,""],setSmoothing:[2,2,1,""],setThresholds:[2,2,1,""],setWeightCol:[2,2,1,""],smoothing:[2,3,1,""],thresholds:[2,3,1,""],weightCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classification.NaiveBayesModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numClasses:[2,3,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],pi:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],theta:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.classification.OneVsRest":{classifier:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getClassifier:[2,2,1,""],getFeaturesCol:[2,2,1,""],getLabelCol:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getWeightCol:[
2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,4,1,""],save:[2,2,1,""],setClassifier:[2,2,1,""],setFeaturesCol:[2,2,1,""],setLabelCol:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setWeightCol:[2,2,1,""],weightCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classification.OneVsRestModel":{classifier:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],getClassifier:[2,2,1,""],getFeaturesCol:[2,2,1,""],getLabelCol:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getWeightCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,4,1,""],save:[2,2,1,""],setClassifier:[2,2,1,""],setFeaturesCol:[2,2,1,""],setLabelCol:[2,2,
1,""],setPredictionCol:[2,2,1,""],setWeightCol:[2,2,1,""],transform:[2,2,1,""],weightCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classification.RandomForestClassificationModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureImportances:[2,3,1,""],getNumTrees:[2,3,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numClasses:[2,3,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],toDebugString:[2,3,1,""],totalNumNodes:[2,3,1,""],transform:[2,2,1,""],treeWeights:[2,3,1,""],trees:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.classification.RandomForestClassifier":{cacheNodeIds:[2,3,1,""],checkpointInterval:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureSubsetStrategy:[2,3,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getCacheNodeIds:[2,2,1,""],getCheckpoint
Interval:[2,2,1,""],getFeatureSubsetStrategy:[2,2,1,""],getFeaturesCol:[2,2,1,""],getImpurity:[2,2,1,""],getLabelCol:[2,2,1,""],getMaxBins:[2,2,1,""],getMaxDepth:[2,2,1,""],getMaxMemoryInMB:[2,2,1,""],getMinInfoGain:[2,2,1,""],getMinInstancesPerNode:[2,2,1,""],getNumTrees:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getProbabilityCol:[2,2,1,""],getRawPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],getSubsamplingRate:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],impurity:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],maxBins:[2,3,1,""],maxDepth:[2,3,1,""],maxMemoryInMB:[2,3,1,""],minInfoGain:[2,3,1,""],minInstancesPerNode:[2,3,1,""],numTrees:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],probabilityCol:[2,3,1,""],rawPredictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setCacheNodeIds:[2,2,1,""],setCheckpointInterval:[2,2,1,""],setFeatureSubsetStrategy:[2,2,1,""],setFeaturesCol:[2
,2,1,""],setImpurity:[2,2,1,""],setLabelCol:[2,2,1,""],setMaxBins:[2,2,1,""],setMaxDepth:[2,2,1,""],setMaxMemoryInMB:[2,2,1,""],setMinInfoGain:[2,2,1,""],setMinInstancesPerNode:[2,2,1,""],setNumTrees:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setProbabilityCol:[2,2,1,""],setRawPredictionCol:[2,2,1,""],setSeed:[2,2,1,""],setSubsamplingRate:[2,2,1,""],subsamplingRate:[2,3,1,""],supportedFeatureSubsetStrategies:[2,3,1,""],supportedImpurities:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.clustering":{BisectingKMeans:[2,1,1,""],BisectingKMeansModel:[2,1,1,""],BisectingKMeansSummary:[2,1,1,""],DistributedLDAModel:[2,1,1,""],GaussianMixture:[2,1,1,""],GaussianMixtureModel:[2,1,1,""],GaussianMixtureSummary:[2,1,1,""],KMeans:[2,1,1,""],KMeansModel:[2,1,1,""],LDA:[2,1,1,""],LDAModel:[2,1,1,""],LocalLDAModel:[2,1,1,""]},"pyspark.ml.clustering.BisectingKMeans":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2
,1,""],getFeaturesCol:[2,2,1,""],getK:[2,2,1,""],getMaxIter:[2,2,1,""],getMinDivisibleClusterSize:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],k:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],minDivisibleClusterSize:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setFeaturesCol:[2,2,1,""],setK:[2,2,1,""],setMaxIter:[2,2,1,""],setMinDivisibleClusterSize:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setSeed:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.clustering.BisectingKMeansModel":{clusterCenters:[2,2,1,""],computeCost:[2,2,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],hasSummary:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params
:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],summary:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.clustering.BisectingKMeansSummary":{cluster:[2,3,1,""],clusterSizes:[2,3,1,""],featuresCol:[2,3,1,""],k:[2,3,1,""],predictionCol:[2,3,1,""],predictions:[2,3,1,""]},"pyspark.ml.clustering.DistributedLDAModel":{copy:[2,2,1,""],describeTopics:[2,2,1,""],estimatedDocConcentration:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getCheckpointFiles:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isDistributed:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],logLikelihood:[2,2,1,""],logPerplexity:[2,2,1,""],logPrior:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],toLocal:[2,2,1,""],topicsMatrix:[2,2,1,""],trainingLogLikelihood:[2,2,1,""],transform:[2,2,1,""],vocabSize:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.clustering.GaussianMixture":{copy:[2,2,1,""],explainPara
m:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getFeaturesCol:[2,2,1,""],getK:[2,2,1,""],getMaxIter:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getProbabilityCol:[2,2,1,""],getSeed:[2,2,1,""],getTol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],k:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],probabilityCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setFeaturesCol:[2,2,1,""],setK:[2,2,1,""],setMaxIter:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setProbabilityCol:[2,2,1,""],setSeed:[2,2,1,""],setTol:[2,2,1,""],tol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.clustering.GaussianMixtureModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],gaussiansDF:[2,3,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam
:[2,2,1,""],hasSummary:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],summary:[2,3,1,""],transform:[2,2,1,""],weights:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.clustering.GaussianMixtureSummary":{cluster:[2,3,1,""],clusterSizes:[2,3,1,""],featuresCol:[2,3,1,""],k:[2,3,1,""],predictionCol:[2,3,1,""],predictions:[2,3,1,""],probability:[2,3,1,""],probabilityCol:[2,3,1,""]},"pyspark.ml.clustering.KMeans":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getFeaturesCol:[2,2,1,""],getInitMode:[2,2,1,""],getInitSteps:[2,2,1,""],getK:[2,2,1,""],getMaxIter:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],getTol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],initMode:[2,3,1,""],initSteps:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],k:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],params
:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setFeaturesCol:[2,2,1,""],setInitMode:[2,2,1,""],setInitSteps:[2,2,1,""],setK:[2,2,1,""],setMaxIter:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setSeed:[2,2,1,""],setTol:[2,2,1,""],tol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.clustering.KMeansModel":{clusterCenters:[2,2,1,""],computeCost:[2,2,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],hasSummary:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],summary:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.clustering.LDA":{checkpointInterval:[2,3,1,""],copy:[2,2,1,""],docConcentration:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getCheckpointInte
rval:[2,2,1,""],getDocConcentration:[2,2,1,""],getFeaturesCol:[2,2,1,""],getK:[2,2,1,""],getKeepLastCheckpoint:[2,2,1,""],getLearningDecay:[2,2,1,""],getLearningOffset:[2,2,1,""],getMaxIter:[2,2,1,""],getOptimizeDocConcentration:[2,2,1,""],getOptimizer:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getSeed:[2,2,1,""],getSubsamplingRate:[2,2,1,""],getTopicConcentration:[2,2,1,""],getTopicDistributionCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],k:[2,3,1,""],keepLastCheckpoint:[2,3,1,""],learningDecay:[2,3,1,""],learningOffset:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],optimizeDocConcentration:[2,3,1,""],optimizer:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setCheckpointInterval:[2,2,1,""],setDocConcentration:[2,2,1,""],setFeaturesCol:[2,2,1,""],setK:[2,2,1,""],setKeepLastCheckpoint:[2,2,1,""],setLearningDecay:[2,2,1,""],setLearningOffset:[2,2,1,""],setMaxIter:[2,2,1,""],setOptimizeDocConcentr
ation:[2,2,1,""],setOptimizer:[2,2,1,""],setParams:[2,2,1,""],setSeed:[2,2,1,""],setSubsamplingRate:[2,2,1,""],setTopicConcentration:[2,2,1,""],setTopicDistributionCol:[2,2,1,""],subsamplingRate:[2,3,1,""],topicConcentration:[2,3,1,""],topicDistributionCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.clustering.LDAModel":{copy:[2,2,1,""],describeTopics:[2,2,1,""],estimatedDocConcentration:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isDistributed:[2,2,1,""],isSet:[2,2,1,""],logLikelihood:[2,2,1,""],logPerplexity:[2,2,1,""],params:[2,3,1,""],topicsMatrix:[2,2,1,""],transform:[2,2,1,""],vocabSize:[2,2,1,""]},"pyspark.ml.clustering.LocalLDAModel":{copy:[2,2,1,""],describeTopics:[2,2,1,""],estimatedDocConcentration:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],
hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isDistributed:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],logLikelihood:[2,2,1,""],logPerplexity:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],topicsMatrix:[2,2,1,""],transform:[2,2,1,""],vocabSize:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.evaluation":{BinaryClassificationEvaluator:[2,1,1,""],Evaluator:[2,1,1,""],MulticlassClassificationEvaluator:[2,1,1,""],RegressionEvaluator:[2,1,1,""]},"pyspark.ml.evaluation.BinaryClassificationEvaluator":{copy:[2,2,1,""],evaluate:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getLabelCol:[2,2,1,""],getMetricName:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getRawPredictionCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isLargerBetter:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],metricName:[2,3,1,""],params:[2,3,1,""],rawPredictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,
""],setLabelCol:[2,2,1,""],setMetricName:[2,2,1,""],setParams:[2,2,1,""],setRawPredictionCol:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.evaluation.Evaluator":{copy:[2,2,1,""],evaluate:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isLargerBetter:[2,2,1,""],isSet:[2,2,1,""],params:[2,3,1,""]},"pyspark.ml.evaluation.MulticlassClassificationEvaluator":{copy:[2,2,1,""],evaluate:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getLabelCol:[2,2,1,""],getMetricName:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isLargerBetter:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],metricName:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setLabelCol:[2,2,1,""],setMet
ricName:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.evaluation.RegressionEvaluator":{copy:[2,2,1,""],evaluate:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getLabelCol:[2,2,1,""],getMetricName:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isLargerBetter:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],metricName:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setLabelCol:[2,2,1,""],setMetricName:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature":{Binarizer:[2,1,1,""],Bucketizer:[2,1,1,""],ChiSqSelector:[2,1,1,""],ChiSqSelectorModel:[2,1,1,""],CountVectorizer:[2,1,1,""],CountVectorizerModel:[2,1,1,""],DCT:[2,1,1,""],ElementwiseProduct:[2,1,1,""],HashingTF:[2,1,1,""],IDF:[2,1,1,""],IDFModel:[2,1,1,""],
IndexToString:[2,1,1,""],MaxAbsScaler:[2,1,1,""],MaxAbsScalerModel:[2,1,1,""],MinMaxScaler:[2,1,1,""],MinMaxScalerModel:[2,1,1,""],NGram:[2,1,1,""],Normalizer:[2,1,1,""],OneHotEncoder:[2,1,1,""],PCA:[2,1,1,""],PCAModel:[2,1,1,""],PolynomialExpansion:[2,1,1,""],QuantileDiscretizer:[2,1,1,""],RFormula:[2,1,1,""],RFormulaModel:[2,1,1,""],RegexTokenizer:[2,1,1,""],SQLTransformer:[2,1,1,""],StandardScaler:[2,1,1,""],StandardScalerModel:[2,1,1,""],StopWordsRemover:[2,1,1,""],StringIndexer:[2,1,1,""],StringIndexerModel:[2,1,1,""],Tokenizer:[2,1,1,""],VectorAssembler:[2,1,1,""],VectorIndexer:[2,1,1,""],VectorIndexerModel:[2,1,1,""],VectorSlicer:[2,1,1,""],Word2Vec:[2,1,1,""],Word2VecModel:[2,1,1,""]},"pyspark.ml.feature.Binarizer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getThreshold:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,
1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setThreshold:[2,2,1,""],threshold:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.Bucketizer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getHandleInvalid:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getSplits:[2,2,1,""],handleInvalid:[2,3,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setHandleInvalid:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setSplits:[2,2,1,""],splits:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.ChiSqSelector":{copy:[2,2,1,""],explainPara
m:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],fpr:[2,3,1,""],getFeaturesCol:[2,2,1,""],getFpr:[2,2,1,""],getLabelCol:[2,2,1,""],getNumTopFeatures:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getPercentile:[2,2,1,""],getSelectorType:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],numTopFeatures:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],percentile:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],selectorType:[2,3,1,""],setFeaturesCol:[2,2,1,""],setFpr:[2,2,1,""],setLabelCol:[2,2,1,""],setNumTopFeatures:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setPercentile:[2,2,1,""],setSelectorType:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.ChiSqSelectorModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,
1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],selectedFeatures:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.CountVectorizer":{binary:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getBinary:[2,2,1,""],getInputCol:[2,2,1,""],getMinDF:[2,2,1,""],getMinTF:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getVocabSize:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],minDF:[2,3,1,""],minTF:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setBinary:[2,2,1,""],setInputCol:[2,2,1,""],setMinDF:[2,2,1,""],setMinTF:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setVocabSize:[2,2,1,""],vocabSize:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.feature.CountVectorizerModel":{copy:
[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],vocabulary:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.feature.DCT":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getInputCol:[2,2,1,""],getInverse:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],inverse:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setInverse:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.ElementwiseProduct":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParam
s:[2,2,1,""],extractParamMap:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getScalingVec:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],scalingVec:[2,3,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setScalingVec:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.HashingTF":{binary:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getBinary:[2,2,1,""],getInputCol:[2,2,1,""],getNumFeatures:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setBinary:
[2,2,1,""],setInputCol:[2,2,1,""],setNumFeatures:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.IDF":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getInputCol:[2,2,1,""],getMinDocFreq:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],minDocFreq:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setMinDocFreq:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.IDFModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],idf:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params
:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.IndexToString":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getInputCol:[2,2,1,""],getLabels:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labels:[2,3,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setLabels:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.MaxAbsScaler":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,
2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.MaxAbsScalerModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],maxAbs:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.MinMaxScaler":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getInputCol:[2,2,1,""],getMax:[2,2,1,""],getMin:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],max:[2,3,1,""],min:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""]
,read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setMax:[2,2,1,""],setMin:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.MinMaxScalerModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],originalMax:[2,3,1,""],originalMin:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.NGram":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getInputCol:[2,2,1,""],getN:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],n:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInput
Col:[2,2,1,""],setN:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.Normalizer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getP:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],p:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setP:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.OneHotEncoder":{copy:[2,2,1,""],dropLast:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getDropLast:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isD
efined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setDropLast:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.PCA":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getInputCol:[2,2,1,""],getK:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],k:[2,3,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setK:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.PCAModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],explainedVariance:[2,3,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefa
ult:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],pc:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.PolynomialExpansion":{copy:[2,2,1,""],degree:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getDegree:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setDegree:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.QuantileDiscretizer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getHandleInvalid:[2,2,1,""],getInputCol:[2,2,1,""],getNumBuckets:[2,2,1,""],g
etOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getRelativeError:[2,2,1,""],handleInvalid:[2,3,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numBuckets:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],relativeError:[2,3,1,""],save:[2,2,1,""],setHandleInvalid:[2,2,1,""],setInputCol:[2,2,1,""],setNumBuckets:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setRelativeError:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.RFormula":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],forceIndexLabel:[2,3,1,""],formula:[2,3,1,""],getFeaturesCol:[2,2,1,""],getForceIndexLabel:[2,2,1,""],getFormula:[2,2,1,""],getLabelCol:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],params:[
2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setFeaturesCol:[2,2,1,""],setForceIndexLabel:[2,2,1,""],setFormula:[2,2,1,""],setLabelCol:[2,2,1,""],setParams:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.RFormulaModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.RegexTokenizer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],gaps:[2,3,1,""],getGaps:[2,2,1,""],getInputCol:[2,2,1,""],getMinTokenLength:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getPattern:[2,2,1,""],getToLowercase:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],minTokenLe
ngth:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],pattern:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setGaps:[2,2,1,""],setInputCol:[2,2,1,""],setMinTokenLength:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setPattern:[2,2,1,""],setToLowercase:[2,2,1,""],toLowercase:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.SQLTransformer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getStatement:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setParams:[2,2,1,""],setStatement:[2,2,1,""],statement:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.StandardScaler":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],ge
tParam:[2,2,1,""],getWithMean:[2,2,1,""],getWithStd:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setWithMean:[2,2,1,""],setWithStd:[2,2,1,""],withMean:[2,3,1,""],withStd:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.feature.StandardScalerModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],mean:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],std:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.StopWordsRemover":{caseSensitive:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getCaseSensitive:[2,2,1,""]
,getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getStopWords:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],loadDefaultStopWords:[2,5,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setCaseSensitive:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setStopWords:[2,2,1,""],stopWords:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.StringIndexer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getHandleInvalid:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],handleInvalid:[2,3,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setHandl
eInvalid:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.StringIndexerModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labels:[2,3,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.Tokenizer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getInputCol:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCol:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"p
yspark.ml.feature.VectorAssembler":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getInputCols:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCols:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInputCols:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.VectorIndexer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getInputCol:[2,2,1,""],getMaxCategories:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],maxCategories:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setInpu
tCol:[2,2,1,""],setMaxCategories:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.VectorIndexerModel":{categoryMaps:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.VectorSlicer":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getIndices:[2,2,1,""],getInputCol:[2,2,1,""],getNames:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],indices:[2,3,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],names:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""
],setIndices:[2,2,1,""],setInputCol:[2,2,1,""],setNames:[2,2,1,""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.feature.Word2Vec":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getInputCol:[2,2,1,""],getMaxIter:[2,2,1,""],getMaxSentenceLength:[2,2,1,""],getMinCount:[2,2,1,""],getNumPartitions:[2,2,1,""],getOrDefault:[2,2,1,""],getOutputCol:[2,2,1,""],getParam:[2,2,1,""],getSeed:[2,2,1,""],getStepSize:[2,2,1,""],getVectorSize:[2,2,1,""],getWindowSize:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],inputCol:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],maxSentenceLength:[2,3,1,""],minCount:[2,3,1,""],numPartitions:[2,3,1,""],outputCol:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setInputCol:[2,2,1,""],setMaxIter:[2,2,1,""],setMaxSentenceLength:[2,2,1,""],setMinCount:[2,2,1,""],setNumPartitions:[2,2,1,
""],setOutputCol:[2,2,1,""],setParams:[2,2,1,""],setSeed:[2,2,1,""],setStepSize:[2,2,1,""],setVectorSize:[2,2,1,""],setWindowSize:[2,2,1,""],stepSize:[2,3,1,""],vectorSize:[2,3,1,""],windowSize:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.feature.Word2VecModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],findSynonyms:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getVectors:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.linalg":{DenseMatrix:[2,1,1,""],DenseVector:[2,1,1,""],Matrices:[2,1,1,""],Matrix:[2,1,1,""],SparseMatrix:[2,1,1,""],SparseVector:[2,1,1,""],Vector:[2,1,1,""],Vectors:[2,1,1,""]},"pyspark.ml.linalg.DenseMatrix":{toArray:[2,2,1,""],toSparse:[2,2,1,""]},"pyspark.ml.linalg.DenseVector":{dot:[2,2,1,""],norm:[2,2,1,""],numNonzeros:[2,2,1,""],squared_distance:[2,
2,1,""],toArray:[2,2,1,""],values:[2,3,1,""]},"pyspark.ml.linalg.Matrices":{dense:[2,5,1,""],sparse:[2,5,1,""]},"pyspark.ml.linalg.Matrix":{toArray:[2,2,1,""]},"pyspark.ml.linalg.SparseMatrix":{toArray:[2,2,1,""],toDense:[2,2,1,""]},"pyspark.ml.linalg.SparseVector":{dot:[2,2,1,""],indices:[2,3,1,""],norm:[2,2,1,""],numNonzeros:[2,2,1,""],size:[2,3,1,""],squared_distance:[2,2,1,""],toArray:[2,2,1,""],values:[2,3,1,""]},"pyspark.ml.linalg.Vector":{toArray:[2,2,1,""]},"pyspark.ml.linalg.Vectors":{dense:[2,5,1,""],norm:[2,5,1,""],sparse:[2,5,1,""],squared_distance:[2,5,1,""],zeros:[2,5,1,""]},"pyspark.ml.param":{Param:[2,1,1,""],Params:[2,1,1,""],TypeConverters:[2,1,1,""]},"pyspark.ml.param.Params":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],params:[2,3,1,""]},"pyspark.ml.param.TypeConverters":{identity:[2,5,1,""],to
Boolean:[2,5,1,""],toFloat:[2,5,1,""],toInt:[2,5,1,""],toList:[2,5,1,""],toListFloat:[2,5,1,""],toListInt:[2,5,1,""],toListString:[2,5,1,""],toString:[2,5,1,""],toVector:[2,5,1,""]},"pyspark.ml.recommendation":{ALS:[2,1,1,""],ALSModel:[2,1,1,""]},"pyspark.ml.recommendation.ALS":{alpha:[2,3,1,""],checkpointInterval:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],finalStorageLevel:[2,3,1,""],fit:[2,2,1,""],getAlpha:[2,2,1,""],getCheckpointInterval:[2,2,1,""],getFinalStorageLevel:[2,2,1,""],getImplicitPrefs:[2,2,1,""],getIntermediateStorageLevel:[2,2,1,""],getItemCol:[2,2,1,""],getMaxIter:[2,2,1,""],getNonnegative:[2,2,1,""],getNumItemBlocks:[2,2,1,""],getNumUserBlocks:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getRank:[2,2,1,""],getRatingCol:[2,2,1,""],getRegParam:[2,2,1,""],getSeed:[2,2,1,""],getUserCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],implicitPrefs:[2,3,1,""],intermedia
teStorageLevel:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],itemCol:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],nonnegative:[2,3,1,""],numItemBlocks:[2,3,1,""],numUserBlocks:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],rank:[2,3,1,""],ratingCol:[2,3,1,""],read:[2,2,1,""],regParam:[2,3,1,""],save:[2,2,1,""],seed:[2,3,1,""],setAlpha:[2,2,1,""],setCheckpointInterval:[2,2,1,""],setFinalStorageLevel:[2,2,1,""],setImplicitPrefs:[2,2,1,""],setIntermediateStorageLevel:[2,2,1,""],setItemCol:[2,2,1,""],setMaxIter:[2,2,1,""],setNonnegative:[2,2,1,""],setNumBlocks:[2,2,1,""],setNumItemBlocks:[2,2,1,""],setNumUserBlocks:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setRank:[2,2,1,""],setRatingCol:[2,2,1,""],setRegParam:[2,2,1,""],setSeed:[2,2,1,""],setUserCol:[2,2,1,""],userCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.recommendation.ALSModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:
[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],itemFactors:[2,3,1,""],load:[2,2,1,""],params:[2,3,1,""],rank:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],userFactors:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.regression":{AFTSurvivalRegression:[2,1,1,""],AFTSurvivalRegressionModel:[2,1,1,""],DecisionTreeRegressionModel:[2,1,1,""],DecisionTreeRegressor:[2,1,1,""],GBTRegressionModel:[2,1,1,""],GBTRegressor:[2,1,1,""],GeneralizedLinearRegression:[2,1,1,""],GeneralizedLinearRegressionModel:[2,1,1,""],GeneralizedLinearRegressionSummary:[2,1,1,""],GeneralizedLinearRegressionTrainingSummary:[2,1,1,""],IsotonicRegression:[2,1,1,""],IsotonicRegressionModel:[2,1,1,""],LinearRegression:[2,1,1,""],LinearRegressionModel:[2,1,1,""],LinearRegressionSummary:[2,1,1,""],LinearRegressionTrainingSummary:[2,1,1,""],RandomForestRegressionModel:[2,1,1,""],RandomForestRegressor:[2,1,1,""]},"pyspark.ml.regression.AFTSurvivalRegression":{aggregatio
nDepth:[2,3,1,""],censorCol:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],fitIntercept:[2,3,1,""],getAggregationDepth:[2,2,1,""],getCensorCol:[2,2,1,""],getFeaturesCol:[2,2,1,""],getFitIntercept:[2,2,1,""],getLabelCol:[2,2,1,""],getMaxIter:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getQuantileProbabilities:[2,2,1,""],getQuantilesCol:[2,2,1,""],getTol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],quantileProbabilities:[2,3,1,""],quantilesCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setAggregationDepth:[2,2,1,""],setCensorCol:[2,2,1,""],setFeaturesCol:[2,2,1,""],setFitIntercept:[2,2,1,""],setLabelCol:[2,2,1,""],setMaxIter:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setQuantileProbabilities:[2,2
,1,""],setQuantilesCol:[2,2,1,""],setTol:[2,2,1,""],tol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.regression.AFTSurvivalRegressionModel":{coefficients:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],intercept:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],predict:[2,2,1,""],predictQuantiles:[2,2,1,""],read:[2,2,1,""],save:[2,2,1,""],scale:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.regression.DecisionTreeRegressionModel":{copy:[2,2,1,""],depth:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureImportances:[2,3,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],numNodes:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],toDeb
ugString:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.regression.DecisionTreeRegressor":{cacheNodeIds:[2,3,1,""],checkpointInterval:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getCacheNodeIds:[2,2,1,""],getCheckpointInterval:[2,2,1,""],getFeaturesCol:[2,2,1,""],getImpurity:[2,2,1,""],getLabelCol:[2,2,1,""],getMaxBins:[2,2,1,""],getMaxDepth:[2,2,1,""],getMaxMemoryInMB:[2,2,1,""],getMinInfoGain:[2,2,1,""],getMinInstancesPerNode:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],getVarianceCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],impurity:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],maxBins:[2,3,1,""],maxDepth:[2,3,1,""],maxMemoryInMB:[2,3,1,""],minInfoGain:[2,3,1,""],minInstancesPerNode:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,
""],seed:[2,3,1,""],setCacheNodeIds:[2,2,1,""],setCheckpointInterval:[2,2,1,""],setFeaturesCol:[2,2,1,""],setImpurity:[2,2,1,""],setLabelCol:[2,2,1,""],setMaxBins:[2,2,1,""],setMaxDepth:[2,2,1,""],setMaxMemoryInMB:[2,2,1,""],setMinInfoGain:[2,2,1,""],setMinInstancesPerNode:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setSeed:[2,2,1,""],setVarianceCol:[2,2,1,""],supportedImpurities:[2,3,1,""],varianceCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.regression.GBTRegressionModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureImportances:[2,3,1,""],getNumTrees:[2,3,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],toDebugString:[2,3,1,""],totalNumNodes:[2,3,1,""],transform:[2,2,1,""],treeWeights:[2,3,1,""],trees:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.reg
ression.GBTRegressor":{cacheNodeIds:[2,3,1,""],checkpointInterval:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getCacheNodeIds:[2,2,1,""],getCheckpointInterval:[2,2,1,""],getFeaturesCol:[2,2,1,""],getImpurity:[2,2,1,""],getLabelCol:[2,2,1,""],getLossType:[2,2,1,""],getMaxBins:[2,2,1,""],getMaxDepth:[2,2,1,""],getMaxIter:[2,2,1,""],getMaxMemoryInMB:[2,2,1,""],getMinInfoGain:[2,2,1,""],getMinInstancesPerNode:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],getStepSize:[2,2,1,""],getSubsamplingRate:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],impurity:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],lossType:[2,3,1,""],maxBins:[2,3,1,""],maxDepth:[2,3,1,""],maxIter:[2,3,1,""],maxMemoryInMB:[2,3,1,""],minInfoGain:[2,3,1,""],minInstancesPerNode:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],
read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setCacheNodeIds:[2,2,1,""],setCheckpointInterval:[2,2,1,""],setFeaturesCol:[2,2,1,""],setImpurity:[2,2,1,""],setLabelCol:[2,2,1,""],setLossType:[2,2,1,""],setMaxBins:[2,2,1,""],setMaxDepth:[2,2,1,""],setMaxIter:[2,2,1,""],setMaxMemoryInMB:[2,2,1,""],setMinInfoGain:[2,2,1,""],setMinInstancesPerNode:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setSeed:[2,2,1,""],setStepSize:[2,2,1,""],setSubsamplingRate:[2,2,1,""],stepSize:[2,3,1,""],subsamplingRate:[2,3,1,""],supportedImpurities:[2,3,1,""],supportedLossTypes:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.regression.GeneralizedLinearRegression":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],family:[2,3,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],fitIntercept:[2,3,1,""],getFamily:[2,2,1,""],getFeaturesCol:[2,2,1,""],getFitIntercept:[2,2,1,""],getLabelCol:[2,2,1,""],getLink:[2,2,1,""],getLinkPredictionCol:[2,2,1,""],getMaxIter:[2,2,1,"
"],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getRegParam:[2,2,1,""],getSolver:[2,2,1,""],getTol:[2,2,1,""],getWeightCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],link:[2,3,1,""],linkPredictionCol:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],regParam:[2,3,1,""],save:[2,2,1,""],setFamily:[2,2,1,""],setFeaturesCol:[2,2,1,""],setFitIntercept:[2,2,1,""],setLabelCol:[2,2,1,""],setLink:[2,2,1,""],setLinkPredictionCol:[2,2,1,""],setMaxIter:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setRegParam:[2,2,1,""],setSolver:[2,2,1,""],setTol:[2,2,1,""],setWeightCol:[2,2,1,""],solver:[2,3,1,""],tol:[2,3,1,""],weightCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.regression.GeneralizedLinearRegressionModel":{coefficients:[2,3,1,""],copy:[2,2,1,""],evaluate:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[
2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],hasSummary:[2,3,1,""],intercept:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],summary:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.regression.GeneralizedLinearRegressionSummary":{aic:[2,3,1,""],degreesOfFreedom:[2,3,1,""],deviance:[2,3,1,""],dispersion:[2,3,1,""],nullDeviance:[2,3,1,""],predictionCol:[2,3,1,""],predictions:[2,3,1,""],rank:[2,3,1,""],residualDegreeOfFreedom:[2,3,1,""],residualDegreeOfFreedomNull:[2,3,1,""],residuals:[2,2,1,""]},"pyspark.ml.regression.GeneralizedLinearRegressionTrainingSummary":{aic:[2,3,1,""],coefficientStandardErrors:[2,3,1,""],degreesOfFreedom:[2,3,1,""],deviance:[2,3,1,""],dispersion:[2,3,1,""],nullDeviance:[2,3,1,""],numIterations:[2,3,1,""],pValues:[2,3,1,""],predictionCol:[2,3,1,""],predictions:[2,3,1,""],rank:[2,3,1,""],residualDegreeOfFreedo
m:[2,3,1,""],residualDegreeOfFreedomNull:[2,3,1,""],residuals:[2,2,1,""],solver:[2,3,1,""],tValues:[2,3,1,""]},"pyspark.ml.regression.IsotonicRegression":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureIndex:[2,3,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getFeatureIndex:[2,2,1,""],getFeaturesCol:[2,2,1,""],getIsotonic:[2,2,1,""],getLabelCol:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getWeightCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],isotonic:[2,3,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],setFeatureIndex:[2,2,1,""],setFeaturesCol:[2,2,1,""],setIsotonic:[2,2,1,""],setLabelCol:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setWeightCol:[2,2,1,""],weightCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.regression.IsotonicRegressionModel":{boundaries:[2,3,1,""],
copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],params:[2,3,1,""],predictions:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.regression.LinearRegression":{aggregationDepth:[2,3,1,""],copy:[2,2,1,""],elasticNetParam:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],fitIntercept:[2,3,1,""],getAggregationDepth:[2,2,1,""],getElasticNetParam:[2,2,1,""],getFeaturesCol:[2,2,1,""],getFitIntercept:[2,2,1,""],getLabelCol:[2,2,1,""],getMaxIter:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getRegParam:[2,2,1,""],getSolver:[2,2,1,""],getStandardization:[2,2,1,""],getTol:[2,2,1,""],getWeightCol:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,
2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],maxIter:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],regParam:[2,3,1,""],save:[2,2,1,""],setAggregationDepth:[2,2,1,""],setElasticNetParam:[2,2,1,""],setFeaturesCol:[2,2,1,""],setFitIntercept:[2,2,1,""],setLabelCol:[2,2,1,""],setMaxIter:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""],setRegParam:[2,2,1,""],setSolver:[2,2,1,""],setStandardization:[2,2,1,""],setTol:[2,2,1,""],setWeightCol:[2,2,1,""],solver:[2,3,1,""],standardization:[2,3,1,""],tol:[2,3,1,""],weightCol:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.regression.LinearRegressionModel":{coefficients:[2,3,1,""],copy:[2,2,1,""],evaluate:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],hasSummary:[2,3,1,""],intercept:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],params:[2,
3,1,""],read:[2,2,1,""],save:[2,2,1,""],summary:[2,3,1,""],transform:[2,2,1,""],write:[2,2,1,""]},"pyspark.ml.regression.LinearRegressionSummary":{coefficientStandardErrors:[2,3,1,""],devianceResiduals:[2,3,1,""],explainedVariance:[2,3,1,""],featuresCol:[2,3,1,""],labelCol:[2,3,1,""],meanAbsoluteError:[2,3,1,""],meanSquaredError:[2,3,1,""],numInstances:[2,3,1,""],pValues:[2,3,1,""],predictionCol:[2,3,1,""],predictions:[2,3,1,""],r2:[2,3,1,""],residuals:[2,3,1,""],rootMeanSquaredError:[2,3,1,""],tValues:[2,3,1,""]},"pyspark.ml.regression.LinearRegressionTrainingSummary":{coefficientStandardErrors:[2,3,1,""],devianceResiduals:[2,3,1,""],explainedVariance:[2,3,1,""],featuresCol:[2,3,1,""],labelCol:[2,3,1,""],meanAbsoluteError:[2,3,1,""],meanSquaredError:[2,3,1,""],numInstances:[2,3,1,""],objectiveHistory:[2,3,1,""],pValues:[2,3,1,""],predictionCol:[2,3,1,""],predictions:[2,3,1,""],r2:[2,3,1,""],residuals:[2,3,1,""],rootMeanSquaredError:[2,3,1,""],tValues:[2,3,1,""],totalIterations:[2,3
,1,""]},"pyspark.ml.regression.RandomForestRegressionModel":{copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureImportances:[2,3,1,""],getNumTrees:[2,3,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],load:[2,2,1,""],numFeatures:[2,3,1,""],params:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],toDebugString:[2,3,1,""],totalNumNodes:[2,3,1,""],transform:[2,2,1,""],treeWeights:[2,3,1,""],trees:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.regression.RandomForestRegressor":{cacheNodeIds:[2,3,1,""],checkpointInterval:[2,3,1,""],copy:[2,2,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],featureSubsetStrategy:[2,3,1,""],featuresCol:[2,3,1,""],fit:[2,2,1,""],getCacheNodeIds:[2,2,1,""],getCheckpointInterval:[2,2,1,""],getFeatureSubsetStrategy:[2,2,1,""],getFeaturesCol:[2,2,1,""],getImpurity:[2,2,1,""],getLabelCol:[2,2,1,""],getMaxBins:[2,2,1
,""],getMaxDepth:[2,2,1,""],getMaxMemoryInMB:[2,2,1,""],getMinInfoGain:[2,2,1,""],getMinInstancesPerNode:[2,2,1,""],getNumTrees:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getPredictionCol:[2,2,1,""],getSeed:[2,2,1,""],getSubsamplingRate:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],impurity:[2,3,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],labelCol:[2,3,1,""],load:[2,2,1,""],maxBins:[2,3,1,""],maxDepth:[2,3,1,""],maxMemoryInMB:[2,3,1,""],minInfoGain:[2,3,1,""],minInstancesPerNode:[2,3,1,""],numTrees:[2,3,1,""],params:[2,3,1,""],predictionCol:[2,3,1,""],read:[2,2,1,""],save:[2,2,1,""],seed:[2,3,1,""],setCacheNodeIds:[2,2,1,""],setCheckpointInterval:[2,2,1,""],setFeatureSubsetStrategy:[2,2,1,""],setFeaturesCol:[2,2,1,""],setImpurity:[2,2,1,""],setLabelCol:[2,2,1,""],setMaxBins:[2,2,1,""],setMaxDepth:[2,2,1,""],setMaxMemoryInMB:[2,2,1,""],setMinInfoGain:[2,2,1,""],setMinInstancesPerNode:[2,2,1,""],setNumTrees:[2,2,1,""],setParams:[2,2,1,""],setPredictionCol:[2,2,1,""]
,setSeed:[2,2,1,""],setSubsamplingRate:[2,2,1,""],subsamplingRate:[2,3,1,""],supportedFeatureSubsetStrategies:[2,3,1,""],supportedImpurities:[2,3,1,""],write:[2,2,1,""]},"pyspark.ml.tuning":{CrossValidator:[2,1,1,""],CrossValidatorModel:[2,1,1,""],ParamGridBuilder:[2,1,1,""],TrainValidationSplit:[2,1,1,""],TrainValidationSplitModel:[2,1,1,""]},"pyspark.ml.tuning.CrossValidator":{copy:[2,2,1,""],estimator:[2,3,1,""],estimatorParamMaps:[2,3,1,""],evaluator:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getEstimator:[2,2,1,""],getEstimatorParamMaps:[2,2,1,""],getEvaluator:[2,2,1,""],getNumFolds:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getSeed:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],numFolds:[2,3,1,""],params:[2,3,1,""],seed:[2,3,1,""],setEstimator:[2,2,1,""],setEstimatorParamMaps:[2,2,1,""],setEvaluator:[2,2,1,""],setNumFolds:[2,2,1,""],setParams:[2,2,1,""],setSeed:[2
,2,1,""]},"pyspark.ml.tuning.CrossValidatorModel":{avgMetrics:[2,3,1,""],bestModel:[2,3,1,""],copy:[2,2,1,""],estimator:[2,3,1,""],estimatorParamMaps:[2,3,1,""],evaluator:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getEstimator:[2,2,1,""],getEstimatorParamMaps:[2,2,1,""],getEvaluator:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getSeed:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],params:[2,3,1,""],seed:[2,3,1,""],setEstimator:[2,2,1,""],setEstimatorParamMaps:[2,2,1,""],setEvaluator:[2,2,1,""],setSeed:[2,2,1,""],transform:[2,2,1,""]},"pyspark.ml.tuning.ParamGridBuilder":{addGrid:[2,2,1,""],baseOn:[2,2,1,""],build:[2,2,1,""]},"pyspark.ml.tuning.TrainValidationSplit":{copy:[2,2,1,""],estimator:[2,3,1,""],estimatorParamMaps:[2,3,1,""],evaluator:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],fit:[2,2,1,""],getEstimator:[2,2,1,""],getEstimatorParamMaps
:[2,2,1,""],getEvaluator:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getSeed:[2,2,1,""],getTrainRatio:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],params:[2,3,1,""],seed:[2,3,1,""],setEstimator:[2,2,1,""],setEstimatorParamMaps:[2,2,1,""],setEvaluator:[2,2,1,""],setParams:[2,2,1,""],setSeed:[2,2,1,""],setTrainRatio:[2,2,1,""],trainRatio:[2,3,1,""]},"pyspark.ml.tuning.TrainValidationSplitModel":{bestModel:[2,3,1,""],copy:[2,2,1,""],estimator:[2,3,1,""],estimatorParamMaps:[2,3,1,""],evaluator:[2,3,1,""],explainParam:[2,2,1,""],explainParams:[2,2,1,""],extractParamMap:[2,2,1,""],getEstimator:[2,2,1,""],getEstimatorParamMaps:[2,2,1,""],getEvaluator:[2,2,1,""],getOrDefault:[2,2,1,""],getParam:[2,2,1,""],getSeed:[2,2,1,""],hasDefault:[2,2,1,""],hasParam:[2,2,1,""],isDefined:[2,2,1,""],isSet:[2,2,1,""],params:[2,3,1,""],seed:[2,3,1,""],setEstimator:[2,2,1,""],setEstimatorParamMaps:[2,2,1,""],setEvaluator:[2,2,1,""],setSeed:[2,2,1,""],
transform:[2,2,1,""],validationMetrics:[2,3,1,""]},"pyspark.mllib":{classification:[3,0,0,"-"],clustering:[3,0,0,"-"],evaluation:[3,0,0,"-"],feature:[3,0,0,"-"],fpm:[3,0,0,"-"],linalg:[3,0,0,"-"],random:[3,0,0,"-"],recommendation:[3,0,0,"-"],regression:[3,0,0,"-"],stat:[3,0,0,"-"],tree:[3,0,0,"-"],util:[3,0,0,"-"]},"pyspark.mllib.classification":{LogisticRegressionModel:[3,1,1,""],LogisticRegressionWithLBFGS:[3,1,1,""],LogisticRegressionWithSGD:[3,1,1,""],NaiveBayes:[3,1,1,""],NaiveBayesModel:[3,1,1,""],SVMModel:[3,1,1,""],SVMWithSGD:[3,1,1,""],StreamingLogisticRegressionWithSGD:[3,1,1,""]},"pyspark.mllib.classification.LogisticRegressionModel":{clearThreshold:[3,2,1,""],intercept:[3,3,1,""],load:[3,4,1,""],numClasses:[3,3,1,""],numFeatures:[3,3,1,""],predict:[3,2,1,""],save:[3,2,1,""],setThreshold:[3,2,1,""],threshold:[3,3,1,""],weights:[3,3,1,""]},"pyspark.mllib.classification.LogisticRegressionWithLBFGS":{train:[3,4,1,""]},"pyspark.mllib.classification.LogisticRegressionWithSGD":
{train:[3,4,1,""]},"pyspark.mllib.classification.NaiveBayes":{train:[3,4,1,""]},"pyspark.mllib.classification.NaiveBayesModel":{load:[3,4,1,""],predict:[3,2,1,""],save:[3,2,1,""]},"pyspark.mllib.classification.SVMModel":{clearThreshold:[3,2,1,""],intercept:[3,3,1,""],load:[3,4,1,""],predict:[3,2,1,""],save:[3,2,1,""],setThreshold:[3,2,1,""],threshold:[3,3,1,""],weights:[3,3,1,""]},"pyspark.mllib.classification.SVMWithSGD":{train:[3,4,1,""]},"pyspark.mllib.classification.StreamingLogisticRegressionWithSGD":{latestModel:[3,2,1,""],predictOn:[3,2,1,""],predictOnValues:[3,2,1,""],setInitialWeights:[3,2,1,""],trainOn:[3,2,1,""]},"pyspark.mllib.clustering":{BisectingKMeans:[3,1,1,""],BisectingKMeansModel:[3,1,1,""],GaussianMixture:[3,1,1,""],GaussianMixtureModel:[3,1,1,""],KMeans:[3,1,1,""],KMeansModel:[3,1,1,""],LDA:[3,1,1,""],LDAModel:[3,1,1,""],PowerIterationClustering:[3,1,1,""],PowerIterationClusteringModel:[3,1,1,""],StreamingKMeans:[3,1,1,""],StreamingKMeansModel:[3,1,1,""]},"pyspa
rk.mllib.clustering.BisectingKMeans":{train:[3,4,1,""]},"pyspark.mllib.clustering.BisectingKMeansModel":{clusterCenters:[3,3,1,""],computeCost:[3,2,1,""],k:[3,3,1,""],predict:[3,2,1,""]},"pyspark.mllib.clustering.GaussianMixture":{train:[3,4,1,""]},"pyspark.mllib.clustering.GaussianMixtureModel":{gaussians:[3,3,1,""],k:[3,3,1,""],load:[3,4,1,""],predict:[3,2,1,""],predictSoft:[3,2,1,""],weights:[3,3,1,""]},"pyspark.mllib.clustering.KMeans":{train:[3,4,1,""]},"pyspark.mllib.clustering.KMeansModel":{clusterCenters:[3,3,1,""],computeCost:[3,2,1,""],k:[3,3,1,""],load:[3,4,1,""],predict:[3,2,1,""],save:[3,2,1,""]},"pyspark.mllib.clustering.LDA":{train:[3,4,1,""]},"pyspark.mllib.clustering.LDAModel":{describeTopics:[3,2,1,""],load:[3,4,1,""],topicsMatrix:[3,2,1,""],vocabSize:[3,2,1,""]},"pyspark.mllib.clustering.PowerIterationClustering":{Assignment:[3,1,1,""],train:[3,4,1,""]},"pyspark.mllib.clustering.PowerIterationClusteringModel":{assignments:[3,2,1,""],k:[3,3,1,""],load:[3,4,1,""]},"
pyspark.mllib.clustering.StreamingKMeans":{latestModel:[3,2,1,""],predictOn:[3,2,1,""],predictOnValues:[3,2,1,""],setDecayFactor:[3,2,1,""],setHalfLife:[3,2,1,""],setInitialCenters:[3,2,1,""],setK:[3,2,1,""],setRandomCenters:[3,2,1,""],trainOn:[3,2,1,""]},"pyspark.mllib.clustering.StreamingKMeansModel":{clusterWeights:[3,3,1,""],update:[3,2,1,""]},"pyspark.mllib.evaluation":{BinaryClassificationMetrics:[3,1,1,""],MulticlassMetrics:[3,1,1,""],RankingMetrics:[3,1,1,""],RegressionMetrics:[3,1,1,""]},"pyspark.mllib.evaluation.BinaryClassificationMetrics":{areaUnderPR:[3,3,1,""],areaUnderROC:[3,3,1,""],unpersist:[3,2,1,""]},"pyspark.mllib.evaluation.MulticlassMetrics":{accuracy:[3,3,1,""],confusionMatrix:[3,2,1,""],fMeasure:[3,2,1,""],falsePositiveRate:[3,2,1,""],precision:[3,2,1,""],recall:[3,2,1,""],truePositiveRate:[3,2,1,""],weightedFMeasure:[3,2,1,""],weightedFalsePositiveRate:[3,3,1,""],weightedPrecision:[3,3,1,""],weightedRecall:[3,3,1,""],weightedTruePositiveRate:[3,3,1,""]},"pys
park.mllib.evaluation.RankingMetrics":{meanAveragePrecision:[3,3,1,""],ndcgAt:[3,2,1,""],precisionAt:[3,2,1,""]},"pyspark.mllib.evaluation.RegressionMetrics":{explainedVariance:[3,3,1,""],meanAbsoluteError:[3,3,1,""],meanSquaredError:[3,3,1,""],r2:[3,3,1,""],rootMeanSquaredError:[3,3,1,""]},"pyspark.mllib.feature":{ChiSqSelector:[3,1,1,""],ChiSqSelectorModel:[3,1,1,""],ElementwiseProduct:[3,1,1,""],HashingTF:[3,1,1,""],IDF:[3,1,1,""],IDFModel:[3,1,1,""],Normalizer:[3,1,1,""],StandardScaler:[3,1,1,""],StandardScalerModel:[3,1,1,""],Word2Vec:[3,1,1,""],Word2VecModel:[3,1,1,""]},"pyspark.mllib.feature.ChiSqSelector":{fit:[3,2,1,""],setFpr:[3,2,1,""],setNumTopFeatures:[3,2,1,""],setPercentile:[3,2,1,""],setSelectorType:[3,2,1,""]},"pyspark.mllib.feature.ChiSqSelectorModel":{transform:[3,2,1,""]},"pyspark.mllib.feature.ElementwiseProduct":{transform:[3,2,1,""]},"pyspark.mllib.feature.HashingTF":{indexOf:[3,2,1,""],setBinary:[3,2,1,""],transform:[3,2,1,""]},"pyspark.mllib.feature.IDF":{fi
t:[3,2,1,""]},"pyspark.mllib.feature.IDFModel":{idf:[3,2,1,""],transform:[3,2,1,""]},"pyspark.mllib.feature.Normalizer":{transform:[3,2,1,""]},"pyspark.mllib.feature.StandardScaler":{fit:[3,2,1,""]},"pyspark.mllib.feature.StandardScalerModel":{mean:[3,3,1,""],setWithMean:[3,2,1,""],setWithStd:[3,2,1,""],std:[3,3,1,""],transform:[3,2,1,""],withMean:[3,3,1,""],withStd:[3,3,1,""]},"pyspark.mllib.feature.Word2Vec":{fit:[3,2,1,""],setLearningRate:[3,2,1,""],setMinCount:[3,2,1,""],setNumIterations:[3,2,1,""],setNumPartitions:[3,2,1,""],setSeed:[3,2,1,""],setVectorSize:[3,2,1,""],setWindowSize:[3,2,1,""]},"pyspark.mllib.feature.Word2VecModel":{findSynonyms:[3,2,1,""],getVectors:[3,2,1,""],load:[3,4,1,""],transform:[3,2,1,""]},"pyspark.mllib.fpm":{FPGrowth:[3,1,1,""],FPGrowthModel:[3,1,1,""],PrefixSpan:[3,1,1,""],PrefixSpanModel:[3,1,1,""]},"pyspark.mllib.fpm.FPGrowth":{FreqItemset:[3,1,1,""],train:[3,4,1,""]},"pyspark.mllib.fpm.FPGrowthModel":{freqItemsets:[3,2,1,""],load:[3,4,1,""]},"pysp
ark.mllib.fpm.PrefixSpan":{FreqSequence:[3,1,1,""],train:[3,4,1,""]},"pyspark.mllib.fpm.PrefixSpanModel":{freqSequences:[3,2,1,""]},"pyspark.mllib.linalg":{DenseMatrix:[3,1,1,""],DenseVector:[3,1,1,""],Matrices:[3,1,1,""],Matrix:[3,1,1,""],QRDecomposition:[3,1,1,""],SparseMatrix:[3,1,1,""],SparseVector:[3,1,1,""],Vector:[3,1,1,""],Vectors:[3,1,1,""],distributed:[3,0,0,"-"]},"pyspark.mllib.linalg.DenseMatrix":{asML:[3,2,1,""],toArray:[3,2,1,""],toSparse:[3,2,1,""]},"pyspark.mllib.linalg.DenseVector":{asML:[3,2,1,""],dot:[3,2,1,""],norm:[3,2,1,""],numNonzeros:[3,2,1,""],parse:[3,5,1,""],squared_distance:[3,2,1,""],toArray:[3,2,1,""],values:[3,3,1,""]},"pyspark.mllib.linalg.Matrices":{dense:[3,5,1,""],fromML:[3,5,1,""],sparse:[3,5,1,""]},"pyspark.mllib.linalg.Matrix":{asML:[3,2,1,""],toArray:[3,2,1,""]},"pyspark.mllib.linalg.QRDecomposition":{Q:[3,3,1,""],R:[3,3,1,""]},"pyspark.mllib.linalg.SparseMatrix":{asML:[3,2,1,""],toArray:[3,2,1,""],toDense:[3,2,1,""]},"pyspark.mllib.linalg.Spar
seVector":{asML:[3,2,1,""],dot:[3,2,1,""],indices:[3,3,1,""],norm:[3,2,1,""],numNonzeros:[3,2,1,""],parse:[3,5,1,""],size:[3,3,1,""],squared_distance:[3,2,1,""],toArray:[3,2,1,""],values:[3,3,1,""]},"pyspark.mllib.linalg.Vector":{asML:[3,2,1,""],toArray:[3,2,1,""]},"pyspark.mllib.linalg.Vectors":{dense:[3,5,1,""],fromML:[3,5,1,""],norm:[3,5,1,""],parse:[3,5,1,""],sparse:[3,5,1,""],squared_distance:[3,5,1,""],stringify:[3,5,1,""],zeros:[3,5,1,""]},"pyspark.mllib.linalg.distributed":{BlockMatrix:[3,1,1,""],CoordinateMatrix:[3,1,1,""],DistributedMatrix:[3,1,1,""],IndexedRow:[3,1,1,""],IndexedRowMatrix:[3,1,1,""],MatrixEntry:[3,1,1,""],RowMatrix:[3,1,1,""]},"pyspark.mllib.linalg.distributed.BlockMatrix":{add:[3,2,1,""],blocks:[3,3,1,""],cache:[3,2,1,""],colsPerBlock:[3,3,1,""],multiply:[3,2,1,""],numColBlocks:[3,3,1,""],numCols:[3,2,1,""],numRowBlocks:[3,3,1,""],numRows:[3,2,1,""],persist:[3,2,1,""],rowsPerBlock:[3,3,1,""],subtract:[3,2,1,""],toCoordinateMatrix:[3,2,1,""],toIndexedRowMa
trix:[3,2,1,""],toLocalMatrix:[3,2,1,""],transpose:[3,2,1,""],validate:[3,2,1,""]},"pyspark.mllib.linalg.distributed.CoordinateMatrix":{entries:[3,3,1,""],numCols:[3,2,1,""],numRows:[3,2,1,""],toBlockMatrix:[3,2,1,""],toIndexedRowMatrix:[3,2,1,""],toRowMatrix:[3,2,1,""],transpose:[3,2,1,""]},"pyspark.mllib.linalg.distributed.DistributedMatrix":{numCols:[3,2,1,""],numRows:[3,2,1,""]},"pyspark.mllib.linalg.distributed.IndexedRowMatrix":{columnSimilarities:[3,2,1,""],computeGramianMatrix:[3,2,1,""],numCols:[3,2,1,""],numRows:[3,2,1,""],rows:[3,3,1,""],toBlockMatrix:[3,2,1,""],toCoordinateMatrix:[3,2,1,""],toRowMatrix:[3,2,1,""]},"pyspark.mllib.linalg.distributed.RowMatrix":{columnSimilarities:[3,2,1,""],computeColumnSummaryStatistics:[3,2,1,""],computeCovariance:[3,2,1,""],computeGramianMatrix:[3,2,1,""],numCols:[3,2,1,""],numRows:[3,2,1,""],rows:[3,3,1,""],tallSkinnyQR:[3,2,1,""]},"pyspark.mllib.random":{RandomRDDs:[3,1,1,""]},"pyspark.mllib.random.RandomRDDs":{exponentialRDD:[3,5,1,"
"],exponentialVectorRDD:[3,5,1,""],gammaRDD:[3,5,1,""],gammaVectorRDD:[3,5,1,""],logNormalRDD:[3,5,1,""],logNormalVectorRDD:[3,5,1,""],normalRDD:[3,5,1,""],normalVectorRDD:[3,5,1,""],poissonRDD:[3,5,1,""],poissonVectorRDD:[3,5,1,""],uniformRDD:[3,5,1,""],uniformVectorRDD:[3,5,1,""]},"pyspark.mllib.recommendation":{ALS:[3,1,1,""],MatrixFactorizationModel:[3,1,1,""],Rating:[3,1,1,""]},"pyspark.mllib.recommendation.ALS":{train:[3,4,1,""],trainImplicit:[3,4,1,""]},"pyspark.mllib.recommendation.MatrixFactorizationModel":{load:[3,4,1,""],predict:[3,2,1,""],predictAll:[3,2,1,""],productFeatures:[3,2,1,""],rank:[3,3,1,""],recommendProducts:[3,2,1,""],recommendProductsForUsers:[3,2,1,""],recommendUsers:[3,2,1,""],recommendUsersForProducts:[3,2,1,""],userFeatures:[3,2,1,""]},"pyspark.mllib.regression":{IsotonicRegression:[3,1,1,""],IsotonicRegressionModel:[3,1,1,""],LabeledPoint:[3,1,1,""],LassoModel:[3,1,1,""],LassoWithSGD:[3,1,1,""],LinearModel:[3,1,1,""],LinearRegressionModel:[3,1,1,""],Li
nearRegressionWithSGD:[3,1,1,""],RidgeRegressionModel:[3,1,1,""],RidgeRegressionWithSGD:[3,1,1,""],StreamingLinearAlgorithm:[3,1,1,""],StreamingLinearRegressionWithSGD:[3,1,1,""]},"pyspark.mllib.regression.IsotonicRegression":{train:[3,4,1,""]},"pyspark.mllib.regression.IsotonicRegressionModel":{load:[3,4,1,""],predict:[3,2,1,""],save:[3,2,1,""]},"pyspark.mllib.regression.LassoModel":{intercept:[3,3,1,""],load:[3,4,1,""],predict:[3,2,1,""],save:[3,2,1,""],weights:[3,3,1,""]},"pyspark.mllib.regression.LassoWithSGD":{train:[3,4,1,""]},"pyspark.mllib.regression.LinearModel":{intercept:[3,3,1,""],weights:[3,3,1,""]},"pyspark.mllib.regression.LinearRegressionModel":{intercept:[3,3,1,""],load:[3,4,1,""],predict:[3,2,1,""],save:[3,2,1,""],weights:[3,3,1,""]},"pyspark.mllib.regression.LinearRegressionWithSGD":{train:[3,4,1,""]},"pyspark.mllib.regression.RidgeRegressionModel":{intercept:[3,3,1,""],load:[3,4,1,""],predict:[3,2,1,""],save:[3,2,1,""],weights:[3,3,1,""]},"pyspark.mllib.regressio
n.RidgeRegressionWithSGD":{train:[3,4,1,""]},"pyspark.mllib.regression.StreamingLinearAlgorithm":{latestModel:[3,2,1,""],predictOn:[3,2,1,""],predictOnValues:[3,2,1,""]},"pyspark.mllib.regression.StreamingLinearRegressionWithSGD":{latestModel:[3,2,1,""],predictOn:[3,2,1,""],predictOnValues:[3,2,1,""],setInitialWeights:[3,2,1,""],trainOn:[3,2,1,""]},"pyspark.mllib.stat":{ChiSqTestResult:[3,1,1,""],KernelDensity:[3,1,1,""],MultivariateGaussian:[3,1,1,""],MultivariateStatisticalSummary:[3,1,1,""],Statistics:[3,1,1,""]},"pyspark.mllib.stat.ChiSqTestResult":{method:[3,3,1,""]},"pyspark.mllib.stat.KernelDensity":{estimate:[3,2,1,""],setBandwidth:[3,2,1,""],setSample:[3,2,1,""]},"pyspark.mllib.stat.MultivariateStatisticalSummary":{count:[3,2,1,""],max:[3,2,1,""],mean:[3,2,1,""],min:[3,2,1,""],normL1:[3,2,1,""],normL2:[3,2,1,""],numNonzeros:[3,2,1,""],variance:[3,2,1,""]},"pyspark.mllib.stat.Statistics":{chiSqTest:[3,5,1,""],colStats:[3,5,1,""],corr:[3,5,1,""],kolmogorovSmirnovTest:[3,5,1,"
"]},"pyspark.mllib.tree":{DecisionTree:[3,1,1,""],DecisionTreeModel:[3,1,1,""],GradientBoostedTrees:[3,1,1,""],GradientBoostedTreesModel:[3,1,1,""],RandomForest:[3,1,1,""],RandomForestModel:[3,1,1,""]},"pyspark.mllib.tree.DecisionTree":{trainClassifier:[3,4,1,""],trainRegressor:[3,4,1,""]},"pyspark.mllib.tree.DecisionTreeModel":{call:[3,2,1,""],depth:[3,2,1,""],load:[3,2,1,""],numNodes:[3,2,1,""],predict:[3,2,1,""],save:[3,2,1,""],toDebugString:[3,2,1,""]},"pyspark.mllib.tree.GradientBoostedTrees":{trainClassifier:[3,4,1,""],trainRegressor:[3,4,1,""]},"pyspark.mllib.tree.GradientBoostedTreesModel":{call:[3,2,1,""],load:[3,2,1,""],numTrees:[3,2,1,""],predict:[3,2,1,""],save:[3,2,1,""],toDebugString:[3,2,1,""],totalNumNodes:[3,2,1,""]},"pyspark.mllib.tree.RandomForest":{supportedFeatureSubsetStrategies:[3,3,1,""],trainClassifier:[3,4,1,""],trainRegressor:[3,4,1,""]},"pyspark.mllib.tree.RandomForestModel":{call:[3,2,1,""],load:[3,2,1,""],numTrees:[3,2,1,""],predict:[3,2,1,""],save:[3,2
,1,""],toDebugString:[3,2,1,""],totalNumNodes:[3,2,1,""]},"pyspark.mllib.util":{JavaLoader:[3,1,1,""],JavaSaveable:[3,1,1,""],LinearDataGenerator:[3,1,1,""],Loader:[3,1,1,""],MLUtils:[3,1,1,""],Saveable:[3,1,1,""]},"pyspark.mllib.util.JavaLoader":{load:[3,4,1,""]},"pyspark.mllib.util.JavaSaveable":{save:[3,2,1,""]},"pyspark.mllib.util.
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[12/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/param/shared.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/param/shared.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/param/shared.html
index f2d428f..60e1169 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/param/shared.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/param/shared.html
@@ -77,7 +77,7 @@
<span class="n">maxIter</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"maxIter"</span><span class="p">,</span> <span class="s2">"max number of iterations (>= 0)."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toInt</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasMaxIter</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasMaxIter</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setMaxIter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -100,7 +100,7 @@
<span class="n">regParam</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"regParam"</span><span class="p">,</span> <span class="s2">"regularization parameter (>= 0)."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toFloat</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasRegParam</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasRegParam</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setRegParam</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -123,7 +123,7 @@
<span class="n">featuresCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"featuresCol"</span><span class="p">,</span> <span class="s2">"features column name."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasFeaturesCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasFeaturesCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">featuresCol</span><span class="o">=</span><span class="s1">'features'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setFeaturesCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -147,7 +147,7 @@
<span class="n">labelCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"labelCol"</span><span class="p">,</span> <span class="s2">"label column name."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasLabelCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasLabelCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">labelCol</span><span class="o">=</span><span class="s1">'label'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setLabelCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -171,7 +171,7 @@
<span class="n">predictionCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"predictionCol"</span><span class="p">,</span> <span class="s2">"prediction column name."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasPredictionCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasPredictionCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">predictionCol</span><span class="o">=</span><span class="s1">'prediction'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setPredictionCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -195,7 +195,7 @@
<span class="n">probabilityCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"probabilityCol"</span><span class="p">,</span> <span class="s2">"Column name for predicted class conditional probabilities. Note: Not all models output well-calibrated probability estimates! These probabilities should be treated as confidences, not precise probabilities."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasProbabilityCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasProbabilityCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">probabilityCol</span><span class="o">=</span><span class="s1">'probability'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setProbabilityCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -219,7 +219,7 @@
<span class="n">rawPredictionCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"rawPredictionCol"</span><span class="p">,</span> <span class="s2">"raw prediction (a.k.a. confidence) column name."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasRawPredictionCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasRawPredictionCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">rawPredictionCol</span><span class="o">=</span><span class="s1">'rawPrediction'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setRawPredictionCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -243,7 +243,7 @@
<span class="n">inputCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"inputCol"</span><span class="p">,</span> <span class="s2">"input column name."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasInputCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasInputCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setInputCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -266,7 +266,7 @@
<span class="n">inputCols</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"inputCols"</span><span class="p">,</span> <span class="s2">"input column names."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toListString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasInputCols</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasInputCols</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setInputCols</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -289,7 +289,7 @@
<span class="n">outputCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"outputCol"</span><span class="p">,</span> <span class="s2">"output column name."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasOutputCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasOutputCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">outputCol</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">uid</span> <span class="o">+</span> <span class="s1">'__output'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setOutputCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -313,7 +313,7 @@
<span class="n">numFeatures</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"numFeatures"</span><span class="p">,</span> <span class="s2">"number of features."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toInt</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasNumFeatures</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasNumFeatures</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setNumFeatures</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -336,7 +336,7 @@
<span class="n">checkpointInterval</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"checkpointInterval"</span><span class="p">,</span> <span class="s2">"set checkpoint interval (>= 1) or disable checkpoint (-1). E.g. 10 means that the cache will get checkpointed every 10 iterations."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toInt</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasCheckpointInterval</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasCheckpointInterval</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setCheckpointInterval</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -359,8 +359,8 @@
<span class="n">seed</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"seed"</span><span class="p">,</span> <span class="s2">"random seed."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toInt</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasSeed</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">seed</span><span class="o">=</span><span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__name__</span><span class="p">))</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasSeed</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">seed</span><span class="o">=</span><span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">setSeed</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -383,7 +383,7 @@
<span class="n">tol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"tol"</span><span class="p">,</span> <span class="s2">"the convergence tolerance for iterative algorithms (>= 0)."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toFloat</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasTol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasTol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setTol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -406,7 +406,7 @@
<span class="n">stepSize</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"stepSize"</span><span class="p">,</span> <span class="s2">"Step size to be used for each iteration of optimization (>= 0)."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toFloat</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasStepSize</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasStepSize</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setStepSize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -429,7 +429,7 @@
<span class="n">handleInvalid</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"handleInvalid"</span><span class="p">,</span> <span class="s2">"how to handle invalid entries. Options are skip (which will filter out rows with bad values), or error (which will throw an error). More options may be added later."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasHandleInvalid</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasHandleInvalid</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setHandleInvalid</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -452,7 +452,7 @@
<span class="n">elasticNetParam</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"elasticNetParam"</span><span class="p">,</span> <span class="s2">"the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toFloat</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasElasticNetParam</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasElasticNetParam</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setElasticNetParam</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -476,7 +476,7 @@
<span class="n">fitIntercept</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"fitIntercept"</span><span class="p">,</span> <span class="s2">"whether to fit an intercept term."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toBoolean</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasFitIntercept</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasFitIntercept</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setFitIntercept</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -500,7 +500,7 @@
<span class="n">standardization</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"standardization"</span><span class="p">,</span> <span class="s2">"whether to standardize the training features before fitting the model."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toBoolean</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasStandardization</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasStandardization</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">standardization</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setStandardization</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -524,7 +524,7 @@
<span class="n">thresholds</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"thresholds"</span><span class="p">,</span> <span class="s2">"Thresholds in multi-class classification to adjust the probability of predicting each class. Array must have length equal to the number of classes, with values > 0, excepting that at most one value may be 0. The class with largest value p/t is predicted, where p is the original probability of that class and t is the class's threshold."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toListFloat</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasThresholds</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasThresholds</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setThresholds</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -547,7 +547,7 @@
<span class="n">weightCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"weightCol"</span><span class="p">,</span> <span class="s2">"weight column name. If this is not set or empty, we treat all instance weights as 1.0."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasWeightCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasWeightCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setWeightCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -570,7 +570,7 @@
<span class="n">solver</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"solver"</span><span class="p">,</span> <span class="s2">"the solver algorithm for optimization. If this is not set or empty, default value is 'auto'."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasSolver</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasSolver</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">solver</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setSolver</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -594,7 +594,7 @@
<span class="n">varianceCol</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"varianceCol"</span><span class="p">,</span> <span class="s2">"column name for the biased sample variance of prediction."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasVarianceCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasVarianceCol</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setVarianceCol</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -617,7 +617,7 @@
<span class="n">aggregationDepth</span> <span class="o">=</span> <span class="n">Param</span><span class="p">(</span><span class="n">Params</span><span class="o">.</span><span class="n">_dummy</span><span class="p">(),</span> <span class="s2">"aggregationDepth"</span><span class="p">,</span> <span class="s2">"suggested depth for treeAggregate (>= 2)."</span><span class="p">,</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toInt</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HasAggregationDepth</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HasAggregationDepth</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">aggregationDepth</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setAggregationDepth</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -647,7 +647,7 @@
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">DecisionTreeParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DecisionTreeParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">setMaxDepth</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/pipeline.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/pipeline.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/pipeline.html
index c67138a..9abb3d8 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/pipeline.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/pipeline.html
@@ -105,7 +105,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, stages=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">Pipeline</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">Pipeline</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -187,18 +187,18 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="Pipeline.read"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.Pipeline.read">[docs]</a> <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+<div class="viewcode-block" id="Pipeline.read"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.Pipeline.read">[docs]</a> <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""Returns an MLReader instance for this class."""</span>
- <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="n">cls</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_from_java</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">java_stage</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_from_java</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">java_stage</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Given a Java Pipeline, create and return a Python wrapper of it.</span>
<span class="sd"> Used for ML persistence.</span>
<span class="sd"> """</span>
<span class="c1"># Create a new instance of this stage.</span>
- <span class="n">py_stage</span> <span class="o">=</span> <span class="n">cls</span><span class="p">()</span>
+ <span class="n">py_stage</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">()</span>
<span class="c1"># Load information from java_stage to the instance.</span>
<span class="n">py_stages</span> <span class="o">=</span> <span class="p">[</span><span class="n">JavaParams</span><span class="o">.</span><span class="n">_from_java</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">java_stage</span><span class="o">.</span><span class="n">getStages</span><span class="p">()]</span>
<span class="n">py_stage</span><span class="o">.</span><span class="n">setStages</span><span class="p">(</span><span class="n">py_stages</span><span class="p">)</span>
@@ -213,8 +213,8 @@
<span class="sd"> """</span>
<span class="n">gateway</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_gateway</span>
- <span class="n">cls</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">PipelineStage</span>
- <span class="n">java_stages</span> <span class="o">=</span> <span class="n">gateway</span><span class="o">.</span><span class="n">new_array</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">getStages</span><span class="p">()))</span>
+ <span class="bp">cls</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">PipelineStage</span>
+ <span class="n">java_stages</span> <span class="o">=</span> <span class="n">gateway</span><span class="o">.</span><span class="n">new_array</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">getStages</span><span class="p">()))</span>
<span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">stage</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">getStages</span><span class="p">()):</span>
<span class="n">java_stages</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span> <span class="o">=</span> <span class="n">stage</span><span class="o">.</span><span class="n">_to_java</span><span class="p">()</span>
@@ -233,7 +233,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">stages</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">PipelineModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PipelineModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">stages</span> <span class="o">=</span> <span class="n">stages</span>
<span class="k">def</span> <span class="nf">_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">):</span>
@@ -266,12 +266,12 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="PipelineModel.read"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.PipelineModel.read">[docs]</a> <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+<div class="viewcode-block" id="PipelineModel.read"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.PipelineModel.read">[docs]</a> <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""Returns an MLReader instance for this class."""</span>
- <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="n">cls</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_from_java</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">java_stage</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_from_java</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">java_stage</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Given a Java PipelineModel, create and return a Python wrapper of it.</span>
<span class="sd"> Used for ML persistence.</span>
@@ -279,7 +279,7 @@
<span class="c1"># Load information from java_stage to the instance.</span>
<span class="n">py_stages</span> <span class="o">=</span> <span class="p">[</span><span class="n">JavaParams</span><span class="o">.</span><span class="n">_from_java</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">java_stage</span><span class="o">.</span><span class="n">stages</span><span class="p">()]</span>
<span class="c1"># Create a new instance of this stage.</span>
- <span class="n">py_stage</span> <span class="o">=</span> <span class="n">cls</span><span class="p">(</span><span class="n">py_stages</span><span class="p">)</span>
+ <span class="n">py_stage</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">py_stages</span><span class="p">)</span>
<span class="n">py_stage</span><span class="o">.</span><span class="n">_resetUid</span><span class="p">(</span><span class="n">java_stage</span><span class="o">.</span><span class="n">uid</span><span class="p">())</span>
<span class="k">return</span> <span class="n">py_stage</span>
@@ -291,8 +291,8 @@
<span class="sd"> """</span>
<span class="n">gateway</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_gateway</span>
- <span class="n">cls</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">Transformer</span>
- <span class="n">java_stages</span> <span class="o">=</span> <span class="n">gateway</span><span class="o">.</span><span class="n">new_array</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">stages</span><span class="p">))</span>
+ <span class="bp">cls</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">Transformer</span>
+ <span class="n">java_stages</span> <span class="o">=</span> <span class="n">gateway</span><span class="o">.</span><span class="n">new_array</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">stages</span><span class="p">))</span>
<span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">stage</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">stages</span><span class="p">):</span>
<span class="n">java_stages</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span> <span class="o">=</span> <span class="n">stage</span><span class="o">.</span><span class="n">_to_java</span><span class="p">()</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/recommendation.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/recommendation.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/recommendation.html
index 418bda9..341a456 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/recommendation.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/recommendation.html
@@ -187,7 +187,7 @@
<span class="sd"> intermediateStorageLevel="MEMORY_AND_DISK", \</span>
<span class="sd"> finalStorageLevel="MEMORY_AND_DISK")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">ALS</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">ALS</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.recommendation.ALS"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">rank</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">numUserBlocks</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">numItemBlocks</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
<span class="n">implicitPrefs</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">userCol</span><span class="o">=</span><span class="s2">"user"</span><span class="p">,</span> <span class="n">itemCol</span><span class="o">=</span><span class="s2">"item"</span><span class="p">,</span>
@@ -413,11 +413,11 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call_java</span><span class="p">(</span><span class="s2">"itemFactors"</span><span class="p">)</span></div>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">import</span> <span class="nn">pyspark.ml.recommendation</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">recommendation</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">recommendation</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="c1"># The small batch size here ensures that we see multiple batches,</span>
<span class="c1"># even in these small test examples:</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[11/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/regression.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/regression.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/regression.html
index f3d5915..4f73b9e 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/regression.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/regression.html
@@ -145,24 +145,24 @@
<span class="nd">@keyword_only</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">standardization</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span> <span class="n">weightCol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">aggregationDepth</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \</span>
<span class="sd"> maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \</span>
<span class="sd"> standardization=True, solver="auto", weightCol=None, aggregationDepth=2)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">LinearRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">LinearRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.regression.LinearRegression"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="nd">@keyword_only</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
<div class="viewcode-block" id="LinearRegression.setParams"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.LinearRegression.setParams">[docs]</a> <span class="k">def</span> <span class="nf">setParams</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">standardization</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span> <span class="n">weightCol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">aggregationDepth</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \</span>
@@ -213,7 +213,7 @@
<span class="k">return</span> <span class="n">LinearRegressionTrainingSummary</span><span class="p">(</span><span class="n">java_lrt_summary</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"No training summary available for this </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
@@ -508,7 +508,7 @@
<span class="sd"> __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \</span>
<span class="sd"> weightCol=None, isotonic=True, featureIndex=0):</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">IsotonicRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">IsotonicRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.regression.IsotonicRegression"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">isotonic</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">featureIndex</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
@@ -589,7 +589,7 @@
<span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toFloat</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">TreeEnsembleParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">TreeEnsembleParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setSubsamplingRate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -618,7 +618,7 @@
<span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">supportedImpurities</span><span class="p">),</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">TreeRegressorParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">TreeRegressorParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setImpurity</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -650,7 +650,7 @@
<span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">RandomForestParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RandomForestParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setNumTrees</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -746,7 +746,7 @@
<span class="sd"> maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \</span>
<span class="sd"> impurity="variance", seed=None, varianceCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">DecisionTreeRegressor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DecisionTreeRegressor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.regression.DecisionTreeRegressor"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
@@ -936,7 +936,7 @@
<span class="sd"> impurity="variance", subsamplingRate=1.0, seed=None, numTrees=20, \</span>
<span class="sd"> featureSubsetStrategy="auto")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">RandomForestRegressor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RandomForestRegressor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.regression.RandomForestRegressor"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
@@ -1064,7 +1064,7 @@
<span class="sd"> checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1, seed=None, \</span>
<span class="sd"> impurity="variance")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">GBTRegressor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GBTRegressor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.regression.GBTRegressor"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
<span class="n">maxMemoryInMB</span><span class="o">=</span><span class="mi">256</span><span class="p">,</span> <span class="n">cacheNodeIds</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">subsamplingRate</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
@@ -1204,7 +1204,7 @@
<span class="nd">@keyword_only</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">E</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">censorCol</span><span class="o">=</span><span class="s2">"censor"</span><span class="p">,</span>
+ <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1E-6</span><span class="p">,</span> <span class="n">censorCol</span><span class="o">=</span><span class="s2">"censor"</span><span class="p">,</span>
<span class="n">quantileProbabilities</span><span class="o">=</span><span class="nb">list</span><span class="p">([</span><span class="mf">0.01</span><span class="p">,</span> <span class="mf">0.05</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.95</span><span class="p">,</span> <span class="mf">0.99</span><span class="p">]),</span>
<span class="n">quantilesCol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">aggregationDepth</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -1213,19 +1213,19 @@
<span class="sd"> quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], \</span>
<span class="sd"> quantilesCol=None, aggregationDepth=2)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">AFTSurvivalRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AFTSurvivalRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.regression.AFTSurvivalRegression"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">censorCol</span><span class="o">=</span><span class="s2">"censor"</span><span class="p">,</span>
<span class="n">quantileProbabilities</span><span class="o">=</span><span class="p">[</span><span class="mf">0.01</span><span class="p">,</span> <span class="mf">0.05</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.95</span><span class="p">,</span> <span class="mf">0.99</span><span class="p">],</span>
- <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">E</span><span class="o">-</span><span class="mi">6</span><span class="p">)</span>
+ <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1E-6</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="nd">@keyword_only</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.6.0"</span><span class="p">)</span>
<div class="viewcode-block" id="AFTSurvivalRegression.setParams"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.AFTSurvivalRegression.setParams">[docs]</a> <span class="k">def</span> <span class="nf">setParams</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">E</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">censorCol</span><span class="o">=</span><span class="s2">"censor"</span><span class="p">,</span>
+ <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1E-6</span><span class="p">,</span> <span class="n">censorCol</span><span class="o">=</span><span class="s2">"censor"</span><span class="p">,</span>
<span class="n">quantileProbabilities</span><span class="o">=</span><span class="nb">list</span><span class="p">([</span><span class="mf">0.01</span><span class="p">,</span> <span class="mf">0.05</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.95</span><span class="p">,</span> <span class="mf">0.99</span><span class="p">]),</span>
<span class="n">quantilesCol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">aggregationDepth</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -1403,24 +1403,24 @@
<span class="nd">@keyword_only</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">family</span><span class="o">=</span><span class="s2">"gaussian"</span><span class="p">,</span> <span class="n">link</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span>
+ <span class="n">family</span><span class="o">=</span><span class="s2">"gaussian"</span><span class="p">,</span> <span class="n">link</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span>
<span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">weightCol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"irls"</span><span class="p">,</span> <span class="n">linkPredictionCol</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> __init__(self, labelCol="label", featuresCol="features", predictionCol="prediction", \</span>
<span class="sd"> family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6, \</span>
<span class="sd"> regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">GeneralizedLinearRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GeneralizedLinearRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.regression.GeneralizedLinearRegression"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">family</span><span class="o">=</span><span class="s2">"gaussian"</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"irls"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">family</span><span class="o">=</span><span class="s2">"gaussian"</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"irls"</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="nd">@keyword_only</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
<div class="viewcode-block" id="GeneralizedLinearRegression.setParams"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.GeneralizedLinearRegression.setParams">[docs]</a> <span class="k">def</span> <span class="nf">setParams</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">family</span><span class="o">=</span><span class="s2">"gaussian"</span><span class="p">,</span> <span class="n">link</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span>
+ <span class="n">family</span><span class="o">=</span><span class="s2">"gaussian"</span><span class="p">,</span> <span class="n">link</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span>
<span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">weightCol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"irls"</span><span class="p">,</span> <span class="n">linkPredictionCol</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> setParams(self, labelCol="label", featuresCol="features", predictionCol="prediction", \</span>
@@ -1516,7 +1516,7 @@
<span class="k">return</span> <span class="n">GeneralizedLinearRegressionTrainingSummary</span><span class="p">(</span><span class="n">java_glrt_summary</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"No training summary available for this </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
@@ -1706,11 +1706,11 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call_java</span><span class="p">(</span><span class="s2">"pValues"</span><span class="p">)</span></div>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">import</span> <span class="nn">pyspark.ml.regression</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">regression</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">regression</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="c1"># The small batch size here ensures that we see multiple batches,</span>
<span class="c1"># even in these small test examples:</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/tuning.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/tuning.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/tuning.html
index b9d5c1f..c152787 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/tuning.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/tuning.html
@@ -80,7 +80,7 @@
<div class="viewcode-block" id="ParamGridBuilder"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.tuning.ParamGridBuilder">[docs]</a><span class="k">class</span> <span class="nc">ParamGridBuilder</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
- <span class="sd">r"""</span>
+ <span class="sa">r</span><span class="sd">"""</span>
<span class="sd"> Builder for a param grid used in grid search-based model selection.</span>
<span class="sd"> >>> from pyspark.ml.classification import LogisticRegression</span>
@@ -232,7 +232,7 @@
<span class="sd"> __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\</span>
<span class="sd"> seed=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">CrossValidator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CrossValidator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">numFolds</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_set</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -325,7 +325,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bestModel</span><span class="p">,</span> <span class="n">avgMetrics</span><span class="o">=</span><span class="p">[]):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">CrossValidatorModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CrossValidatorModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="c1">#: best model from cross validation</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bestModel</span> <span class="o">=</span> <span class="n">bestModel</span>
<span class="c1">#: Average cross-validation metrics for each paramMap in</span>
@@ -392,7 +392,7 @@
<span class="sd"> __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,\</span>
<span class="sd"> seed=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">TrainValidationSplit</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">TrainValidationSplit</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">trainRatio</span><span class="o">=</span><span class="mf">0.75</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_set</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -478,7 +478,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bestModel</span><span class="p">,</span> <span class="n">validationMetrics</span><span class="o">=</span><span class="p">[]):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">TrainValidationSplitModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">TrainValidationSplitModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="c1">#: best model from cross validation</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bestModel</span> <span class="o">=</span> <span class="n">bestModel</span>
<span class="c1">#: evaluated validation metrics</span>
@@ -506,7 +506,7 @@
<span class="k">return</span> <span class="n">TrainValidationSplitModel</span><span class="p">(</span><span class="n">bestModel</span><span class="p">,</span> <span class="n">validationMetrics</span><span class="p">)</span></div></div>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/util.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/util.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/util.html
index d07cd84..cda1458 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/util.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/util.html
@@ -100,12 +100,12 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_randomUID</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_randomUID</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Generate a unique unicode id for the object. The default implementation</span>
<span class="sd"> concatenates the class name, "_", and 12 random hex chars.</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="n">unicode</span><span class="p">(</span><span class="n">cls</span><span class="o">.</span><span class="n">__name__</span> <span class="o">+</span> <span class="s2">"_"</span> <span class="o">+</span> <span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">()</span><span class="o">.</span><span class="n">hex</span><span class="p">[</span><span class="mi">12</span><span class="p">:])</span>
+ <span class="k">return</span> <span class="n">unicode</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">+</span> <span class="s2">"_"</span> <span class="o">+</span> <span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">()</span><span class="o">.</span><span class="n">hex</span><span class="p">[</span><span class="mi">12</span><span class="p">:])</span>
<span class="nd">@inherit_doc</span>
@@ -143,7 +143,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">instance</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">JavaMLWriter</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">JavaMLWriter</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="n">_java_obj</span> <span class="o">=</span> <span class="n">instance</span><span class="o">.</span><span class="n">_to_java</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_jwrite</span> <span class="o">=</span> <span class="n">_java_obj</span><span class="o">.</span><span class="n">write</span><span class="p">()</span>
@@ -260,22 +260,22 @@
<span class="k">return</span> <span class="bp">self</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">clazz</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">clazz</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns the full class name of the Java ML instance. The default</span>
<span class="sd"> implementation replaces "pyspark" by "org.apache.spark" in</span>
<span class="sd"> the Python full class name.</span>
<span class="sd"> """</span>
- <span class="n">java_package</span> <span class="o">=</span> <span class="n">clazz</span><span class="o">.</span><span class="n">__module__</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"pyspark"</span><span class="p">,</span> <span class="s2">"org.apache.spark"</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">clazz</span><span class="o">.</span><span class="n">__name__</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"Pipeline"</span><span class="p">,</span> <span class="s2">"PipelineModel"</span><span class="p">):</span>
+ <span class="n">java_package</span> <span class="o">=</span> <span class="n">clazz</span><span class="o">.</span><span class="vm">__module__</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"pyspark"</span><span class="p">,</span> <span class="s2">"org.apache.spark"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">clazz</span><span class="o">.</span><span class="vm">__name__</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"Pipeline"</span><span class="p">,</span> <span class="s2">"PipelineModel"</span><span class="p">):</span>
<span class="c1"># Remove the last package name "pipeline" for Pipeline and PipelineModel.</span>
<span class="n">java_package</span> <span class="o">=</span> <span class="s2">"."</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">java_package</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"."</span><span class="p">)[</span><span class="mi">0</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
- <span class="k">return</span> <span class="n">java_package</span> <span class="o">+</span> <span class="s2">"."</span> <span class="o">+</span> <span class="n">clazz</span><span class="o">.</span><span class="n">__name__</span>
+ <span class="k">return</span> <span class="n">java_package</span> <span class="o">+</span> <span class="s2">"."</span> <span class="o">+</span> <span class="n">clazz</span><span class="o">.</span><span class="vm">__name__</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_load_java_obj</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">clazz</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_load_java_obj</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">clazz</span><span class="p">):</span>
<span class="sd">"""Load the peer Java object of the ML instance."""</span>
- <span class="n">java_class</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_java_loader_class</span><span class="p">(</span><span class="n">clazz</span><span class="p">)</span>
+ <span class="n">java_class</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_java_loader_class</span><span class="p">(</span><span class="n">clazz</span><span class="p">)</span>
<span class="n">java_obj</span> <span class="o">=</span> <span class="n">_jvm</span><span class="p">()</span>
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">java_class</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"."</span><span class="p">):</span>
<span class="n">java_obj</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">java_obj</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
@@ -291,14 +291,14 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""Returns an MLReader instance for this class."""</span>
- <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"MLReadable.read() not implemented for type: </span><span class="si">%r</span><span class="s2">"</span> <span class="o">%</span> <span class="n">cls</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"MLReadable.read() not implemented for type: </span><span class="si">%r</span><span class="s2">"</span> <span class="o">%</span> <span class="bp">cls</span><span class="p">)</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Reads an ML instance from the input path, a shortcut of `read().load(path)`."""</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
<span class="nd">@inherit_doc</span>
@@ -308,9 +308,9 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""Returns an MLReader instance for this class."""</span>
- <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="n">cls</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span>
<span class="nd">@inherit_doc</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/wrapper.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/wrapper.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/wrapper.html
index 036d53b..e1abca7 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/wrapper.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/wrapper.html
@@ -78,16 +78,16 @@
<span class="sd"> Wrapper class for a Java companion object</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">java_obj</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">JavaWrapper</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">JavaWrapper</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="n">java_obj</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_create_from_java_class</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">java_class</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_create_from_java_class</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">java_class</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Construct this object from given Java classname and arguments</span>
<span class="sd"> """</span>
<span class="n">java_obj</span> <span class="o">=</span> <span class="n">JavaWrapper</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="n">java_class</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">cls</span><span class="p">(</span><span class="n">java_obj</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">java_obj</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_call_java</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
<span class="n">m</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
@@ -324,7 +324,7 @@
<span class="sd"> these wrappers depend on pyspark.ml.util (both directly and via</span>
<span class="sd"> other ML classes).</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">JavaModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">JavaModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
<span class="k">if</span> <span class="n">java_model</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_resetUid</span><span class="p">(</span><span class="n">java_model</span><span class="o">.</span><span class="n">uid</span><span class="p">())</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/classification.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/classification.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/classification.html
index 9c732df..77aebbd 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/classification.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/classification.html
@@ -90,7 +90,7 @@
<span class="sd"> model. The categories are represented by int values: 0, 1, 2, etc.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">LinearClassificationModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">LinearClassificationModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_threshold</span> <span class="o">=</span> <span class="kc">None</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
@@ -211,7 +211,7 @@
<span class="sd"> .. versionadded:: 0.9.0</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">,</span> <span class="n">numFeatures</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">LogisticRegressionModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">LogisticRegressionModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_numFeatures</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">numFeatures</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_numClasses</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">numClasses</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_threshold</span> <span class="o">=</span> <span class="mf">0.5</span>
@@ -290,7 +290,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="LogisticRegressionModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="LogisticRegressionModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a model from the given path.</span>
<span class="sd"> """</span>
@@ -314,7 +314,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'0.9.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="LogisticRegressionWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">miniBatchFraction</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
+<div class="viewcode-block" id="LogisticRegressionWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">miniBatchFraction</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
<span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span> <span class="n">regType</span><span class="o">=</span><span class="s2">"l2"</span><span class="p">,</span> <span class="n">intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">validateData</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">convergenceTol</span><span class="o">=</span><span class="mf">0.001</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -375,8 +375,8 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.2.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="LogisticRegressionWithLBFGS.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionWithLBFGS.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">regType</span><span class="o">=</span><span class="s2">"l2"</span><span class="p">,</span>
- <span class="n">intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">corrections</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">validateData</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">numClasses</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
+<div class="viewcode-block" id="LogisticRegressionWithLBFGS.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionWithLBFGS.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">regType</span><span class="o">=</span><span class="s2">"l2"</span><span class="p">,</span>
+ <span class="n">intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">corrections</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">validateData</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">numClasses</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Train a logistic regression model on the given data.</span>
@@ -499,7 +499,7 @@
<span class="sd"> .. versionadded:: 0.9.0</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">SVMModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SVMModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">weights</span><span class="p">,</span> <span class="n">intercept</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_threshold</span> <span class="o">=</span> <span class="mf">0.0</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'0.9.0'</span><span class="p">)</span>
@@ -529,7 +529,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="SVMModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.SVMModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="SVMModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.SVMModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a model from the given path.</span>
<span class="sd"> """</span>
@@ -550,7 +550,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'0.9.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="SVMWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.SVMWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
+<div class="viewcode-block" id="SVMWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.SVMWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
<span class="n">miniBatchFraction</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">regType</span><span class="o">=</span><span class="s2">"l2"</span><span class="p">,</span>
<span class="n">intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">validateData</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">convergenceTol</span><span class="o">=</span><span class="mf">0.001</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -680,7 +680,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="NaiveBayesModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.NaiveBayesModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="NaiveBayesModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.NaiveBayesModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a model from the given path.</span>
<span class="sd"> """</span>
@@ -700,7 +700,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'0.9.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="NaiveBayes.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.NaiveBayes.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">lambda_</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
+<div class="viewcode-block" id="NaiveBayes.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.NaiveBayes.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">lambda_</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Train a Naive Bayes model given an RDD of (label, features)</span>
<span class="sd"> vectors.</span>
@@ -763,7 +763,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">miniBatchFraction</span> <span class="o">=</span> <span class="n">miniBatchFraction</span>
<span class="bp">self</span><span class="o">.</span><span class="n">convergenceTol</span> <span class="o">=</span> <span class="n">convergenceTol</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_model</span> <span class="o">=</span> <span class="kc">None</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">StreamingLogisticRegressionWithSGD</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">StreamingLogisticRegressionWithSGD</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
<span class="n">model</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_model</span><span class="p">)</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.5.0'</span><span class="p">)</span>
@@ -800,7 +800,7 @@
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
<span class="kn">import</span> <span class="nn">pyspark.mllib.classification</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">classification</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">classification</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
<span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[4]"</span><span class="p">)</span>\
<span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"mllib.classification tests"</span><span class="p">)</span>\
@@ -811,7 +811,7 @@
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[03/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/pyspark.sql.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/pyspark.sql.html b/site/docs/2.1.2/api/python/pyspark.sql.html
index e2fbad9..5220468 100644
--- a/site/docs/2.1.2/api/python/pyspark.sql.html
+++ b/site/docs/2.1.2/api/python/pyspark.sql.html
@@ -213,7 +213,7 @@ cluster.</p>
<dl class="attribute">
<dt id="pyspark.sql.SparkSession.builder">
-<code class="descclassname">SparkSession.</code><code class="descname">builder</code><em class="property"> = <pyspark.sql.session.Builder object></em><a class="headerlink" href="#pyspark.sql.SparkSession.builder" title="Permalink to this definition">¶</a></dt>
+<code class="descclassname">SparkSession.</code><code class="descname">builder</code><em class="property"> = <pyspark.sql.session.SparkSession.Builder object></em><a class="headerlink" href="#pyspark.sql.SparkSession.builder" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>
<dl class="attribute">
@@ -282,22 +282,22 @@ omit the <code class="docutils literal"><span class="pre">struct<></span><
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">l</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">'Alice'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)]</span>
<span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">l</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(_1=u'Alice', _2=1)]</span>
+<span class="go">[Row(_1='Alice', _2=1)]</span>
<span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">l</span><span class="p">,</span> <span class="p">[</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">])</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="p">[{</span><span class="s1">'name'</span><span class="p">:</span> <span class="s1">'Alice'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span>
<span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=1, name=u'Alice')]</span>
+<span class="go">[Row(age=1, name='Alice')]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">rdd</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(_1=u'Alice', _2=1)]</span>
+<span class="go">[Row(_1='Alice', _2=1)]</span>
<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="p">[</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span>
@@ -305,7 +305,7 @@ omit the <code class="docutils literal"><span class="pre">struct<></span><
<span class="gp">>>> </span><span class="n">person</span> <span class="o">=</span> <span class="n">rdd</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">Person</span><span class="p">(</span><span class="o">*</span><span class="n">r</span><span class="p">))</span>
<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">person</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df2</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql.types</span> <span class="k">import</span> <span class="o">*</span>
@@ -314,17 +314,17 @@ omit the <code class="docutils literal"><span class="pre">struct<></span><
<span class="gp">... </span> <span class="n">StructField</span><span class="p">(</span><span class="s2">"age"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="kc">True</span><span class="p">)])</span>
<span class="gp">>>> </span><span class="n">df3</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df3</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">toPandas</span><span class="p">())</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
<span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]]))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
<span class="go">[Row(0=1, 1=2)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="s2">"a: string, b: int"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(a=u'Alice', b=1)]</span>
+<span class="go">[Row(a='Alice', b=1)]</span>
<span class="gp">>>> </span><span class="n">rdd</span> <span class="o">=</span> <span class="n">rdd</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">row</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="s2">"int"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
<span class="go">[Row(value=1)]</span>
@@ -451,7 +451,7 @@ as a streaming <a class="reference internal" href="#pyspark.sql.DataFrame" title
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">createOrReplaceTempView</span><span class="p">(</span><span class="s2">"table1"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT field1 AS f1, field2 as f2 from table1"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df2</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(f1=1, f2=u'row1'), Row(f1=2, f2=u'row2'), Row(f1=3, f2=u'row3')]</span>
+<span class="go">[Row(f1=1, f2='row1'), Row(f1=2, f2='row2'), Row(f1=3, f2='row3')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -632,22 +632,22 @@ If it’s not a <a class="reference internal" href="#pyspark.sql.types.Struc
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">l</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">'Alice'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)]</span>
<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">l</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(_1=u'Alice', _2=1)]</span>
+<span class="go">[Row(_1='Alice', _2=1)]</span>
<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">l</span><span class="p">,</span> <span class="p">[</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">])</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="p">[{</span><span class="s1">'name'</span><span class="p">:</span> <span class="s1">'Alice'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span>
<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=1, name=u'Alice')]</span>
+<span class="go">[Row(age=1, name='Alice')]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">rdd</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(_1=u'Alice', _2=1)]</span>
+<span class="go">[Row(_1='Alice', _2=1)]</span>
<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="p">[</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span>
@@ -655,7 +655,7 @@ If it’s not a <a class="reference internal" href="#pyspark.sql.types.Struc
<span class="gp">>>> </span><span class="n">person</span> <span class="o">=</span> <span class="n">rdd</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">Person</span><span class="p">(</span><span class="o">*</span><span class="n">r</span><span class="p">))</span>
<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">person</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df2</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql.types</span> <span class="k">import</span> <span class="o">*</span>
@@ -664,17 +664,17 @@ If it’s not a <a class="reference internal" href="#pyspark.sql.types.Struc
<span class="gp">... </span> <span class="n">StructField</span><span class="p">(</span><span class="s2">"age"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="kc">True</span><span class="p">)])</span>
<span class="gp">>>> </span><span class="n">df3</span> <span class="o">=</span> <span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df3</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">toPandas</span><span class="p">())</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=1)]</span>
+<span class="go">[Row(name='Alice', age=1)]</span>
<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]]))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
<span class="go">[Row(0=1, 1=2)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="s2">"a: string, b: int"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(a=u'Alice', b=1)]</span>
+<span class="go">[Row(a='Alice', b=1)]</span>
<span class="gp">>>> </span><span class="n">rdd</span> <span class="o">=</span> <span class="n">rdd</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">row</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="s2">"int"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
<span class="go">[Row(value=1)]</span>
@@ -733,12 +733,12 @@ created external table.</p>
defaultValue. If the key is not set and defaultValue is None, return
the system default value.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">getConf</span><span class="p">(</span><span class="s2">"spark.sql.shuffle.partitions"</span><span class="p">)</span>
-<span class="go">u'200'</span>
-<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">getConf</span><span class="p">(</span><span class="s2">"spark.sql.shuffle.partitions"</span><span class="p">,</span> <span class="s2">u"10"</span><span class="p">)</span>
-<span class="go">u'10'</span>
-<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">setConf</span><span class="p">(</span><span class="s2">"spark.sql.shuffle.partitions"</span><span class="p">,</span> <span class="s2">u"50"</span><span class="p">)</span>
-<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">getConf</span><span class="p">(</span><span class="s2">"spark.sql.shuffle.partitions"</span><span class="p">,</span> <span class="s2">u"10"</span><span class="p">)</span>
-<span class="go">u'50'</span>
+<span class="go">'200'</span>
+<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">getConf</span><span class="p">(</span><span class="s2">"spark.sql.shuffle.partitions"</span><span class="p">,</span> <span class="sa">u</span><span class="s2">"10"</span><span class="p">)</span>
+<span class="go">'10'</span>
+<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">setConf</span><span class="p">(</span><span class="s2">"spark.sql.shuffle.partitions"</span><span class="p">,</span> <span class="sa">u</span><span class="s2">"50"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">getConf</span><span class="p">(</span><span class="s2">"spark.sql.shuffle.partitions"</span><span class="p">,</span> <span class="sa">u</span><span class="s2">"10"</span><span class="p">)</span>
+<span class="go">'50'</span>
</pre></div>
</div>
<div class="versionadded">
@@ -892,7 +892,7 @@ be done. For any other return type, the produced object must match the specifie
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">registerFunction</span><span class="p">(</span><span class="s2">"stringLengthString"</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT stringLengthString('test')"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(stringLengthString(test)=u'4')]</span>
+<span class="go">[Row(stringLengthString(test)='4')]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql.types</span> <span class="k">import</span> <span class="n">IntegerType</span>
@@ -960,7 +960,7 @@ When the return type is not specified we would infer it via reflection.
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">registerDataFrameAsTable</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="s2">"table1"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">sqlContext</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT field1 AS f1, field2 as f2 from table1"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df2</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(f1=1, f2=u'row1'), Row(f1=2, f2=u'row2'), Row(f1=3, f2=u'row3')]</span>
+<span class="go">[Row(f1=1, f2='row1'), Row(f1=2, f2='row2'), Row(f1=3, f2='row3')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1051,7 +1051,7 @@ When the return type is not specified we would infer it via reflection.
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">registerDataFrameAsTable</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="s2">"table1"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">sqlContext</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span>
<span class="gp">>>> </span><span class="n">df2</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="s2">"tableName = 'table1'"</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
-<span class="go">Row(database=u'', tableName=u'table1', isTemporary=True)</span>
+<span class="go">Row(database='', tableName='table1', isTemporary=True)</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1149,7 +1149,7 @@ be done. For any other return type, the produced object must match the specifie
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">registerFunction</span><span class="p">(</span><span class="s2">"stringLengthString"</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
<span class="gp">>>> </span><span class="n">sqlContext</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT stringLengthString('test')"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(stringLengthString(test)=u'4')]</span>
+<span class="go">[Row(stringLengthString(test)='4')]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql.types</span> <span class="k">import</span> <span class="n">IntegerType</span>
@@ -1224,7 +1224,7 @@ and can be created using various functions in <a class="reference internal" href
<span class="gp">>>> </span><span class="n">df_as2</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"df_as2"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">joined_df</span> <span class="o">=</span> <span class="n">df_as1</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df_as2</span><span class="p">,</span> <span class="n">col</span><span class="p">(</span><span class="s2">"df_as1.name"</span><span class="p">)</span> <span class="o">==</span> <span class="n">col</span><span class="p">(</span><span class="s2">"df_as2.name"</span><span class="p">),</span> <span class="s1">'inner'</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">joined_df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"df_as1.name"</span><span class="p">,</span> <span class="s2">"df_as2.name"</span><span class="p">,</span> <span class="s2">"df_as2.age"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Bob', name=u'Bob', age=5), Row(name=u'Alice', name=u'Alice', age=2)]</span>
+<span class="go">[Row(name='Bob', name='Bob', age=5), Row(name='Alice', name='Alice', age=2)]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1341,7 +1341,7 @@ the current partitioning is).</p>
<code class="descname">collect</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/sql/dataframe.html#DataFrame.collect"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.sql.DataFrame.collect" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns all the records as a list of <a class="reference internal" href="#pyspark.sql.Row" title="pyspark.sql.Row"><code class="xref py py-class docutils literal"><span class="pre">Row</span></code></a>.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=2, name='Alice'), Row(age=5, name='Bob')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1501,12 +1501,12 @@ catalog.</p>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"age"</span><span class="p">,</span> <span class="s2">"name"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=2, name='Alice'), Row(age=5, name='Bob')]</span>
<span class="gp">>>> </span><span class="n">df2</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"name"</span><span class="p">,</span> <span class="s2">"height"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Tom', height=80), Row(name=u'Bob', height=85)]</span>
+<span class="go">[Row(name='Tom', height=80), Row(name='Bob', height=85)]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">crossJoin</span><span class="p">(</span><span class="n">df2</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"height"</span><span class="p">))</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"age"</span><span class="p">,</span> <span class="s2">"name"</span><span class="p">,</span> <span class="s2">"height"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice', height=80), Row(age=2, name=u'Alice', height=85),</span>
-<span class="go"> Row(age=5, name=u'Bob', height=80), Row(age=5, name=u'Bob', height=85)]</span>
+<span class="go">[Row(age=2, name='Alice', height=80), Row(age=2, name='Alice', height=85),</span>
+<span class="go"> Row(age=5, name='Bob', height=80), Row(age=5, name='Bob', height=85)]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1633,23 +1633,23 @@ This is a no-op if schema doesn’t contain the given column name(s).</p>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s1">'age'</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice'), Row(name=u'Bob')]</span>
+<span class="go">[Row(name='Alice'), Row(name='Bob')]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice'), Row(name=u'Bob')]</span>
+<span class="go">[Row(name='Alice'), Row(name='Bob')]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="n">df2</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">'inner'</span><span class="p">)</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, height=85, name=u'Bob')]</span>
+<span class="go">[Row(age=5, height=85, name='Bob')]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="n">df2</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">'inner'</span><span class="p">)</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">df2</span><span class="o">.</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob', height=85)]</span>
+<span class="go">[Row(age=5, name='Bob', height=85)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">,</span> <span class="s1">'inner'</span><span class="p">)</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s1">'age'</span><span class="p">,</span> <span class="s1">'height'</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Bob')]</span>
+<span class="go">[Row(name='Bob')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1846,15 +1846,15 @@ or a string of SQL expression.</td>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span> <span class="o">></span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=5, name='Bob')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span> <span class="o">==</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=2, name='Alice')]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="s2">"age > 3"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=5, name='Bob')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="s2">"age = 2"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=2, name='Alice')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1867,7 +1867,7 @@ or a string of SQL expression.</td>
<code class="descname">first</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/sql/dataframe.html#DataFrame.first"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.sql.DataFrame.first" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the first row as a <a class="reference internal" href="#pyspark.sql.Row" title="pyspark.sql.Row"><code class="xref py py-class docutils literal"><span class="pre">Row</span></code></a>.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
-<span class="go">Row(age=2, name=u'Alice')</span>
+<span class="go">Row(age=2, name='Alice')</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1956,11 +1956,11 @@ Each element should be a column name (string) or an expression (<a class="refere
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">()</span><span class="o">.</span><span class="n">avg</span><span class="p">()</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
<span class="go">[Row(avg(age)=3.5)]</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="s1">'name'</span><span class="p">)</span><span class="o">.</span><span class="n">agg</span><span class="p">({</span><span class="s1">'age'</span><span class="p">:</span> <span class="s1">'mean'</span><span class="p">})</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[Row(name=u'Alice', avg(age)=2.0), Row(name=u'Bob', avg(age)=5.0)]</span>
+<span class="go">[Row(name='Alice', avg(age)=2.0), Row(name='Bob', avg(age)=5.0)]</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">avg</span><span class="p">()</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[Row(name=u'Alice', avg(age)=2.0), Row(name=u'Bob', avg(age)=5.0)]</span>
+<span class="go">[Row(name='Alice', avg(age)=2.0), Row(name='Bob', avg(age)=5.0)]</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">([</span><span class="s1">'name'</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="p">])</span><span class="o">.</span><span class="n">count</span><span class="p">()</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[Row(name=u'Alice', age=2, count=1), Row(name=u'Bob', age=5, count=1)]</span>
+<span class="go">[Row(name='Alice', age=2, count=1), Row(name='Bob', age=5, count=1)]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -1998,9 +1998,9 @@ If n is 1, return a single Row.</td>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(age=2, name=u'Alice')</span>
+<span class="go">Row(age=2, name='Alice')</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">head</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-<span class="go">[Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=2, name='Alice')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2071,24 +2071,24 @@ the column(s) must exist on both sides, and this performs an equi-join.</li>
</table>
<p>The following performs a full outer join between <code class="docutils literal"><span class="pre">df1</span></code> and <code class="docutils literal"><span class="pre">df2</span></code>.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="n">df2</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">'outer'</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">df2</span><span class="o">.</span><span class="n">height</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=None, height=80), Row(name=u'Bob', height=85), Row(name=u'Alice', height=None)]</span>
+<span class="go">[Row(name=None, height=80), Row(name='Bob', height=85), Row(name='Alice', height=None)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">,</span> <span class="s1">'outer'</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'height'</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Tom', height=80), Row(name=u'Bob', height=85), Row(name=u'Alice', height=None)]</span>
+<span class="go">[Row(name='Tom', height=80), Row(name='Bob', height=85), Row(name='Alice', height=None)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">cond</span> <span class="o">=</span> <span class="p">[</span><span class="n">df</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="n">df3</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">age</span> <span class="o">==</span> <span class="n">df3</span><span class="o">.</span><span class="n">age</span><span class="p">]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df3</span><span class="p">,</span> <span class="n">cond</span><span class="p">,</span> <span class="s1">'outer'</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">df3</span><span class="o">.</span><span class="n">age</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]</span>
+<span class="go">[Row(name='Alice', age=2), Row(name='Bob', age=5)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">df2</span><span class="o">.</span><span class="n">height</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Bob', height=85)]</span>
+<span class="go">[Row(name='Bob', height=85)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">df4</span><span class="p">,</span> <span class="p">[</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">])</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Bob', age=5)]</span>
+<span class="go">[Row(name='Bob', age=5)]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2101,7 +2101,7 @@ the column(s) must exist on both sides, and this performs an equi-join.</li>
<code class="descname">limit</code><span class="sig-paren">(</span><em>num</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/sql/dataframe.html#DataFrame.limit"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.sql.DataFrame.limit" title="Permalink to this definition">¶</a></dt>
<dd><p>Limits the result count to the number specified.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
<span class="go">[]</span>
</pre></div>
@@ -2139,18 +2139,18 @@ If a list is specified, length of the list must equal length of the <cite>cols</
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"age"</span><span class="p">,</span> <span class="n">ascending</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql.functions</span> <span class="k">import</span> <span class="o">*</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">asc</span><span class="p">(</span><span class="s2">"age"</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=2, name='Alice'), Row(age=5, name='Bob')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="n">desc</span><span class="p">(</span><span class="s2">"age"</span><span class="p">),</span> <span class="s2">"name"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">orderBy</span><span class="p">([</span><span class="s2">"age"</span><span class="p">,</span> <span class="s2">"name"</span><span class="p">],</span> <span class="n">ascending</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2467,11 +2467,11 @@ in the current DataFrame.</td>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s1">'*'</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=2, name='Alice'), Row(age=5, name='Bob')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]</span>
+<span class="go">[Row(name='Alice', age=2), Row(name='Bob', age=5)]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span> <span class="o">+</span> <span class="mi">10</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'age'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(name=u'Alice', age=12), Row(name=u'Bob', age=15)]</span>
+<span class="go">[Row(name='Alice', age=12), Row(name='Bob', age=15)]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2553,18 +2553,18 @@ If a list is specified, length of the list must equal length of the <cite>cols</
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"age"</span><span class="p">,</span> <span class="n">ascending</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql.functions</span> <span class="k">import</span> <span class="o">*</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">asc</span><span class="p">(</span><span class="s2">"age"</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=2, name='Alice'), Row(age=5, name='Bob')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">orderBy</span><span class="p">(</span><span class="n">desc</span><span class="p">(</span><span class="s2">"age"</span><span class="p">),</span> <span class="s2">"name"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">orderBy</span><span class="p">([</span><span class="s2">"age"</span><span class="p">,</span> <span class="s2">"name"</span><span class="p">],</span> <span class="n">ascending</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=5, name='Bob'), Row(age=2, name='Alice')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2646,7 +2646,7 @@ but not in another frame.</p>
<code class="descname">take</code><span class="sig-paren">(</span><em>num</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/sql/dataframe.html#DataFrame.take"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.sql.DataFrame.take" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the first <code class="docutils literal"><span class="pre">num</span></code> rows as a <code class="xref py py-class docutils literal"><span class="pre">list</span></code> of <a class="reference internal" href="#pyspark.sql.Row" title="pyspark.sql.Row"><code class="xref py py-class docutils literal"><span class="pre">Row</span></code></a>.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
-<span class="go">[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=2, name='Alice'), Row(age=5, name='Bob')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2667,7 +2667,7 @@ but not in another frame.</p>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">toDF</span><span class="p">(</span><span class="s1">'f1'</span><span class="p">,</span> <span class="s1">'f2'</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(f1=2, f2=u'Alice'), Row(f1=5, f2=u'Bob')]</span>
+<span class="go">[Row(f1=2, f2='Alice'), Row(f1=5, f2='Bob')]</span>
</pre></div>
</div>
</dd></dl>
@@ -2678,7 +2678,7 @@ but not in another frame.</p>
<dd><p>Converts a <a class="reference internal" href="#pyspark.sql.DataFrame" title="pyspark.sql.DataFrame"><code class="xref py py-class docutils literal"><span class="pre">DataFrame</span></code></a> into a <code class="xref py py-class docutils literal"><span class="pre">RDD</span></code> of string.</p>
<p>Each row is turned into a JSON document as one element in the returned RDD.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">toJSON</span><span class="p">()</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
-<span class="go">u'{"age":2,"name":"Alice"}'</span>
+<span class="go">'{"age":2,"name":"Alice"}'</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2692,7 +2692,7 @@ but not in another frame.</p>
<dd><p>Returns an iterator that contains all of the rows in this <a class="reference internal" href="#pyspark.sql.DataFrame" title="pyspark.sql.DataFrame"><code class="xref py py-class docutils literal"><span class="pre">DataFrame</span></code></a>.
The iterator will consume as much memory as the largest partition in this DataFrame.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="nb">list</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">toLocalIterator</span><span class="p">())</span>
-<span class="go">[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=2, name='Alice'), Row(age=5, name='Bob')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2788,7 +2788,7 @@ existing column that has the same name.</p>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="s1">'age2'</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">age</span> <span class="o">+</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice', age2=4), Row(age=5, name=u'Bob', age2=7)]</span>
+<span class="go">[Row(age=2, name='Alice', age2=4), Row(age=5, name='Bob', age2=7)]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2814,7 +2814,7 @@ This is a no-op if schema doesn’t contain the given column name.</p>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="s1">'age'</span><span class="p">,</span> <span class="s1">'age2'</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age2=2, name=u'Alice'), Row(age2=5, name=u'Bob')]</span>
+<span class="go">[Row(age2=2, name='Alice'), Row(age2=5, name='Bob')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -2941,12 +2941,12 @@ or a list of <a class="reference internal" href="#pyspark.sql.Column" title="pys
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">gdf</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">gdf</span><span class="o">.</span><span class="n">agg</span><span class="p">({</span><span class="s2">"*"</span><span class="p">:</span> <span class="s2">"count"</span><span class="p">})</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[Row(name=u'Alice', count(1)=1), Row(name=u'Bob', count(1)=1)]</span>
+<span class="go">[Row(name='Alice', count(1)=1), Row(name='Bob', count(1)=1)]</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">functions</span> <span class="k">as</span> <span class="n">F</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">gdf</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[Row(name=u'Alice', min(age)=2), Row(name=u'Bob', min(age)=5)]</span>
+<span class="go">[Row(name='Alice', min(age)=2), Row(name='Bob', min(age)=5)]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -3201,9 +3201,9 @@ expression is between the given columns.</p>
<code class="descname">cast</code><span class="sig-paren">(</span><em>dataType</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/sql/column.html#Column.cast"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.sql.Column.cast" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert the column into type <code class="docutils literal"><span class="pre">dataType</span></code>.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="s2">"string"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'ages'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(ages=u'2'), Row(ages=u'5')]</span>
+<span class="go">[Row(ages='2'), Row(ages='5')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">StringType</span><span class="p">())</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'ages'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(ages=u'2'), Row(ages=u'5')]</span>
+<span class="go">[Row(ages='2'), Row(ages='5')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -3291,9 +3291,9 @@ or gets an item by key out of a dict.</p>
<dd><p>A boolean expression that is evaluated to true if the value of this
expression is contained by the evaluated values of the arguments.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="p">[</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="s2">"Bob"</span><span class="p">,</span> <span class="s2">"Mike"</span><span class="p">)]</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=5, name=u'Bob')]</span>
+<span class="go">[Row(age=5, name='Bob')]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="p">[</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="o">.</span><span class="n">isin</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])]</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(age=2, name=u'Alice')]</span>
+<span class="go">[Row(age=2, name='Alice')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -3399,7 +3399,7 @@ If <a class="reference internal" href="#pyspark.sql.Column.otherwise" title="pys
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">substr</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"col"</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(col=u'Ali'), Row(col=u'Bob')]</span>
+<span class="go">[Row(col='Ali'), Row(col='Bob')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -3962,12 +3962,12 @@ any value greater than or equal to 9223372036854775807.</li>
<dl class="attribute">
<dt id="pyspark.sql.Window.unboundedFollowing">
-<code class="descname">unboundedFollowing</code><em class="property"> = 9223372036854775807L</em><a class="headerlink" href="#pyspark.sql.Window.unboundedFollowing" title="Permalink to this definition">¶</a></dt>
+<code class="descname">unboundedFollowing</code><em class="property"> = 9223372036854775807</em><a class="headerlink" href="#pyspark.sql.Window.unboundedFollowing" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="pyspark.sql.Window.unboundedPreceding">
-<code class="descname">unboundedPreceding</code><em class="property"> = -9223372036854775808L</em><a class="headerlink" href="#pyspark.sql.Window.unboundedPreceding" title="Permalink to this definition">¶</a></dt>
+<code class="descname">unboundedPreceding</code><em class="property"> = -9223372036854775808</em><a class="headerlink" href="#pyspark.sql.Window.unboundedPreceding" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>
</dd></dl>
@@ -4475,7 +4475,7 @@ are any.</p>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">text</span><span class="p">(</span><span class="s1">'python/test_support/sql/text-test.txt'</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(value=u'hello'), Row(value=u'this')]</span>
+<span class="go">[Row(value='hello'), Row(value='this')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -5532,7 +5532,7 @@ elements and value must be of the same type.</p>
<code class="descclassname">pyspark.sql.functions.</code><code class="descname">bin</code><span class="sig-paren">(</span><em>col</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/sql/functions.html#bin"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.sql.functions.bin" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the string representation of the binary value of the given column.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="nb">bin</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'c'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(c=u'10'), Row(c=u'101')]</span>
+<span class="go">[Row(c='10'), Row(c='101')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -5672,7 +5672,7 @@ or at integral part when <cite>scale</cite> < 0.</p>
<dd><p>Concatenates multiple input string columns together into a single string column.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s1">'abcd'</span><span class="p">,</span><span class="s1">'123'</span><span class="p">)],</span> <span class="p">[</span><span class="s1">'s'</span><span class="p">,</span> <span class="s1">'d'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">concat</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">s</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'s'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(s=u'abcd123')]</span>
+<span class="go">[Row(s='abcd123')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -5687,7 +5687,7 @@ or at integral part when <cite>scale</cite> < 0.</p>
using the given separator.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s1">'abcd'</span><span class="p">,</span><span class="s1">'123'</span><span class="p">)],</span> <span class="p">[</span><span class="s1">'s'</span><span class="p">,</span> <span class="s1">'d'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">concat_ws</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">s</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'s'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(s=u'abcd-123')]</span>
+<span class="go">[Row(s='abcd-123')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -5701,7 +5701,7 @@ using the given separator.</p>
<dd><p>Convert a number in a string column from one base to another.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s2">"010101"</span><span class="p">,)],</span> <span class="p">[</span><span class="s1">'n'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">conv</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">n</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">16</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'hex'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(hex=u'15')]</span>
+<span class="go">[Row(hex='15')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -5832,9 +5832,9 @@ as key-value pairs, e.g. (key1, value1, key2, value2, ...).</td>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">create_map</span><span class="p">(</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'age'</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"map"</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(map={u'Alice': 2}), Row(map={u'Bob': 5})]</span>
+<span class="go">[Row(map={'Alice': 2}), Row(map={'Bob': 5})]</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">create_map</span><span class="p">([</span><span class="n">df</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">age</span><span class="p">])</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"map"</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(map={u'Alice': 2}), Row(map={u'Bob': 5})]</span>
+<span class="go">[Row(map={'Alice': 2}), Row(map={'Bob': 5})]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -5895,7 +5895,7 @@ specialized implementation.</p>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s1">'2015-04-08'</span><span class="p">,)],</span> <span class="p">[</span><span class="s1">'a'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">date_format</span><span class="p">(</span><span class="s1">'a'</span><span class="p">,</span> <span class="s1">'MM/dd/yyy'</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'date'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(date=u'04/08/2015')]</span>
+<span class="go">[Row(date='04/08/2015')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -6118,7 +6118,7 @@ and returns the result as a string.</p>
</tbody>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="mi">5</span><span class="p">,)],</span> <span class="p">[</span><span class="s1">'a'</span><span class="p">])</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">format_number</span><span class="p">(</span><span class="s1">'a'</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'v'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(v=u'5.0000')]</span>
+<span class="go">[Row(v='5.0000')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -6144,7 +6144,7 @@ and returns the result as a string.</p>
</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="mi">5</span><span class="p">,</span> <span class="s2">"hello"</span><span class="p">)],</span> <span class="p">[</span><span class="s1">'a'</span><span class="p">,</span> <span class="s1">'b'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">format_string</span><span class="p">(</span><span class="s1">'</span><span class="si">%d</span><span class="s1"> </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">a</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">b</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s1">'v'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(v=u'5 hello')]</span>
+<span class="go">[Row(v='5 hello')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -6230,7 +6230,7 @@ of the extracted json object. It will return null if the input json string is in
<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">(</span><span class="s2">"key"</span><span class="p">,</span> <span class="s2">"jstring"</span><span class="p">))</span>
<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">key</span><span class="p">,</span> <span class="n">get_json_object</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">jstring</span><span class="p">,</span> <span class="s1">'$.f1'</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"c0"</span><span class="p">),</span> \
<span class="gp">... </span> <span class="n">get_json_object</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">jstring</span><span class="p">,</span> <span class="s1">'$.f2'</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"c1"</span><span class="p">)</span> <span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(key=u'1', c0=u'value1', c1=u'value2'), Row(key=u'2', c0=u'value12', c1=None)]</span>
+<span class="go">[Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -6319,7 +6319,7 @@ the grouping columns).</p>
<a class="reference internal" href="#pyspark.sql.types.BinaryType" title="pyspark.sql.types.BinaryType"><code class="xref py py-class docutils literal"><span class="pre">pyspark.sql.types.BinaryType</span></code></a>, <a class="reference internal" href="#pyspark.sql.types.IntegerType" title="pyspark.sql.types.IntegerType"><code class="xref py py-class docutils literal"><span class="pre">pyspark.sql.types.IntegerType</span></code></a> or
<a class="reference internal" href="#pyspark.sql.types.LongType" title="pyspark.sql.types.LongType"><code class="xref py py-class docutils literal"><span class="pre">pyspark.sql.types.LongType</span></code></a>.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s1">'ABC'</span><span class="p">,</span> <span class="mi">3</span><span class="p">)],</span> <span class="p">[</span><span class="s1">'a'</span><span class="p">,</span> <span class="s1">'b'</span><span class="p">])</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="nb">hex</span><span class="p">(</span><span class="s1">'a'</span><span class="p">),</span> <span class="nb">hex</span><span class="p">(</span><span class="s1">'b'</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[Row(hex(a)=u'414243', hex(b)=u'3')]</span>
+<span class="go">[Row(hex(a)='414243', hex(b)='3')]</span>
</pre></div>
</div>
<div class="versionadded">
@@ -6355,7 +6355,7 @@ the grouping columns).</p>
<code class="descclassname">pyspark.sql.functions.</code><code class="descname">initcap</code><span class="sig-paren">(</span><em>col</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/sql/functions.html#initcap"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.sql.functions.initcap" title="Permalink to this definition">¶</a></dt>
<dd><p>Translate the first letter of each word to upper case in the sentence.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s1">'ab cd'</span><span class="p">,)],</span> <span class="p">[</span><span class="s1">'a'</span><span class="p">])</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">initcap</span><span class="p">(</span><span class="s2
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[05/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/pyspark.ml.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/pyspark.ml.html b/site/docs/2.1.2/api/python/pyspark.ml.html
index c7034f0..557d570 100644
--- a/site/docs/2.1.2/api/python/pyspark.ml.html
+++ b/site/docs/2.1.2/api/python/pyspark.ml.html
@@ -567,7 +567,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.Pipeline">
-<em class="property">class </em><code class="descclassname">pyspark.ml.</code><code class="descname">Pipeline</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/pipeline.html#Pipeline"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.Pipeline" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.</code><code class="descname">Pipeline</code><span class="sig-paren">(</span><em>stages=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/pipeline.html#Pipeline"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.Pipeline" title="Permalink to this definition">¶</a></dt>
<dd><p>A simple pipeline, which acts as an estimator. A Pipeline consists
of a sequence of stages, each of which is either an
<a class="reference internal" href="#pyspark.ml.Estimator" title="pyspark.ml.Estimator"><code class="xref py py-class docutils literal"><span class="pre">Estimator</span></code></a> or a <a class="reference internal" href="#pyspark.ml.Transformer" title="pyspark.ml.Transformer"><code class="xref py py-class docutils literal"><span class="pre">Transformer</span></code></a>. When
@@ -1250,7 +1250,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<span id="pyspark-ml-feature-module"></span><h2>pyspark.ml.feature module<a class="headerlink" href="#module-pyspark.ml.feature" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="pyspark.ml.feature.Binarizer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Binarizer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Binarizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Binarizer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Binarizer</code><span class="sig-paren">(</span><em>threshold=0.0</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Binarizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Binarizer" title="Permalink to this definition">¶</a></dt>
<dd><p>Binarize a column of continuous features given a threshold.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="mf">0.5</span><span class="p">,)],</span> <span class="p">[</span><span class="s2">"values"</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">binarizer</span> <span class="o">=</span> <span class="n">Binarizer</span><span class="p">(</span><span class="n">threshold</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">inputCol</span><span class="o">=</span><span class="s2">"values"</span><span class="p">,</span> <span class="n">outputCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">)</span>
@@ -1520,7 +1520,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.Bucketizer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Bucketizer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Bucketizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Bucketizer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Bucketizer</code><span class="sig-paren">(</span><em>splits=None</em>, <em>inputCol=None</em>, <em>outputCol=None</em>, <em>handleInvalid='error'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Bucketizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Bucketizer" title="Permalink to this definition">¶</a></dt>
<dd><p>Maps a column of continuous features to a column of feature buckets.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">values</span> <span class="o">=</span> <span class="p">[(</span><span class="mf">0.1</span><span class="p">,),</span> <span class="p">(</span><span class="mf">0.4</span><span class="p">,),</span> <span class="p">(</span><span class="mf">1.2</span><span class="p">,),</span> <span class="p">(</span><span class="mf">1.5</span><span class="p">,),</span> <span class="p">(</span><span class="nb">float</span><span class="p">(</span><span class="s2">"nan"</span><span class="p">),),</span> <span class="p">(</span><span class="nb">float</span><span class="p">(</span><span class="s2">"nan"</span><span class="p">),)]</span>
<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="p">[</span><span class="s2">"values"</span><span class="p">])</span>
@@ -1824,7 +1824,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.ChiSqSelector">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">ChiSqSelector</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#ChiSqSelector"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.ChiSqSelector" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">ChiSqSelector</code><span class="sig-paren">(</span><em>numTopFeatures=50</em>, <em>featuresCol='features'</em>, <em>outputCol=None</em>, <em>labelCol='label'</em>, <em>selectorType='numTopFeatures'</em>, <em>percentile=0.1</em>, <em>fpr=0.05</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#ChiSqSelector"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.ChiSqSelector" title="Permalink to this definition">¶</a></dt>
<dd><div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Experimental</p>
@@ -2399,7 +2399,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.CountVectorizer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">CountVectorizer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#CountVectorizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.CountVectorizer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">CountVectorizer</code><span class="sig-paren">(</span><em>minTF=1.0</em>, <em>minDF=1.0</em>, <em>vocabSize=262144</em>, <em>binary=False</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#CountVectorizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.CountVectorizer" title="Permalink to this definition">¶</a></dt>
<dd><p>Extracts a vocabulary from document collections and generates a <a class="reference internal" href="#pyspark.ml.feature.CountVectorizerModel" title="pyspark.ml.feature.CountVectorizerModel"><code class="xref py py-attr docutils literal"><span class="pre">CountVectorizerModel</span></code></a>.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span>
<span class="gp">... </span> <span class="p">[(</span><span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">]),</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">,</span> <span class="s2">"a"</span><span class="p">])],</span>
@@ -2952,7 +2952,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.DCT">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">DCT</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#DCT"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.DCT" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">DCT</code><span class="sig-paren">(</span><em>inverse=False</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#DCT"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.DCT" title="Permalink to this definition">¶</a></dt>
<dd><p>A feature transformer that takes the 1D discrete cosine transform
of a real vector. No zero padding is performed on the input vector.
It returns a real vector of the same length representing the DCT.
@@ -3230,7 +3230,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.ElementwiseProduct">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">ElementwiseProduct</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#ElementwiseProduct"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.ElementwiseProduct" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">ElementwiseProduct</code><span class="sig-paren">(</span><em>scalingVec=None</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#ElementwiseProduct"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.ElementwiseProduct" title="Permalink to this definition">¶</a></dt>
<dd><p>Outputs the Hadamard product (i.e., the element-wise product) of each input vector
with a provided “weight” vector. In other words, it scales each column of the dataset
by a scalar multiplier.</p>
@@ -3501,7 +3501,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.HashingTF">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">HashingTF</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#HashingTF"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.HashingTF" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">HashingTF</code><span class="sig-paren">(</span><em>numFeatures=262144</em>, <em>binary=False</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#HashingTF"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.HashingTF" title="Permalink to this definition">¶</a></dt>
<dd><p>Maps a sequence of terms to their term frequencies using the hashing trick.
Currently we use Austin Appleby’s MurmurHash 3 algorithm (MurmurHash3_x86_32)
to calculate the hash code value for the term object.
@@ -3793,7 +3793,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.IDF">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">IDF</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#IDF"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.IDF" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">IDF</code><span class="sig-paren">(</span><em>minDocFreq=0</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#IDF"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.IDF" title="Permalink to this definition">¶</a></dt>
<dd><p>Compute the Inverse Document Frequency (IDF) given a collection of documents.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.ml.linalg</span> <span class="k">import</span> <span class="n">DenseVector</span>
<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="n">DenseVector</span><span class="p">([</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">]),),</span>
@@ -4272,7 +4272,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.IndexToString">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">IndexToString</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#IndexToString"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.IndexToString" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">IndexToString</code><span class="sig-paren">(</span><em>inputCol=None</em>, <em>outputCol=None</em>, <em>labels=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#IndexToString"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.IndexToString" title="Permalink to this definition">¶</a></dt>
<dd><p>A <code class="xref py py-class docutils literal"><span class="pre">Transformer</span></code> that maps a column of indices back to a new column of
corresponding string values.
The index-string mapping is either from the ML attributes of the input column,
@@ -4530,7 +4530,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.MaxAbsScaler">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">MaxAbsScaler</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#MaxAbsScaler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.MaxAbsScaler" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">MaxAbsScaler</code><span class="sig-paren">(</span><em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#MaxAbsScaler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.MaxAbsScaler" title="Permalink to this definition">¶</a></dt>
<dd><p>Rescale each feature individually to range [-1, 1] by dividing through the largest maximum
absolute value in each feature. It does not shift/center the data, and thus does not destroy
any sparsity.</p>
@@ -4988,7 +4988,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.MinMaxScaler">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">MinMaxScaler</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#MinMaxScaler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.MinMaxScaler" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">MinMaxScaler</code><span class="sig-paren">(</span><em>min=0.0</em>, <em>max=1.0</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#MinMaxScaler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.MinMaxScaler" title="Permalink to this definition">¶</a></dt>
<dd><p>Rescale each feature individually to a common range [min, max] linearly using column summary
statistics, which is also known as min-max normalization or Rescaling. The rescaled value for
feature E is calculated as,</p>
@@ -5514,7 +5514,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.NGram">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">NGram</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#NGram"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.NGram" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">NGram</code><span class="sig-paren">(</span><em>n=2</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#NGram"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.NGram" title="Permalink to this definition">¶</a></dt>
<dd><p>A feature transformer that converts the input array of strings into an array of n-grams. Null
values in the input array are ignored.
It returns an array of n-grams where each n-gram is represented by a space-separated string of
@@ -5525,15 +5525,15 @@ returned.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([</span><span class="n">Row</span><span class="p">(</span><span class="n">inputTokens</span><span class="o">=</span><span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">,</span> <span class="s2">"d"</span><span class="p">,</span> <span class="s2">"e"</span><span class="p">])])</span>
<span class="gp">>>> </span><span class="n">ngram</span> <span class="o">=</span> <span class="n">NGram</span><span class="p">(</span><span class="n">n</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">inputCol</span><span class="o">=</span><span class="s2">"inputTokens"</span><span class="p">,</span> <span class="n">outputCol</span><span class="o">=</span><span class="s2">"nGrams"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">ngram</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b', u'b c', u'c d', u'd e'])</span>
+<span class="go">Row(inputTokens=['a', 'b', 'c', 'd', 'e'], nGrams=['a b', 'b c', 'c d', 'd e'])</span>
<span class="gp">>>> </span><span class="c1"># Change n-gram length</span>
<span class="gp">>>> </span><span class="n">ngram</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="n">n</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b c d', u'b c d e'])</span>
+<span class="go">Row(inputTokens=['a', 'b', 'c', 'd', 'e'], nGrams=['a b c d', 'b c d e'])</span>
<span class="gp">>>> </span><span class="c1"># Temporarily modify output column.</span>
<span class="gp">>>> </span><span class="n">ngram</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="p">{</span><span class="n">ngram</span><span class="o">.</span><span class="n">outputCol</span><span class="p">:</span> <span class="s2">"output"</span><span class="p">})</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], output=[u'a b c d', u'b c d e'])</span>
+<span class="go">Row(inputTokens=['a', 'b', 'c', 'd', 'e'], output=['a b c d', 'b c d e'])</span>
<span class="gp">>>> </span><span class="n">ngram</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(inputTokens=[u'a', u'b', u'c', u'd', u'e'], nGrams=[u'a b c d', u'b c d e'])</span>
+<span class="go">Row(inputTokens=['a', 'b', 'c', 'd', 'e'], nGrams=['a b c d', 'b c d e'])</span>
<span class="gp">>>> </span><span class="c1"># Must use keyword arguments to specify params.</span>
<span class="gp">>>> </span><span class="n">ngram</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="s2">"text"</span><span class="p">)</span>
<span class="gt">Traceback (most recent call last):</span>
@@ -5798,7 +5798,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.Normalizer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Normalizer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Normalizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Normalizer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Normalizer</code><span class="sig-paren">(</span><em>p=2.0</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Normalizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Normalizer" title="Permalink to this definition">¶</a></dt>
<dd><blockquote>
<div>Normalize a vector to have unit norm using the given p-norm.</div></blockquote>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.ml.linalg</span> <span class="k">import</span> <span class="n">Vectors</span>
@@ -6071,7 +6071,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.OneHotEncoder">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">OneHotEncoder</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#OneHotEncoder"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.OneHotEncoder" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">OneHotEncoder</code><span class="sig-paren">(</span><em>dropLast=True</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#OneHotEncoder"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.OneHotEncoder" title="Permalink to this definition">¶</a></dt>
<dd><p>A one-hot encoder that maps a column of category indices to a
column of binary vectors, with at most a single one-value per row
that indicates the input category index.
@@ -6361,7 +6361,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.PCA">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">PCA</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#PCA"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.PCA" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">PCA</code><span class="sig-paren">(</span><em>k=None</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#PCA"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.PCA" title="Permalink to this definition">¶</a></dt>
<dd><p>PCA trains a model to project vectors to a lower dimensional space of the
top <a class="reference internal" href="#pyspark.ml.feature.PCA.k" title="pyspark.ml.feature.PCA.k"><code class="xref py py-attr docutils literal"><span class="pre">k</span></code></a> principal components.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.ml.linalg</span> <span class="k">import</span> <span class="n">Vectors</span>
@@ -6851,7 +6851,7 @@ Each column is one principal component.</p>
<dl class="class">
<dt id="pyspark.ml.feature.PolynomialExpansion">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">PolynomialExpansion</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#PolynomialExpansion"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.PolynomialExpansion" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">PolynomialExpansion</code><span class="sig-paren">(</span><em>degree=2</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#PolynomialExpansion"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.PolynomialExpansion" title="Permalink to this definition">¶</a></dt>
<dd><p>Perform feature expansion in a polynomial space. As said in <a class="reference external" href="http://en.wikipedia.org/wiki/Polynomial_expansion">wikipedia of Polynomial Expansion</a>, “In mathematics, an
expansion of a product of sums expresses it as a sum of products by using the fact that
multiplication distributes over addition”. Take a 2-variable feature vector as an example:
@@ -7122,7 +7122,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.QuantileDiscretizer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">QuantileDiscretizer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#QuantileDiscretizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.QuantileDiscretizer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">QuantileDiscretizer</code><span class="sig-paren">(</span><em>numBuckets=2</em>, <em>inputCol=None</em>, <em>outputCol=None</em>, <em>relativeError=0.001</em>, <em>handleInvalid='error'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#QuantileDiscretizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.QuantileDiscretizer" title="Permalink to this definition">¶</a></dt>
<dd><div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Experimental</p>
@@ -7460,7 +7460,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.RegexTokenizer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">RegexTokenizer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#RegexTokenizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.RegexTokenizer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">RegexTokenizer</code><span class="sig-paren">(</span><em>minTokenLength=1</em>, <em>gaps=True</em>, <em>pattern='\s+'</em>, <em>inputCol=None</em>, <em>outputCol=None</em>, <em>toLowercase=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#RegexTokenizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.RegexTokenizer" title="Permalink to this definition">¶</a></dt>
<dd><p>A regex based tokenizer that extracts tokens either by using the
provided regex pattern (in Java dialect) to split the text
(default) or repeatedly matching the regex (if gaps is false).
@@ -7470,15 +7470,15 @@ It returns an array of strings that can be empty.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s2">"A B c"</span><span class="p">,)],</span> <span class="p">[</span><span class="s2">"text"</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">reTokenizer</span> <span class="o">=</span> <span class="n">RegexTokenizer</span><span class="p">(</span><span class="n">inputCol</span><span class="o">=</span><span class="s2">"text"</span><span class="p">,</span> <span class="n">outputCol</span><span class="o">=</span><span class="s2">"words"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">reTokenizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(text=u'A B c', words=[u'a', u'b', u'c'])</span>
+<span class="go">Row(text='A B c', words=['a', 'b', 'c'])</span>
<span class="gp">>>> </span><span class="c1"># Change a parameter.</span>
<span class="gp">>>> </span><span class="n">reTokenizer</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="n">outputCol</span><span class="o">=</span><span class="s2">"tokens"</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(text=u'A B c', tokens=[u'a', u'b', u'c'])</span>
+<span class="go">Row(text='A B c', tokens=['a', 'b', 'c'])</span>
<span class="gp">>>> </span><span class="c1"># Temporarily modify a parameter.</span>
<span class="gp">>>> </span><span class="n">reTokenizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="p">{</span><span class="n">reTokenizer</span><span class="o">.</span><span class="n">outputCol</span><span class="p">:</span> <span class="s2">"words"</span><span class="p">})</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(text=u'A B c', words=[u'a', u'b', u'c'])</span>
+<span class="go">Row(text='A B c', words=['a', 'b', 'c'])</span>
<span class="gp">>>> </span><span class="n">reTokenizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(text=u'A B c', tokens=[u'a', u'b', u'c'])</span>
+<span class="go">Row(text='A B c', tokens=['a', 'b', 'c'])</span>
<span class="gp">>>> </span><span class="c1"># Must use keyword arguments to specify params.</span>
<span class="gp">>>> </span><span class="n">reTokenizer</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="s2">"text"</span><span class="p">)</span>
<span class="gt">Traceback (most recent call last):</span>
@@ -7814,7 +7814,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.RFormula">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">RFormula</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#RFormula"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.RFormula" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">RFormula</code><span class="sig-paren">(</span><em>formula=None</em>, <em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>forceIndexLabel=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#RFormula"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.RFormula" title="Permalink to this definition">¶</a></dt>
<dd><div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Experimental</p>
@@ -8346,7 +8346,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.SQLTransformer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">SQLTransformer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#SQLTransformer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.SQLTransformer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">SQLTransformer</code><span class="sig-paren">(</span><em>statement=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#SQLTransformer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.SQLTransformer" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements the transforms which are defined by SQL statement.
Currently we only support SQL syntax like ‘SELECT ... FROM __THIS__’
where ‘__THIS__’ represents the underlying table of the input dataset.</p>
@@ -8580,7 +8580,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.StandardScaler">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">StandardScaler</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#StandardScaler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.StandardScaler" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">StandardScaler</code><span class="sig-paren">(</span><em>withMean=False</em>, <em>withStd=True</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#StandardScaler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.StandardScaler" title="Permalink to this definition">¶</a></dt>
<dd><p>Standardizes features by removing the mean and scaling to unit variance using column summary
statistics on the samples in the training set.</p>
<p>The “unit std” is computed using the <a class="reference external" href="https://en.wikipedia.org/wiki/Standard_deviation#Corrected_sample_standard_deviation">corrected sample standard deviation</a>,
@@ -9094,7 +9094,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.StopWordsRemover">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">StopWordsRemover</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#StopWordsRemover"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.StopWordsRemover" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">StopWordsRemover</code><span class="sig-paren">(</span><em>inputCol=None</em>, <em>outputCol=None</em>, <em>stopWords=None</em>, <em>caseSensitive=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#StopWordsRemover"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.StopWordsRemover" title="Permalink to this definition">¶</a></dt>
<dd><p>A feature transformer that filters out stop words from input.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
@@ -9399,7 +9399,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.StringIndexer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">StringIndexer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#StringIndexer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.StringIndexer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">StringIndexer</code><span class="sig-paren">(</span><em>inputCol=None</em>, <em>outputCol=None</em>, <em>handleInvalid='error'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#StringIndexer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.StringIndexer" title="Permalink to this definition">¶</a></dt>
<dd><p>A label indexer that maps a string column of labels to an ML column of label indices.
If the input column is numeric, we cast it to string and index the string values.
The indices are in [0, numLabels), ordered by label frequencies.
@@ -9877,21 +9877,21 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.Tokenizer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Tokenizer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Tokenizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Tokenizer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Tokenizer</code><span class="sig-paren">(</span><em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Tokenizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Tokenizer" title="Permalink to this definition">¶</a></dt>
<dd><p>A tokenizer that converts the input string to lowercase and then
splits it by white spaces.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="s2">"a b c"</span><span class="p">,)],</span> <span class="p">[</span><span class="s2">"text"</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">tokenizer</span> <span class="o">=</span> <span class="n">Tokenizer</span><span class="p">(</span><span class="n">inputCol</span><span class="o">=</span><span class="s2">"text"</span><span class="p">,</span> <span class="n">outputCol</span><span class="o">=</span><span class="s2">"words"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">tokenizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(text=u'a b c', words=[u'a', u'b', u'c'])</span>
+<span class="go">Row(text='a b c', words=['a', 'b', 'c'])</span>
<span class="gp">>>> </span><span class="c1"># Change a parameter.</span>
<span class="gp">>>> </span><span class="n">tokenizer</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="n">outputCol</span><span class="o">=</span><span class="s2">"tokens"</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(text=u'a b c', tokens=[u'a', u'b', u'c'])</span>
+<span class="go">Row(text='a b c', tokens=['a', 'b', 'c'])</span>
<span class="gp">>>> </span><span class="c1"># Temporarily modify a parameter.</span>
<span class="gp">>>> </span><span class="n">tokenizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="p">{</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">outputCol</span><span class="p">:</span> <span class="s2">"words"</span><span class="p">})</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(text=u'a b c', words=[u'a', u'b', u'c'])</span>
+<span class="go">Row(text='a b c', words=['a', 'b', 'c'])</span>
<span class="gp">>>> </span><span class="n">tokenizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
-<span class="go">Row(text=u'a b c', tokens=[u'a', u'b', u'c'])</span>
+<span class="go">Row(text='a b c', tokens=['a', 'b', 'c'])</span>
<span class="gp">>>> </span><span class="c1"># Must use keyword arguments to specify params.</span>
<span class="gp">>>> </span><span class="n">tokenizer</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="s2">"text"</span><span class="p">)</span>
<span class="gt">Traceback (most recent call last):</span>
@@ -10133,7 +10133,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.VectorAssembler">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">VectorAssembler</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#VectorAssembler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.VectorAssembler" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">VectorAssembler</code><span class="sig-paren">(</span><em>inputCols=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#VectorAssembler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.VectorAssembler" title="Permalink to this definition">¶</a></dt>
<dd><p>A feature transformer that merges multiple columns into a vector column.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">)],</span> <span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">vecAssembler</span> <span class="o">=</span> <span class="n">VectorAssembler</span><span class="p">(</span><span class="n">inputCols</span><span class="o">=</span><span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">],</span> <span class="n">outputCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">)</span>
@@ -10380,15 +10380,15 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.VectorIndexer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">VectorIndexer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#VectorIndexer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.VectorIndexer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">VectorIndexer</code><span class="sig-paren">(</span><em>maxCategories=20</em>, <em>inputCol=None</em>, <em>outputCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#VectorIndexer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.VectorIndexer" title="Permalink to this definition">¶</a></dt>
<dd><p>Class for indexing categorical feature columns in a dataset of <cite>Vector</cite>.</p>
<dl class="docutils">
<dt>This has 2 usage modes:</dt>
<dd><blockquote class="first">
-<div><ul>
+<div><ul class="simple">
<li><dl class="first docutils">
<dt>Automatically identify categorical features (default behavior)</dt>
-<dd><ul class="first last simple">
+<dd><ul class="first last">
<li>This helps process a dataset of unknown vectors into a dataset with some continuous
features and some categorical features. The choice between continuous and categorical
is based upon a maxCategories parameter.</li>
@@ -10403,7 +10403,7 @@ and feature 1 will be declared continuous.</li>
</li>
<li><dl class="first docutils">
<dt>Index all features, if all features are categorical</dt>
-<dd><ul class="first last simple">
+<dd><ul class="first last">
<li>If maxCategories is set to be very large, then this will build an index of unique
values for all features.</li>
<li>Warning: This can cause problems if features are continuous since this will collect ALL
@@ -10945,7 +10945,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.feature.VectorSlicer">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">VectorSlicer</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#VectorSlicer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.VectorSlicer" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">VectorSlicer</code><span class="sig-paren">(</span><em>inputCol=None</em>, <em>outputCol=None</em>, <em>indices=None</em>, <em>names=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#VectorSlicer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.VectorSlicer" title="Permalink to this definition">¶</a></dt>
<dd><p>This class takes a feature vector and outputs a new feature vector with a subarray
of the original features.</p>
<p>The subset of features can be specified with either indices (<cite>setIndices()</cite>)
@@ -11204,7 +11204,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="method">
<dt id="pyspark.ml.feature.VectorSlicer.setParams">
-<code class="descname">setParams</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#VectorSlicer.setParams"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.VectorSlicer.setParams" title="Permalink to this definition">¶</a></dt>
+<code class="descname">setParams</code><span class="sig-paren">(</span><em>inputCol=None</em>, <em>outputCol=None</em>, <em>indices=None</em>, <em>names=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#VectorSlicer.setParams"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.VectorSlicer.setParams" title="Permalink to this definition">¶</a></dt>
<dd><p>setParams(self, inputCol=None, outputCol=None, indices=None, names=None):
Sets params for this VectorSlicer.</p>
<div class="versionadded">
@@ -11246,7 +11246,7 @@ Sets params for this VectorSlicer.</p>
<dl class="class">
<dt id="pyspark.ml.feature.Word2Vec">
-<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Word2Vec</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Word2Vec"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Word2Vec" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.feature.</code><code class="descname">Word2Vec</code><span class="sig-paren">(</span><em>vectorSize=100</em>, <em>minCount=5</em>, <em>numPartitions=1</em>, <em>stepSize=0.025</em>, <em>maxIter=1</em>, <em>seed=None</em>, <em>inputCol=None</em>, <em>outputCol=None</em>, <em>windowSize=5</em>, <em>maxSentenceLength=1000</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/feature.html#Word2Vec"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.feature.Word2Vec" title="Permalink to this definition">¶</a></dt>
<dd><p>Word2Vec trains a model of <cite>Map(String, Vector)</cite>, i.e. transforms a word into a code for further
natural language processing or machine learning process.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sent</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"a b "</span> <span class="o">*</span> <span class="mi">100</span> <span class="o">+</span> <span class="s2">"a c "</span> <span class="o">*</span> <span class="mi">10</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">" "</span><span class="p">)</span>
@@ -11901,7 +11901,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<span id="pyspark-ml-classification-module"></span><h2>pyspark.ml.classification module<a class="headerlink" href="#module-pyspark.ml.classification" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="pyspark.ml.classification.LogisticRegression">
-<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">LogisticRegression</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#LogisticRegression"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.LogisticRegression" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">LogisticRegression</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>predictionCol='prediction'</em>, <em>maxIter=100</em>, <em>regParam=0.0</em>, <em>elasticNetParam=0.0</em>, <em>tol=1e-06</em>, <em>fitIntercept=True</em>, <em>threshold=0.5</em>, <em>thresholds=None</em>, <em>probabilityCol='probability'</em>, <em>rawPredictionCol='rawPrediction'</em>, <em>standardization=True</em>, <em>weightCol=None</em>, <em>aggregationDepth=2</em>, <em>family='auto'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#LogisticRegression"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.LogisticRegression" title="Permalink to this definition">¶</a></dt>
<dd><p>Logistic regression.
This class supports multinomial logistic (softmax) and binomial logistic regression.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span>
@@ -13192,7 +13192,7 @@ versions.</p>
<dl class="class">
<dt id="pyspark.ml.classification.DecisionTreeClassifier">
-<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">DecisionTreeClassifier</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#DecisionTreeClassifier"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.DecisionTreeClassifier" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">DecisionTreeClassifier</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>predictionCol='prediction'</em>, <em>probabilityCol='probability'</em>, <em>rawPredictionCol='rawPrediction'</em>, <em>maxDepth=5</em>, <em>maxBins=32</em>, <em>minInstancesPerNode=1</em>, <em>minInfoGain=0.0</em>, <em>maxMemoryInMB=256</em>, <em>cacheNodeIds=False</em>, <em>checkpointInterval=10</em>, <em>impurity='gini'</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#DecisionTreeClassifier"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.DecisionTreeClassifier" title="Permalink to this definition">¶</a></dt>
<dd><p><a class="reference external" href="http://en.wikipedia.org/wiki/Decision_tree_learning">Decision tree</a>
learning algorithm for classification.
It supports both binary and multiclass labels, as well as both continuous and categorical
@@ -13950,7 +13950,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.classification.GBTClassifier">
-<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">GBTClassifier</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#GBTClassifier"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.GBTClassifier" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">GBTClassifier</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>predictionCol='prediction'</em>, <em>maxDepth=5</em>, <em>maxBins=32</em>, <em>minInstancesPerNode=1</em>, <em>minInfoGain=0.0</em>, <em>maxMemoryInMB=256</em>, <em>cacheNodeIds=False</em>, <em>checkpointInterval=10</em>, <em>lossType='logistic'</em>, <em>maxIter=20</em>, <em>stepSize=0.1</em>, <em>seed=None</em>, <em>subsamplingRate=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#GBTClassifier"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.GBTClassifier" title="Permalink to this definition">¶</a></dt>
<dd><p><a class="reference external" href="http://en.wikipedia.org/wiki/Gradient_boosting">Gradient-Boosted Trees (GBTs)</a>
learning algorithm for classification.
It supports binary labels, as well as both continuous and categorical features.</p>
@@ -14735,7 +14735,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.classification.RandomForestClassifier">
-<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">RandomForestClassifier</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#RandomForestClassifier"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.RandomForestClassifier" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">RandomForestClassifier</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>predictionCol='prediction'</em>, <em>probabilityCol='probability'</em>, <em>rawPredictionCol='rawPrediction'</em>, <em>maxDepth=5</em>, <em>maxBins=32</em>, <em>minInstancesPerNode=1</em>, <em>minInfoGain=0.0</em>, <em>maxMemoryInMB=256</em>, <em>cacheNodeIds=False</em>, <em>checkpointInterval=10</em>, <em>impurity='gini'</em>, <em>numTrees=20</em>, <em>featureSubsetStrategy='auto'</em>, <em>seed=None</em>, <em>subsamplingRate=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#RandomForestClassifier"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.RandomForestClassifier" title="Permalink to this definition">¶</a></dt>
<dd><p><a class="reference external" href="http://en.wikipedia.org/wiki/Random_forest">Random Forest</a>
learning algorithm for classification.
It supports both binary and multiclass labels, as well as both continuous and categorical
@@ -15569,7 +15569,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.classification.NaiveBayes">
-<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">NaiveBayes</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#NaiveBayes"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.NaiveBayes" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">NaiveBayes</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>predictionCol='prediction'</em>, <em>probabilityCol='probability'</em>, <em>rawPredictionCol='rawPrediction'</em>, <em>smoothing=1.0</em>, <em>modelType='multinomial'</em>, <em>thresholds=None</em>, <em>weightCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#NaiveBayes"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.NaiveBayes" title="Permalink to this definition">¶</a></dt>
<dd><p>Naive Bayes Classifiers.
It supports both Multinomial and Bernoulli NB. <a class="reference external" href="http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html">Multinomial NB</a>
can handle finitely supported discrete data. For example, by converting documents into
@@ -16204,7 +16204,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.classification.MultilayerPerceptronClassifier">
-<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">MultilayerPerceptronClassifier</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#MultilayerPerceptronClassifier"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.MultilayerPerceptronClassifier" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">MultilayerPerceptronClassifier</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>predictionCol='prediction'</em>, <em>maxIter=100</em>, <em>tol=1e-06</em>, <em>seed=None</em>, <em>layers=None</em>, <em>blockSize=128</em>, <em>stepSize=0.03</em>, <em>solver='l-bfgs'</em>, <em>initialWeights=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#MultilayerPerceptronClassifier"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.MultilayerPerceptronClassifier" title="Permalink to this definition">¶</a></dt>
<dd><p>Classifier trainer based on the Multilayer Perceptron.
Each layer has sigmoid activation function, output layer has softmax.
Number of inputs has to be equal to the size of feature vectors.
@@ -16881,7 +16881,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.classification.OneVsRest">
-<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">OneVsRest</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#OneVsRest"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.OneVsRest" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.classification.</code><code class="descname">OneVsRest</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>predictionCol='prediction'</em>, <em>classifier=None</em>, <em>weightCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#OneVsRest"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.OneVsRest" title="Permalink to this definition">¶</a></dt>
<dd><div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Experimental</p>
@@ -17180,7 +17180,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="method">
<dt id="pyspark.ml.classification.OneVsRest.setParams">
-<code class="descname">setParams</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#OneVsRest.setParams"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.OneVsRest.setParams" title="Permalink to this definition">¶</a></dt>
+<code class="descname">setParams</code><span class="sig-paren">(</span><em>featuresCol=None</em>, <em>labelCol=None</em>, <em>predictionCol=None</em>, <em>classifier=None</em>, <em>weightCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/classification.html#OneVsRest.setParams"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.classification.OneVsRest.setParams" title="Permalink to this definition">¶</a></dt>
<dd><p>setParams(self, featuresCol=None, labelCol=None, predictionCol=None, classifier=None, weightCol=None):
Sets params for OneVsRest.</p>
<div class="versionadded">
@@ -17521,7 +17521,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<span id="pyspark-ml-clustering-module"></span><h2>pyspark.ml.clustering module<a class="headerlink" href="#module-pyspark.ml.clustering" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="pyspark.ml.clustering.BisectingKMeans">
-<em class="property">class </em><code class="descclassname">pyspark.ml.clustering.</code><code class="descname">BisectingKMeans</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#BisectingKMeans"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.BisectingKMeans" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.clustering.</code><code class="descname">BisectingKMeans</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>predictionCol='prediction'</em>, <em>maxIter=20</em>, <em>seed=None</em>, <em>k=4</em>, <em>minDivisibleClusterSize=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#BisectingKMeans"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.BisectingKMeans" title="Permalink to this definition">¶</a></dt>
<dd><p>A bisecting k-means algorithm based on the paper “A comparison of document clustering
techniques” by Steinbach, Karypis, and Kumar, with modification to fit Spark.
The algorithm starts from a single cluster that contains all points.
@@ -18174,7 +18174,7 @@ training set. An exception is thrown if no summary exists.</p>
<dl class="class">
<dt id="pyspark.ml.clustering.KMeans">
-<em class="property">class </em><code class="descclassname">pyspark.ml.clustering.</code><code class="descname">KMeans</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#KMeans"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.KMeans" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.clustering.</code><code class="descname">KMeans</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>predictionCol='prediction'</em>, <em>k=2</em>, <em>initMode='k-means||'</em>, <em>initSteps=2</em>, <em>tol=0.0001</em>, <em>maxIter=20</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#KMeans"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.KMeans" title="Permalink to this definition">¶</a></dt>
<dd><p>K-means clustering with a k-means++ like initialization mode
(the k-means|| algorithm by Bahmani et al).</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.ml.linalg</span> <span class="k">import</span> <span class="n">Vectors</span>
@@ -18794,7 +18794,7 @@ training set. An exception is thrown if no summary exists.</p>
<dl class="class">
<dt id="pyspark.ml.clustering.GaussianMixture">
-<em class="property">class </em><code class="descclassname">pyspark.ml.clustering.</code><code class="descname">GaussianMixture</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#GaussianMixture"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.GaussianMixture" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.clustering.</code><code class="descname">GaussianMixture</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>predictionCol='prediction'</em>, <em>k=2</em>, <em>probabilityCol='probability'</em>, <em>tol=0.01</em>, <em>maxIter=100</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#GaussianMixture"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.GaussianMixture" title="Permalink to this definition">¶</a></dt>
<dd><p>GaussianMixture clustering.
This class performs expectation maximization for multivariate Gaussian
Mixture Models (GMMs). A GMM represents a composite distribution of
@@ -19510,7 +19510,7 @@ where weights[i] is the weight for Gaussian i, and weights sum to 1.</p>
<dl class="class">
<dt id="pyspark.ml.clustering.LDA">
-<em class="property">class </em><code class="descclassname">pyspark.ml.clustering.</code><code class="descname">LDA</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#LDA"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.LDA" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.clustering.</code><code class="descname">LDA</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>maxIter=20</em>, <em>seed=None</em>, <em>checkpointInterval=10</em>, <em>k=10</em>, <em>optimizer='online'</em>, <em>learningOffset=1024.0</em>, <em>learningDecay=0.51</em>, <em>subsamplingRate=0.05</em>, <em>optimizeDocConcentration=True</em>, <em>docConcentration=None</em>, <em>topicConcentration=None</em>, <em>topicDistributionCol='topicDistribution'</em>, <em>keepLastCheckpoint=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#LDA"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.LDA" title="Permalink to this definition">¶</a></dt>
<dd><p>Latent Dirichlet Allocation (LDA), a topic model designed for text documents.</p>
<p>Terminology:</p>
<blockquote>
@@ -20031,7 +20031,7 @@ Currenlty only support ‘em’ and ‘online’.</p>
<dl class="method">
<dt id="pyspark.ml.clustering.LDA.setParams">
-<code class="descname">setParams</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#LDA.setParams"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.LDA.setParams" title="Permalink to this definition">¶</a></dt>
+<code class="descname">setParams</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>maxIter=20</em>, <em>seed=None</em>, <em>checkpointInterval=10</em>, <em>k=10</em>, <em>optimizer='online'</em>, <em>learningOffset=1024.0</em>, <em>learningDecay=0.51</em>, <em>subsamplingRate=0.05</em>, <em>optimizeDocConcentration=True</em>, <em>docConcentration=None</em>, <em>topicConcentration=None</em>, <em>topicDistributionCol='topicDistribution'</em>, <em>keepLastCheckpoint=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/clustering.html#LDA.setParams"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.clustering.LDA.setParams" title="Permalink to this definition">¶</a></dt>
<dd><p>setParams(self, featuresCol=”features”, maxIter=20, seed=None, checkpointInterval=10, k=10, optimizer=”online”, learningOffset=1024.0, learningDecay=0.51, subsamplingRate=0.05, optimizeDocConcentration=True, docConcentration=None, topicConcentration=None, topicDistributionCol=”topicDistribution”, keepLastCheckpoint=True):</p>
<p>Sets params for LDA.</p>
<div class="versionadded">
@@ -21356,7 +21356,7 @@ or array.array.</p>
<span id="pyspark-ml-recommendation-module"></span><h2>pyspark.ml.recommendation module<a class="headerlink" href="#module-pyspark.ml.recommendation" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="pyspark.ml.recommendation.ALS">
-<em class="property">class </em><code class="descclassname">pyspark.ml.recommendation.</code><code class="descname">ALS</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/recommendation.html#ALS"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.recommendation.ALS" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.recommendation.</code><code class="descname">ALS</code><span class="sig-paren">(</span><em>rank=10</em>, <em>maxIter=10</em>, <em>regParam=0.1</em>, <em>numUserBlocks=10</em>, <em>numItemBlocks=10</em>, <em>implicitPrefs=False</em>, <em>alpha=1.0</em>, <em>userCol='user'</em>, <em>itemCol='item'</em>, <em>seed=None</em>, <em>ratingCol='rating'</em>, <em>nonnegative=False</em>, <em>checkpointInterval=10</em>, <em>intermediateStorageLevel='MEMORY_AND_DISK'</em>, <em>finalStorageLevel='MEMORY_AND_DISK'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/recommendation.html#ALS"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.recommendation.ALS" title="Permalink to this definition">¶</a></dt>
<dd><p>Alternating Least Squares (ALS) matrix factorization.</p>
<p>ALS attempts to estimate the ratings matrix <cite>R</cite> as the product of
two lower-rank matrices, <cite>X</cite> and <cite>Y</cite>, i.e. <cite>X * Yt = R</cite>. Typically
@@ -22184,7 +22184,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<span id="pyspark-ml-regression-module"></span><h2>pyspark.ml.regression module<a class="headerlink" href="#module-pyspark.ml.regression" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="pyspark.ml.regression.AFTSurvivalRegression">
-<em class="property">class </em><code class="descclassname">pyspark.ml.regression.</code><code class="descname">AFTSurvivalRegression</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/regression.html#AFTSurvivalRegression"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.regression.AFTSurvivalRegression" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.regression.</code><code class="descname">AFTSurvivalRegression</code><span class="sig-paren">(</span><em>featuresCol='features', labelCol='label', predictionCol='prediction', fitIntercept=True, maxIter=100, tol=1e-06, censorCol='censor', quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], quantilesCol=None, aggregationDepth=2</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/regression.html#AFTSurvivalRegression"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.regression.AFTSurvivalRegression" title="Permalink to this definition">¶</a></dt>
<dd><div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Experimental</p>
@@ -22563,7 +22563,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="method">
<dt id="pyspark.ml.regression.AFTSurvivalRegression.setParams">
-<code class="descname">setParams</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/regression.html#AFTSurvivalRegression.setParams"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.regression.AFTSurvivalRegression.setParams" title="Permalink to this definition">¶</a></dt>
+<code class="descname">setParams</code><span class="sig-paren">(</span><em>featuresCol='features', labelCol='label', predictionCol='prediction', fitIntercept=True, maxIter=100, tol=1e-06, censorCol='censor', quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], quantilesCol=None, aggregationDepth=2</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/regression.html#AFTSurvivalRegression.setParams"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.regression.AFTSurvivalRegression.setParams" title="Permalink to this definition">¶</a></dt>
<dd><p>setParams(self, featuresCol=”features”, labelCol=”label”, predictionCol=”prediction”, fitIntercept=True, maxIter=100, tol=1E-6, censorCol=”censor”, quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], quantilesCol=None, aggregationDepth=2):</p>
<div class="versionadded">
<p><span class="versionmodified">New in version 1.6.0.</span></p>
@@ -22852,7 +22852,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.regression.DecisionTreeRegressor">
-<em class="property">class </em><code class="descclassname">pyspark.ml.regression.</code><code class="descname">DecisionTreeRegressor</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/regression.html#DecisionTreeRegressor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.regression.DecisionTreeRegressor" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.ml.regression.</code><code class="descname">DecisionTreeRegressor</code><span class="sig-paren">(</span><em>featuresCol='features'</em>, <em>labelCol='label'</em>, <em>predictionCol='prediction'</em>, <em>maxDepth=5</em>, <em>maxBins=32</em>, <em>minInstancesPerNode=1</em>, <em>minInfoGain=0.0</em>, <em>maxMemoryInMB=256</em>, <em>cacheNodeIds=False</em>, <em>checkpointInterval=10</em>, <em>impurity='variance'</em>, <em>seed=None</em>, <em>varianceCol=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/regression.html#DecisionTreeRegressor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.regression.DecisionTreeRegressor" title="Permalink to this definition">¶</a></dt>
<dd><p><a class="reference external" href="http://en.wikipedia.org/wiki/Decision_tree_learning">Decision tree</a>
learning algorithm for regression.
It supports both continuous and categorical features.</p>
@@ -23572,7 +23572,7 @@ uses <code class="xref py py-func docutils literal"><span class="pre">dir()</spa
<dl class="class">
<dt id="pyspark.ml.regression.GBTRegressor">
-<em class="property">class </em><code class="descclassname">pyspark.ml.regression.</code><code class="descname">GBTRegressor</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/ml/regression.html#GBTRegressor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.ml.regression.GBTRegressor" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="descclassname">pyspark.m
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[07/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html
index 61efde4..3c07c92 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html
@@ -86,8 +86,8 @@
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span>
<span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">functions</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="n">col</span><span class="o">.</span><span class="n">_jc</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">Column</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">name</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
<span class="k">return</span> <span class="n">_</span>
@@ -99,8 +99,8 @@
<span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">functions</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="n">col1</span><span class="o">.</span><span class="n">_jc</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">Column</span><span class="p">)</span> <span class="k">else</span> <span class="nb">float</span><span class="p">(</span><span class="n">col1</span><span class="p">),</span>
<span class="n">col2</span><span class="o">.</span><span class="n">_jc</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col2</span><span class="p">,</span> <span class="n">Column</span><span class="p">)</span> <span class="k">else</span> <span class="nb">float</span><span class="p">(</span><span class="n">col2</span><span class="p">))</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">name</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
<span class="k">return</span> <span class="n">_</span>
@@ -110,8 +110,8 @@
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span>
<span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">functions</span><span class="p">,</span> <span class="n">name</span><span class="p">)()</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">name</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="s1">'Window function: '</span> <span class="o">+</span> <span class="n">doc</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="s1">'Window function: '</span> <span class="o">+</span> <span class="n">doc</span>
<span class="k">return</span> <span class="n">_</span>
@@ -1885,7 +1885,7 @@
<span class="n">jdt</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">_jsparkSession</span><span class="o">.</span><span class="n">parseDataType</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">returnType</span><span class="o">.</span><span class="n">json</span><span class="p">())</span>
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">f</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span> <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="s1">'__name__'</span><span class="p">)</span> <span class="k">else</span> <span class="n">f</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span> <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="s1">'__name__'</span><span class="p">)</span> <span class="k">else</span> <span class="n">f</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span>
<span class="n">judf</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">execution</span><span class="o">.</span><span class="n">python</span><span class="o">.</span><span class="n">UserDefinedPythonFunction</span><span class="p">(</span>
<span class="n">name</span><span class="p">,</span> <span class="n">wrapped_func</span><span class="p">,</span> <span class="n">jdt</span><span class="p">)</span>
<span class="k">return</span> <span class="n">judf</span>
@@ -1929,7 +1929,7 @@
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span><span class="p">,</span> <span class="n">SparkSession</span>
<span class="kn">import</span> <span class="nn">pyspark.sql.functions</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
<span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[4]"</span><span class="p">)</span>\
<span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"sql.functions tests"</span><span class="p">)</span>\
@@ -1946,7 +1946,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html
index f6687c9..c056446 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html
@@ -74,21 +74,21 @@
<span class="k">def</span> <span class="nf">dfapi</span><span class="p">(</span><span class="n">f</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">_api</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jgd</span><span class="p">,</span> <span class="n">name</span><span class="p">)()</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
- <span class="n">_api</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span>
- <span class="n">_api</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">_api</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span>
+ <span class="n">_api</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__doc__</span>
<span class="k">return</span> <span class="n">_api</span>
<span class="k">def</span> <span class="nf">df_varargs_api</span><span class="p">(</span><span class="n">f</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">_api</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">):</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jgd</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="n">_to_seq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">cols</span><span class="p">))</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
- <span class="n">_api</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span>
- <span class="n">_api</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">_api</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span>
+ <span class="n">_api</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__doc__</span>
<span class="k">return</span> <span class="n">_api</span>
@@ -247,7 +247,7 @@
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span><span class="p">,</span> <span class="n">SparkSession</span>
<span class="kn">import</span> <span class="nn">pyspark.sql.group</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">group</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">group</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
<span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[4]"</span><span class="p">)</span>\
<span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"sql.group tests"</span><span class="p">)</span>\
@@ -273,7 +273,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html
index bac533f..49f98d9 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html
@@ -824,7 +824,7 @@
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"SPARK_HOME"</span><span class="p">])</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">readwriter</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">readwriter</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s1">'local[4]'</span><span class="p">,</span> <span class="s1">'PythonTest'</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span><span class="o">.</span><span class="n">enableHiveSupport</span><span class="p">()</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">()</span>
@@ -844,7 +844,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html
index 919470d..c872244 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html
@@ -276,7 +276,7 @@
<span class="sd"> registered temporary views and UDFs, but shared SparkContext and</span>
<span class="sd"> table cache.</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jsparkSession</span><span class="o">.</span><span class="n">newSession</span><span class="p">())</span></div>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jsparkSession</span><span class="o">.</span><span class="n">newSession</span><span class="p">())</span></div>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">2.0</span><span class="p">)</span>
@@ -675,7 +675,7 @@
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"SPARK_HOME"</span><span class="p">])</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s1">'local[4]'</span><span class="p">,</span> <span class="s1">'PythonTest'</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'sc'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'spark'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="p">(</span><span class="n">sc</span><span class="p">)</span>
@@ -691,7 +691,7 @@
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html
index 7a8e78d..174b7dd 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html
@@ -886,7 +886,7 @@
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"SPARK_HOME"</span><span class="p">])</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">streaming</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">streaming</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">()</span>
<span class="k">except</span> <span class="n">py4j</span><span class="o">.</span><span class="n">protocol</span><span class="o">.</span><span class="n">Py4JError</span><span class="p">:</span>
@@ -911,7 +911,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html
index d3c861c..95b5e2b 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html
@@ -92,20 +92,20 @@
<span class="sd">"""Base class for data types."""</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span>
<span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">__dict__</span>
+ <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="vm">__dict__</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+ <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="DataType.typeName"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.typeName">[docs]</a> <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">__name__</span><span class="p">[:</span><span class="o">-</span><span class="mi">4</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span></div>
+<div class="viewcode-block" id="DataType.typeName"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.typeName">[docs]</a> <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span><span class="p">[:</span><span class="o">-</span><span class="mi">4</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span></div>
<div class="viewcode-block" id="DataType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">typeName</span><span class="p">()</span></div>
@@ -146,10 +146,10 @@
<span class="n">_instances</span> <span class="o">=</span> <span class="p">{}</span>
- <span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
- <span class="k">if</span> <span class="n">cls</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">:</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="n">cls</span><span class="p">]</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">DataTypeSingleton</span><span class="p">,</span> <span class="n">cls</span><span class="p">)</span><span class="o">.</span><span class="n">__call__</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="n">cls</span><span class="p">]</span>
+ <span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">cls</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">:</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="bp">cls</span><span class="p">]</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">DataTypeSingleton</span><span class="p">,</span> <span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="fm">__call__</span><span class="p">()</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="bp">cls</span><span class="p">]</span>
<div class="viewcode-block" id="NullType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.NullType">[docs]</a><span class="k">class</span> <span class="nc">NullType</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span>
@@ -352,7 +352,7 @@
<span class="s2">"containsNull"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">containsNull</span><span class="p">}</span></div>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="ArrayType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+<div class="viewcode-block" id="ArrayType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="k">return</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"elementType"</span><span class="p">]),</span>
<span class="n">json</span><span class="p">[</span><span class="s2">"containsNull"</span><span class="p">])</span></div>
@@ -409,7 +409,7 @@
<span class="s2">"valueContainsNull"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueContainsNull</span><span class="p">}</span></div>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="MapType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+<div class="viewcode-block" id="MapType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="k">return</span> <span class="n">MapType</span><span class="p">(</span><span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"keyType"</span><span class="p">]),</span>
<span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"valueType"</span><span class="p">]),</span>
<span class="n">json</span><span class="p">[</span><span class="s2">"valueContainsNull"</span><span class="p">])</span></div>
@@ -471,7 +471,7 @@
<span class="s2">"metadata"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">}</span></div>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="StructField.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+<div class="viewcode-block" id="StructField.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructField</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"name"</span><span class="p">],</span>
<span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]),</span>
<span class="n">json</span><span class="p">[</span><span class="s2">"nullable"</span><span class="p">],</span>
@@ -606,7 +606,7 @@
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">]}</span></div>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="StructType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+<div class="viewcode-block" id="StructType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">([</span><span class="n">StructField</span><span class="o">.</span><span class="n">fromJson</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">json</span><span class="p">[</span><span class="s2">"fields"</span><span class="p">]])</span></div>
<div class="viewcode-block" id="StructType.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -623,7 +623,7 @@
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">tuple</span><span class="p">,</span> <span class="nb">list</span><span class="p">)):</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">,</span> <span class="n">obj</span><span class="p">))</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span>
- <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span>
+ <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">n</span><span class="p">))</span> <span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Unexpected tuple </span><span class="si">%r</span><span class="s2"> with StructType"</span> <span class="o">%</span> <span class="n">obj</span><span class="p">)</span>
@@ -635,7 +635,7 @@
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span>
- <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span>
+ <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">n</span><span class="p">)</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Unexpected tuple </span><span class="si">%r</span><span class="s2"> with StructType"</span> <span class="o">%</span> <span class="n">obj</span><span class="p">)</span></div>
@@ -660,25 +660,25 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">__name__</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+ <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Underlying SQL storage type for this UDT.</span>
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"UDT must implement sqlType()."</span><span class="p">)</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> The Python module of the UDT.</span>
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"UDT must implement module()."</span><span class="p">)</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> The class name of the paired Scala UDT (could be '', if there</span>
<span class="sd"> is no corresponding one).</span>
@@ -689,13 +689,13 @@
<span class="k">return</span> <span class="kc">True</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_cachedSqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_cachedSqlType</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Cache the sqlType() into class, because it's heavy used in `toInternal`.</span>
<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="s2">"_cached_sql_type"</span><span class="p">):</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="s2">"_cached_sql_type"</span><span class="p">):</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span>
<span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
@@ -730,7 +730,7 @@
<span class="n">schema</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"udt"</span><span class="p">,</span>
<span class="s2">"class"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">scalaUDT</span><span class="p">(),</span>
- <span class="s2">"pyClass"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__name__</span><span class="p">),</span>
+ <span class="s2">"pyClass"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">),</span>
<span class="s2">"sqlType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span>
<span class="p">}</span>
<span class="k">else</span><span class="p">:</span>
@@ -738,14 +738,14 @@
<span class="n">b</span> <span class="o">=</span> <span class="n">ser</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
<span class="n">schema</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"udt"</span><span class="p">,</span>
- <span class="s2">"pyClass"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__name__</span><span class="p">),</span>
+ <span class="s2">"pyClass"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">),</span>
<span class="s2">"serializedClass"</span><span class="p">:</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">b</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf8'</span><span class="p">),</span>
<span class="s2">"sqlType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span>
<span class="p">}</span>
<span class="k">return</span> <span class="n">schema</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="n">pyUDT</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"pyClass"</span><span class="p">])</span> <span class="c1"># convert unicode to str</span>
<span class="n">split</span> <span class="o">=</span> <span class="n">pyUDT</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span>
<span class="n">pyModule</span> <span class="o">=</span> <span class="n">pyUDT</span><span class="p">[:</span><span class="n">split</span><span class="p">]</span>
@@ -1034,7 +1034,7 @@
<span class="n">items</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">row</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span> <span class="c1"># object</span>
- <span class="n">items</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
+ <span class="n">items</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Can not infer schema for type: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">row</span><span class="p">))</span>
@@ -1137,7 +1137,7 @@
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">d</span> <span class="o">=</span> <span class="n">obj</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span> <span class="c1"># object</span>
- <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span>
+ <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Unexpected obj type: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
@@ -1407,7 +1407,7 @@
<span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">):</span>
<span class="n">_verify_type</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">nullable</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span>
- <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span>
+ <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span>
<span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">:</span>
<span class="n">_verify_type</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">nullable</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
@@ -1472,14 +1472,14 @@
<span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
<span class="c1"># create row objects</span>
<span class="n">names</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">kwargs</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
- <span class="n">row</span> <span class="o">=</span> <span class="nb">tuple</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="p">[</span><span class="n">kwargs</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">names</span><span class="p">])</span>
+ <span class="n">row</span> <span class="o">=</span> <span class="nb">tuple</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="p">[</span><span class="n">kwargs</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">names</span><span class="p">])</span>
<span class="n">row</span><span class="o">.</span><span class="n">__fields__</span> <span class="o">=</span> <span class="n">names</span>
<span class="n">row</span><span class="o">.</span><span class="n">__from_dict__</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">return</span> <span class="n">row</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># create row class or objects</span>
- <span class="k">return</span> <span class="nb">tuple</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">tuple</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
<div class="viewcode-block" id="Row.asDict"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.Row.asDict">[docs]</a> <span class="k">def</span> <span class="nf">asDict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -1516,7 +1516,7 @@
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">"__fields__"</span><span class="p">):</span>
<span class="k">return</span> <span class="n">item</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span>
<span class="k">else</span><span class="p">:</span>
- <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__contains__</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__contains__</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="c1"># let object acts like class</span>
<span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
@@ -1525,12 +1525,12 @@
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">slice</span><span class="p">)):</span>
- <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__getitem__</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__getitem__</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="c1"># it will be slow when it has many fields,</span>
<span class="c1"># but this will not be used in normal cases</span>
<span class="n">idx</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
- <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__getitem__</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__getitem__</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">IndexError</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
@@ -1552,7 +1552,7 @@
<span class="k">def</span> <span class="nf">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="k">if</span> <span class="n">key</span> <span class="o">!=</span> <span class="s1">'__fields__'</span> <span class="ow">and</span> <span class="n">key</span> <span class="o">!=</span> <span class="s2">"__from_dict__"</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"Row is read-only"</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
<span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns a tuple so Python knows how to pickle Row."""</span>
@@ -1610,7 +1610,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html
index b4012cc..1e9f4e5 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html
@@ -281,7 +281,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html b/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html
index acbebb9..ba0427c 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html
@@ -119,7 +119,7 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">Duration</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">seconds</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">))</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_ensure_initialized</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_ensure_initialized</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="n">SparkContext</span><span class="o">.</span><span class="n">_ensure_initialized</span><span class="p">()</span>
<span class="n">gw</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_gateway</span>
@@ -129,7 +129,7 @@
<span class="c1"># start callback server</span>
<span class="c1"># getattr will fallback to JVM, so we cannot test by hasattr()</span>
- <span class="k">if</span> <span class="s2">"_callback_server"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">gw</span><span class="o">.</span><span class="n">__dict__</span> <span class="ow">or</span> <span class="n">gw</span><span class="o">.</span><span class="n">_callback_server</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="s2">"_callback_server"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">gw</span><span class="o">.</span><span class="vm">__dict__</span> <span class="ow">or</span> <span class="n">gw</span><span class="o">.</span><span class="n">_callback_server</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">gw</span><span class="o">.</span><span class="n">callback_server_parameters</span><span class="o">.</span><span class="n">eager_load</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">gw</span><span class="o">.</span><span class="n">callback_server_parameters</span><span class="o">.</span><span class="n">daemonize</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">gw</span><span class="o">.</span><span class="n">callback_server_parameters</span><span class="o">.</span><span class="n">daemonize_connections</span> <span class="o">=</span> <span class="kc">True</span>
@@ -146,11 +146,11 @@
<span class="c1"># register serializer for TransformFunction</span>
<span class="c1"># it happens before creating SparkContext when loading from checkpointing</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">_transformerSerializer</span> <span class="o">=</span> <span class="n">TransformFunctionSerializer</span><span class="p">(</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">_transformerSerializer</span> <span class="o">=</span> <span class="n">TransformFunctionSerializer</span><span class="p">(</span>
<span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span><span class="p">,</span> <span class="n">CloudPickleSerializer</span><span class="p">(),</span> <span class="n">gw</span><span class="p">)</span>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="StreamingContext.getOrCreate"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getOrCreate</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">):</span>
+<div class="viewcode-block" id="StreamingContext.getOrCreate"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getOrCreate</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Either recreate a StreamingContext from checkpoint data or create a new StreamingContext.</span>
<span class="sd"> If checkpoint data exists in the provided `checkpointPath`, then StreamingContext will be</span>
@@ -160,7 +160,7 @@
<span class="sd"> @param checkpointPath: Checkpoint directory used in an earlier streaming program</span>
<span class="sd"> @param setupFunc: Function to create a new context and setup DStreams</span>
<span class="sd"> """</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">_ensure_initialized</span><span class="p">()</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">_ensure_initialized</span><span class="p">()</span>
<span class="n">gw</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_gateway</span>
<span class="c1"># Check whether valid checkpoint information exists in the given path</span>
@@ -181,16 +181,16 @@
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span>
<span class="c1"># update ctx in serializer</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">_transformerSerializer</span><span class="o">.</span><span class="n">ctx</span> <span class="o">=</span> <span class="n">sc</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">_transformerSerializer</span><span class="o">.</span><span class="n">ctx</span> <span class="o">=</span> <span class="n">sc</span>
<span class="k">return</span> <span class="n">StreamingContext</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="n">jssc</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="StreamingContext.getActive"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getActive">[docs]</a> <span class="k">def</span> <span class="nf">getActive</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+<div class="viewcode-block" id="StreamingContext.getActive"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getActive">[docs]</a> <span class="k">def</span> <span class="nf">getActive</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Return either the currently active StreamingContext (i.e., if there is a context started</span>
<span class="sd"> but not stopped) or None.</span>
<span class="sd"> """</span>
- <span class="n">activePythonContext</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_activeContext</span>
+ <span class="n">activePythonContext</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_activeContext</span>
<span class="k">if</span> <span class="n">activePythonContext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># Verify that the current running Java StreamingContext is active and is the same one</span>
<span class="c1"># backing the supposedly active Python context</span>
@@ -198,15 +198,15 @@
<span class="n">activeJvmContextOption</span> <span class="o">=</span> <span class="n">activePythonContext</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">StreamingContext</span><span class="o">.</span><span class="n">getActive</span><span class="p">()</span>
<span class="k">if</span> <span class="n">activeJvmContextOption</span><span class="o">.</span><span class="n">isEmpty</span><span class="p">():</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">elif</span> <span class="n">activeJvmContextOption</span><span class="o">.</span><span class="n">get</span><span class="p">()</span><span class="o">.</span><span class="n">hashCode</span><span class="p">()</span> <span class="o">!=</span> <span class="n">activePythonContextJavaId</span><span class="p">:</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"JVM's active JavaStreamingContext is not the JavaStreamingContext "</span>
<span class="s2">"backing the action Python StreamingContext. This is unexpected."</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_activeContext</span></div>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_activeContext</span></div>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="StreamingContext.getActiveOrCreate"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getActiveOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getActiveOrCreate</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">):</span>
+<div class="viewcode-block" id="StreamingContext.getActiveOrCreate"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getActiveOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getActiveOrCreate</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Either return the active StreamingContext (i.e. currently started but not stopped),</span>
<span class="sd"> or recreate a StreamingContext from checkpoint data or create a new StreamingContext</span>
@@ -222,11 +222,11 @@
<span class="k">if</span> <span class="n">setupFunc</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"setupFunc cannot be None"</span><span class="p">)</span>
- <span class="n">activeContext</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">getActive</span><span class="p">()</span>
+ <span class="n">activeContext</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">getActive</span><span class="p">()</span>
<span class="k">if</span> <span class="n">activeContext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">activeContext</span>
<span class="k">elif</span> <span class="n">checkpointPath</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">(</span><span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">(</span><span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">setupFunc</span><span class="p">()</span></div>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[04/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/pyspark.mllib.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/pyspark.mllib.html b/site/docs/2.1.2/api/python/pyspark.mllib.html
index 354fa24..53418fa 100644
--- a/site/docs/2.1.2/api/python/pyspark.mllib.html
+++ b/site/docs/2.1.2/api/python/pyspark.mllib.html
@@ -936,7 +936,7 @@ of points (if < 1.0) of a divisible cluster.
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">data</span> <span class="o">=</span> <span class="n">array</span><span class="p">([</span><span class="mf">0.0</span><span class="p">,</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">9.0</span><span class="p">,</span><span class="mf">8.0</span><span class="p">,</span> <span class="mf">8.0</span><span class="p">,</span><span class="mf">9.0</span><span class="p">])</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">KMeans</span><span class="o">.</span><span class="n">train</span><span class="p">(</span>
<span class="gp">... </span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="n">data</span><span class="p">),</span> <span class="mi">2</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">initializationMode</span><span class="o">=</span><span class="s2">"random"</span><span class="p">,</span>
-<span class="gp">... </span> <span class="n">seed</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span> <span class="n">initializationSteps</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">4</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">seed</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span> <span class="n">initializationSteps</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-4</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">array</span><span class="p">([</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">]))</span> <span class="o">==</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">array</span><span class="p">([</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">]))</span>
<span class="go">True</span>
<span class="gp">>>> </span><span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">array</span><span class="p">([</span><span class="mf">8.0</span><span class="p">,</span> <span class="mf">9.0</span><span class="p">]))</span> <span class="o">==</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">array</span><span class="p">([</span><span class="mf">9.0</span><span class="p">,</span> <span class="mf">8.0</span><span class="p">]))</span>
@@ -953,7 +953,7 @@ of points (if < 1.0) of a divisible cluster.
<span class="gp">... </span> <span class="n">SparseVector</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="p">{</span><span class="mi">2</span><span class="p">:</span> <span class="mf">1.1</span><span class="p">})</span>
<span class="gp">... </span><span class="p">]</span>
<span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">KMeans</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="n">sparse_data</span><span class="p">),</span> <span class="mi">2</span><span class="p">,</span> <span class="n">initializationMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span>
-<span class="gp">... </span> <span class="n">seed</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span> <span class="n">initializationSteps</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">4</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">seed</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span> <span class="n">initializationSteps</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-4</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">array</span><span class="p">([</span><span class="mf">0.</span><span class="p">,</span> <span class="mf">1.</span><span class="p">,</span> <span class="mf">0.</span><span class="p">]))</span> <span class="o">==</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mf">1.1</span><span class="p">,</span> <span class="mf">0.</span><span class="p">]))</span>
<span class="go">True</span>
<span class="gp">>>> </span><span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">array</span><span class="p">([</span><span class="mf">0.</span><span class="p">,</span> <span class="mf">0.</span><span class="p">,</span> <span class="mf">1.</span><span class="p">]))</span> <span class="o">==</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mf">1.1</span><span class="p">]))</span>
@@ -1579,25 +1579,18 @@ a gaussian population with constant weights.</p>
<li>n_t+1 = n_t * a + m_t</li>
</ul>
<p>where</p>
-<ul>
-<li><p class="first">c_t: Centroid at the n_th iteration.</p>
-</li>
+<ul class="simple">
+<li>c_t: Centroid at the n_th iteration.</li>
<li><dl class="first docutils">
<dt>n_t: Number of samples (or) weights associated with the centroid</dt>
-<dd><p class="first last">at the n_th iteration.</p>
-</dd>
+<dd>at the n_th iteration.</dd>
</dl>
</li>
-<li><p class="first">x_t: Centroid of the new data closest to c_t.</p>
-</li>
-<li><p class="first">m_t: Number of samples (or) weights of the new data closest to c_t</p>
-</li>
-<li><p class="first">c_t+1: New centroid.</p>
-</li>
-<li><p class="first">n_t+1: New number of weights.</p>
-</li>
-<li><p class="first">a: Decay Factor, which gives the forgetfulness.</p>
-</li>
+<li>x_t: Centroid of the new data closest to c_t.</li>
+<li>m_t: Number of samples (or) weights of the new data closest to c_t</li>
+<li>c_t+1: New centroid.</li>
+<li>n_t+1: New number of weights.</li>
+<li>a: Decay Factor, which gives the forgetfulness.</li>
</ul>
<div class="admonition note">
<p class="first admonition-title">Note</p>
@@ -1622,7 +1615,7 @@ forgotten.</p>
<span class="gp">>>> </span><span class="n">stkm</span> <span class="o">=</span> <span class="n">StreamingKMeansModel</span><span class="p">(</span><span class="n">initCenters</span><span class="p">,</span> <span class="n">initWeights</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">data</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([[</span><span class="o">-</span><span class="mf">0.1</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.1</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">],</span>
<span class="gp">... </span> <span class="p">[</span><span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.1</span><span class="p">,</span> <span class="mf">1.1</span><span class="p">]])</span>
-<span class="gp">>>> </span><span class="n">stkm</span> <span class="o">=</span> <span class="n">stkm</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">u"batches"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">stkm</span> <span class="o">=</span> <span class="n">stkm</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="sa">u</span><span class="s2">"batches"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">stkm</span><span class="o">.</span><span class="n">centers</span>
<span class="go">array([[ 0., 0.],</span>
<span class="go"> [ 1., 1.]])</span>
@@ -1634,7 +1627,7 @@ forgotten.</p>
<span class="go">[3.0, 3.0]</span>
<span class="gp">>>> </span><span class="n">decayFactor</span> <span class="o">=</span> <span class="mf">0.0</span>
<span class="gp">>>> </span><span class="n">data</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="n">DenseVector</span><span class="p">([</span><span class="mf">1.5</span><span class="p">,</span> <span class="mf">1.5</span><span class="p">]),</span> <span class="n">DenseVector</span><span class="p">([</span><span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">])])</span>
-<span class="gp">>>> </span><span class="n">stkm</span> <span class="o">=</span> <span class="n">stkm</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="s2">u"batches"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">stkm</span> <span class="o">=</span> <span class="n">stkm</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="sa">u</span><span class="s2">"batches"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">stkm</span><span class="o">.</span><span class="n">centers</span>
<span class="go">array([[ 0.2, 0.2],</span>
<span class="go"> [ 1.5, 1.5]])</span>
@@ -2643,7 +2636,7 @@ Compositionality.</p>
<p>Querying for synonyms of a word will not return that word:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">syms</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">findSynonyms</span><span class="p">(</span><span class="s2">"a"</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="p">[</span><span class="n">s</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">syms</span><span class="p">]</span>
-<span class="go">[u'b', u'c']</span>
+<span class="go">['b', 'c']</span>
</pre></div>
</div>
<p>But querying for synonyms of a vector may return the word whose
@@ -2651,7 +2644,7 @@ representation is that vector:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">vec</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">syms</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">findSynonyms</span><span class="p">(</span><span class="n">vec</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="p">[</span><span class="n">s</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">syms</span><span class="p">]</span>
-<span class="go">[u'a', u'b']</span>
+<span class="go">['a', 'b']</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">os</span><span class="o">,</span> <span class="nn">tempfile</span>
@@ -2662,7 +2655,7 @@ representation is that vector:</p>
<span class="go">True</span>
<span class="gp">>>> </span><span class="n">syms</span> <span class="o">=</span> <span class="n">sameModel</span><span class="o">.</span><span class="n">findSynonyms</span><span class="p">(</span><span class="s2">"a"</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="p">[</span><span class="n">s</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">syms</span><span class="p">]</span>
-<span class="go">[u'b', u'c']</span>
+<span class="go">['b', 'c']</span>
<span class="gp">>>> </span><span class="kn">from</span> <span class="nn">shutil</span> <span class="k">import</span> <span class="n">rmtree</span>
<span class="gp">>>> </span><span class="k">try</span><span class="p">:</span>
<span class="gp">... </span> <span class="n">rmtree</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
@@ -3053,7 +3046,7 @@ using the Parallel FP-Growth algorithm.</p>
<span class="gp">>>> </span><span class="n">rdd</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">FPGrowth</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">freqItemsets</span><span class="p">()</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[FreqItemset(items=[u'a'], freq=4), FreqItemset(items=[u'c'], freq=3), ...</span>
+<span class="go">[FreqItemset(items=['a'], freq=4), FreqItemset(items=['c'], freq=3), ...</span>
<span class="gp">>>> </span><span class="n">model_path</span> <span class="o">=</span> <span class="n">temp_path</span> <span class="o">+</span> <span class="s2">"/fpm"</span>
<span class="gp">>>> </span><span class="n">model</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">model_path</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">sameModel</span> <span class="o">=</span> <span class="n">FPGrowthModel</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">model_path</span><span class="p">)</span>
@@ -3151,7 +3144,7 @@ another iteration of distributed prefix growth is run.
<span class="gp">>>> </span><span class="n">rdd</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">PrefixSpan</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">rdd</span><span class="p">)</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">freqSequences</span><span class="p">()</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[FreqSequence(sequence=[[u'a']], freq=3), FreqSequence(sequence=[[u'a'], [u'a']], freq=1), ...</span>
+<span class="go">[FreqSequence(sequence=[['a']], freq=3), FreqSequence(sequence=[['a'], ['a']], freq=1), ...</span>
</pre></div>
</div>
<div class="versionadded">
@@ -4903,7 +4896,7 @@ distribution with the input mean.</p>
<dl class="staticmethod">
<dt id="pyspark.mllib.random.RandomRDDs.exponentialVectorRDD">
-<em class="property">static </em><code class="descname">exponentialVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>*a</em>, <em>**kw</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.exponentialVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.exponentialVectorRDD" title="Permalink to this definition">¶</a></dt>
+<em class="property">static </em><code class="descname">exponentialVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>mean</em>, <em>numRows</em>, <em>numCols</em>, <em>numPartitions=None</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.exponentialVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.exponentialVectorRDD" title="Permalink to this definition">¶</a></dt>
<dd><p>Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the Exponential distribution with the input mean.</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -4989,7 +4982,7 @@ distribution with the input shape and scale.</p>
<dl class="staticmethod">
<dt id="pyspark.mllib.random.RandomRDDs.gammaVectorRDD">
-<em class="property">static </em><code class="descname">gammaVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>*a</em>, <em>**kw</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.gammaVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.gammaVectorRDD" title="Permalink to this definition">¶</a></dt>
+<em class="property">static </em><code class="descname">gammaVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>shape</em>, <em>scale</em>, <em>numRows</em>, <em>numCols</em>, <em>numPartitions=None</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.gammaVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.gammaVectorRDD" title="Permalink to this definition">¶</a></dt>
<dd><p>Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the Gamma distribution.</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -5079,7 +5072,7 @@ distribution with the input mean and standard distribution.</p>
<dl class="staticmethod">
<dt id="pyspark.mllib.random.RandomRDDs.logNormalVectorRDD">
-<em class="property">static </em><code class="descname">logNormalVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>*a</em>, <em>**kw</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.logNormalVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.logNormalVectorRDD" title="Permalink to this definition">¶</a></dt>
+<em class="property">static </em><code class="descname">logNormalVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>mean</em>, <em>std</em>, <em>numRows</em>, <em>numCols</em>, <em>numPartitions=None</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.logNormalVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.logNormalVectorRDD" title="Permalink to this definition">¶</a></dt>
<dd><p>Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the log normal distribution.</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -5165,7 +5158,7 @@ to some other normal N(mean, sigma^2), use
<dl class="staticmethod">
<dt id="pyspark.mllib.random.RandomRDDs.normalVectorRDD">
-<em class="property">static </em><code class="descname">normalVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>*a</em>, <em>**kw</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.normalVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.normalVectorRDD" title="Permalink to this definition">¶</a></dt>
+<em class="property">static </em><code class="descname">normalVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>numRows</em>, <em>numCols</em>, <em>numPartitions=None</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.normalVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.normalVectorRDD" title="Permalink to this definition">¶</a></dt>
<dd><p>Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the standard normal distribution.</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -5243,7 +5236,7 @@ distribution with the input mean.</p>
<dl class="staticmethod">
<dt id="pyspark.mllib.random.RandomRDDs.poissonVectorRDD">
-<em class="property">static </em><code class="descname">poissonVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>*a</em>, <em>**kw</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.poissonVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.poissonVectorRDD" title="Permalink to this definition">¶</a></dt>
+<em class="property">static </em><code class="descname">poissonVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>mean</em>, <em>numRows</em>, <em>numCols</em>, <em>numPartitions=None</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.poissonVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.poissonVectorRDD" title="Permalink to this definition">¶</a></dt>
<dd><p>Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the Poisson distribution with the input mean.</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -5327,7 +5320,7 @@ to U(a, b), use
<dl class="staticmethod">
<dt id="pyspark.mllib.random.RandomRDDs.uniformVectorRDD">
-<em class="property">static </em><code class="descname">uniformVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>*a</em>, <em>**kw</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.uniformVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.uniformVectorRDD" title="Permalink to this definition">¶</a></dt>
+<em class="property">static </em><code class="descname">uniformVectorRDD</code><span class="sig-paren">(</span><em>sc</em>, <em>numRows</em>, <em>numCols</em>, <em>numPartitions=None</em>, <em>seed=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/mllib/random.html#RandomRDDs.uniformVectorRDD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.mllib.random.RandomRDDs.uniformVectorRDD" title="Permalink to this definition">¶</a></dt>
<dd><p>Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the uniform distribution U(0.0, 1.0).</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -6598,9 +6591,9 @@ of freedom, p-value, the method used, and the null hypothesis.</p>
<span class="gp">>>> </span><span class="nb">print</span><span class="p">(</span><span class="nb">round</span><span class="p">(</span><span class="n">pearson</span><span class="o">.</span><span class="n">pValue</span><span class="p">,</span> <span class="mi">4</span><span class="p">))</span>
<span class="go">0.8187</span>
<span class="gp">>>> </span><span class="n">pearson</span><span class="o">.</span><span class="n">method</span>
-<span class="go">u'pearson'</span>
+<span class="go">'pearson'</span>
<span class="gp">>>> </span><span class="n">pearson</span><span class="o">.</span><span class="n">nullHypothesis</span>
-<span class="go">u'observed follows the same distribution as expected.'</span>
+<span class="go">'observed follows the same distribution as expected.'</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">observed</span> <span class="o">=</span> <span class="n">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="p">([</span><span class="mi">21</span><span class="p">,</span> <span class="mi">38</span><span class="p">,</span> <span class="mi">43</span><span class="p">,</span> <span class="mi">80</span><span class="p">])</span>
@@ -6700,7 +6693,7 @@ Supported: <cite>pearson</cite> (default), <cite>spearman</cite></li>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">x</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">2.0</span><span class="p">],</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">y</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="mf">4.0</span><span class="p">,</span> <span class="mf">5.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">],</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">zeros</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">],</span> <span class="mi">2</span><span class="p">)</span>
-<span class="gp">>>> </span><span class="nb">abs</span><span class="p">(</span><span class="n">Statistics</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="o">-</span> <span class="mf">0.6546537</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">7</span>
+<span class="gp">>>> </span><span class="nb">abs</span><span class="p">(</span><span class="n">Statistics</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="o">-</span> <span class="mf">0.6546537</span><span class="p">)</span> <span class="o"><</span> <span class="mf">1e-7</span>
<span class="go">True</span>
<span class="gp">>>> </span><span class="n">Statistics</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="o">==</span> <span class="n">Statistics</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="s2">"pearson"</span><span class="p">)</span>
<span class="go">True</span>
@@ -6780,7 +6773,7 @@ the method used, and the null hypothesis.</p>
<span class="gp">>>> </span><span class="nb">print</span><span class="p">(</span><span class="nb">round</span><span class="p">(</span><span class="n">ksmodel</span><span class="o">.</span><span class="n">statistic</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
<span class="go">0.175</span>
<span class="gp">>>> </span><span class="n">ksmodel</span><span class="o">.</span><span class="n">nullHypothesis</span>
-<span class="go">u'Sample follows theoretical distribution'</span>
+<span class="go">'Sample follows theoretical distribution'</span>
</pre></div>
</div>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">data</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="mf">2.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">,</span> <span class="mf">4.0</span><span class="p">])</span>
@@ -8013,7 +8006,7 @@ dimensions.</li>
<span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.mllib.util</span> <span class="k">import</span> <span class="n">MLUtils</span>
<span class="gp">>>> </span><span class="kn">from</span> <span class="nn">pyspark.mllib.regression</span> <span class="k">import</span> <span class="n">LabeledPoint</span>
<span class="gp">>>> </span><span class="n">tempFile</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-<span class="gp">>>> </span><span class="n">_</span> <span class="o">=</span> <span class="n">tempFile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">b</span><span class="s2">"+1 1:1.0 3:2.0 5:3.0</span><span class="se">\n</span><span class="s2">-1</span><span class="se">\n</span><span class="s2">-1 2:4.0 4:5.0 6:6.0"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">_</span> <span class="o">=</span> <span class="n">tempFile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="sa">b</span><span class="s2">"+1 1:1.0 3:2.0 5:3.0</span><span class="se">\n</span><span class="s2">-1</span><span class="se">\n</span><span class="s2">-1 2:4.0 4:5.0 6:6.0"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">tempFile</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
<span class="gp">>>> </span><span class="n">examples</span> <span class="o">=</span> <span class="n">MLUtils</span><span class="o">.</span><span class="n">loadLibSVMFile</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">tempFile</span><span class="o">.</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
<span class="gp">>>> </span><span class="n">tempFile</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[09/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/regression.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/regression.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/regression.html
index 3c5041f..7aca653 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/regression.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/regression.html
@@ -238,7 +238,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="LinearRegressionModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LinearRegressionModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="LinearRegressionModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LinearRegressionModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Load a LinearRegressionModel."""</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">regression</span><span class="o">.</span><span class="n">LinearRegressionModel</span><span class="o">.</span><span class="n">load</span><span class="p">(</span>
<span class="n">sc</span><span class="o">.</span><span class="n">_jsc</span><span class="o">.</span><span class="n">sc</span><span class="p">(),</span> <span class="n">path</span><span class="p">)</span>
@@ -274,7 +274,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"0.9.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="LinearRegressionWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LinearRegressionWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">miniBatchFraction</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
+<div class="viewcode-block" id="LinearRegressionWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LinearRegressionWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">miniBatchFraction</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
<span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">regType</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">validateData</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">convergenceTol</span><span class="o">=</span><span class="mf">0.001</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -405,7 +405,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="LassoModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LassoModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="LassoModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LassoModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Load a LassoModel."""</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">regression</span><span class="o">.</span><span class="n">LassoModel</span><span class="o">.</span><span class="n">load</span><span class="p">(</span>
<span class="n">sc</span><span class="o">.</span><span class="n">_jsc</span><span class="o">.</span><span class="n">sc</span><span class="p">(),</span> <span class="n">path</span><span class="p">)</span>
@@ -423,7 +423,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"0.9.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="LassoWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LassoWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
+<div class="viewcode-block" id="LassoWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.LassoWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
<span class="n">miniBatchFraction</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">validateData</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">convergenceTol</span><span class="o">=</span><span class="mf">0.001</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -548,7 +548,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="RidgeRegressionModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.RidgeRegressionModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="RidgeRegressionModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.RidgeRegressionModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Load a RidgeRegressionMode."""</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">regression</span><span class="o">.</span><span class="n">RidgeRegressionModel</span><span class="o">.</span><span class="n">load</span><span class="p">(</span>
<span class="n">sc</span><span class="o">.</span><span class="n">_jsc</span><span class="o">.</span><span class="n">sc</span><span class="p">(),</span> <span class="n">path</span><span class="p">)</span>
@@ -567,7 +567,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"0.9.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="RidgeRegressionWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.RidgeRegressionWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
+<div class="viewcode-block" id="RidgeRegressionWithSGD.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.RidgeRegressionWithSGD.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
<span class="n">miniBatchFraction</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">validateData</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">convergenceTol</span><span class="o">=</span><span class="mf">0.001</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -705,7 +705,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="IsotonicRegressionModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.IsotonicRegressionModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="IsotonicRegressionModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.IsotonicRegressionModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Load an IsotonicRegressionModel."""</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">regression</span><span class="o">.</span><span class="n">IsotonicRegressionModel</span><span class="o">.</span><span class="n">load</span><span class="p">(</span>
<span class="n">sc</span><span class="o">.</span><span class="n">_jsc</span><span class="o">.</span><span class="n">sc</span><span class="p">(),</span> <span class="n">path</span><span class="p">)</span>
@@ -740,7 +740,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="IsotonicRegression.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.IsotonicRegression.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">isotonic</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+<div class="viewcode-block" id="IsotonicRegression.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.classification.IsotonicRegression.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">isotonic</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Train an isotonic regression model on the given data.</span>
@@ -840,7 +840,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">miniBatchFraction</span> <span class="o">=</span> <span class="n">miniBatchFraction</span>
<span class="bp">self</span><span class="o">.</span><span class="n">convergenceTol</span> <span class="o">=</span> <span class="n">convergenceTol</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_model</span> <span class="o">=</span> <span class="kc">None</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">StreamingLinearRegressionWithSGD</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">StreamingLinearRegressionWithSGD</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
<span class="n">model</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_model</span><span class="p">)</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.5.0"</span><span class="p">)</span>
@@ -874,7 +874,7 @@
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
<span class="kn">import</span> <span class="nn">pyspark.mllib.regression</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">regression</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">regression</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
<span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[2]"</span><span class="p">)</span>\
<span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"mllib.regression tests"</span><span class="p">)</span>\
@@ -885,7 +885,7 @@
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/tree.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/tree.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/tree.html
index e060769..ed86923 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/tree.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/tree.html
@@ -170,7 +170,7 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_java_model</span><span class="o">.</span><span class="n">toDebugString</span><span class="p">()</span></div>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"org.apache.spark.mllib.tree.model.DecisionTreeModel"</span></div>
@@ -183,7 +183,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="nb">type</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">features</span><span class="p">,</span> <span class="n">impurity</span><span class="o">=</span><span class="s2">"gini"</span><span class="p">,</span> <span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
+ <span class="k">def</span> <span class="nf">_train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="nb">type</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">features</span><span class="p">,</span> <span class="n">impurity</span><span class="o">=</span><span class="s2">"gini"</span><span class="p">,</span> <span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
<span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">):</span>
<span class="n">first</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">first</span><span class="p">,</span> <span class="n">LabeledPoint</span><span class="p">),</span> <span class="s2">"the data should be RDD of LabeledPoint"</span>
@@ -193,7 +193,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.1.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="DecisionTree.trainClassifier"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.DecisionTree.trainClassifier">[docs]</a> <span class="k">def</span> <span class="nf">trainClassifier</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+<div class="viewcode-block" id="DecisionTree.trainClassifier"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.DecisionTree.trainClassifier">[docs]</a> <span class="k">def</span> <span class="nf">trainClassifier</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">impurity</span><span class="o">=</span><span class="s2">"gini"</span><span class="p">,</span> <span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -260,12 +260,12 @@
<span class="sd"> >>> model.predict(rdd).collect()</span>
<span class="sd"> [1.0, 0.0]</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"classification"</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"classification"</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">impurity</span><span class="p">,</span> <span class="n">maxDepth</span><span class="p">,</span> <span class="n">maxBins</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.1.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="DecisionTree.trainRegressor"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.DecisionTree.trainRegressor">[docs]</a> <span class="k">def</span> <span class="nf">trainRegressor</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+<div class="viewcode-block" id="DecisionTree.trainRegressor"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.DecisionTree.trainRegressor">[docs]</a> <span class="k">def</span> <span class="nf">trainRegressor</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">impurity</span><span class="o">=</span><span class="s2">"variance"</span><span class="p">,</span> <span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -320,7 +320,7 @@
<span class="sd"> >>> model.predict(rdd).collect()</span>
<span class="sd"> [1.0, 0.0]</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"regression"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"regression"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">impurity</span><span class="p">,</span> <span class="n">maxDepth</span><span class="p">,</span> <span class="n">maxBins</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="p">)</span></div></div>
@@ -333,7 +333,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"org.apache.spark.mllib.tree.model.RandomForestModel"</span></div>
@@ -348,11 +348,11 @@
<span class="n">supportedFeatureSubsetStrategies</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"auto"</span><span class="p">,</span> <span class="s2">"all"</span><span class="p">,</span> <span class="s2">"sqrt"</span><span class="p">,</span> <span class="s2">"log2"</span><span class="p">,</span> <span class="s2">"onethird"</span><span class="p">)</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">algo</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span>
+ <span class="k">def</span> <span class="nf">_train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">algo</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span>
<span class="n">featureSubsetStrategy</span><span class="p">,</span> <span class="n">impurity</span><span class="p">,</span> <span class="n">maxDepth</span><span class="p">,</span> <span class="n">maxBins</span><span class="p">,</span> <span class="n">seed</span><span class="p">):</span>
<span class="n">first</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">first</span><span class="p">,</span> <span class="n">LabeledPoint</span><span class="p">),</span> <span class="s2">"the data should be RDD of LabeledPoint"</span>
- <span class="k">if</span> <span class="n">featureSubsetStrategy</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">cls</span><span class="o">.</span><span class="n">supportedFeatureSubsetStrategies</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">featureSubsetStrategy</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">supportedFeatureSubsetStrategies</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"unsupported featureSubsetStrategy: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">featureSubsetStrategy</span><span class="p">)</span>
<span class="k">if</span> <span class="n">seed</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">seed</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span> <span class="o"><<</span> <span class="mi">30</span><span class="p">)</span>
@@ -363,7 +363,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.2.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="RandomForest.trainClassifier"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.RandomForest.trainClassifier">[docs]</a> <span class="k">def</span> <span class="nf">trainClassifier</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span>
+<div class="viewcode-block" id="RandomForest.trainClassifier"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.RandomForest.trainClassifier">[docs]</a> <span class="k">def</span> <span class="nf">trainClassifier</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span>
<span class="n">featureSubsetStrategy</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span> <span class="n">impurity</span><span class="o">=</span><span class="s2">"gini"</span><span class="p">,</span> <span class="n">maxDepth</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
<span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -449,13 +449,13 @@
<span class="sd"> >>> model.predict(rdd).collect()</span>
<span class="sd"> [1.0, 0.0]</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"classification"</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"classification"</span><span class="p">,</span> <span class="n">numClasses</span><span class="p">,</span>
<span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span> <span class="n">featureSubsetStrategy</span><span class="p">,</span> <span class="n">impurity</span><span class="p">,</span>
<span class="n">maxDepth</span><span class="p">,</span> <span class="n">maxBins</span><span class="p">,</span> <span class="n">seed</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.2.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="RandomForest.trainRegressor"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.RandomForest.trainRegressor">[docs]</a> <span class="k">def</span> <span class="nf">trainRegressor</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span> <span class="n">featureSubsetStrategy</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
+<div class="viewcode-block" id="RandomForest.trainRegressor"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.RandomForest.trainRegressor">[docs]</a> <span class="k">def</span> <span class="nf">trainRegressor</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span> <span class="n">featureSubsetStrategy</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
<span class="n">impurity</span><span class="o">=</span><span class="s2">"variance"</span><span class="p">,</span> <span class="n">maxDepth</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Train a random forest model for regression.</span>
@@ -519,7 +519,7 @@
<span class="sd"> >>> model.predict(rdd).collect()</span>
<span class="sd"> [1.0, 0.5]</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"regression"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"regression"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span> <span class="n">numTrees</span><span class="p">,</span>
<span class="n">featureSubsetStrategy</span><span class="p">,</span> <span class="n">impurity</span><span class="p">,</span> <span class="n">maxDepth</span><span class="p">,</span> <span class="n">maxBins</span><span class="p">,</span> <span class="n">seed</span><span class="p">)</span></div></div>
@@ -532,7 +532,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"org.apache.spark.mllib.tree.model.GradientBoostedTreesModel"</span></div>
@@ -545,7 +545,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">algo</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+ <span class="k">def</span> <span class="nf">_train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">algo</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">loss</span><span class="p">,</span> <span class="n">numIterations</span><span class="p">,</span> <span class="n">learningRate</span><span class="p">,</span> <span class="n">maxDepth</span><span class="p">,</span> <span class="n">maxBins</span><span class="p">):</span>
<span class="n">first</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">first</span><span class="p">,</span> <span class="n">LabeledPoint</span><span class="p">),</span> <span class="s2">"the data should be RDD of LabeledPoint"</span>
@@ -555,7 +555,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.3.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="GradientBoostedTrees.trainClassifier"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.GradientBoostedTrees.trainClassifier">[docs]</a> <span class="k">def</span> <span class="nf">trainClassifier</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+<div class="viewcode-block" id="GradientBoostedTrees.trainClassifier"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.GradientBoostedTrees.trainClassifier">[docs]</a> <span class="k">def</span> <span class="nf">trainClassifier</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">loss</span><span class="o">=</span><span class="s2">"logLoss"</span><span class="p">,</span> <span class="n">numIterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">learningRate</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">maxDepth</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
<span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -619,12 +619,12 @@
<span class="sd"> >>> model.predict(rdd).collect()</span>
<span class="sd"> [1.0, 0.0]</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"classification"</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"classification"</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">loss</span><span class="p">,</span> <span class="n">numIterations</span><span class="p">,</span> <span class="n">learningRate</span><span class="p">,</span> <span class="n">maxDepth</span><span class="p">,</span> <span class="n">maxBins</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.3.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="GradientBoostedTrees.trainRegressor"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.GradientBoostedTrees.trainRegressor">[docs]</a> <span class="k">def</span> <span class="nf">trainRegressor</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+<div class="viewcode-block" id="GradientBoostedTrees.trainRegressor"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.GradientBoostedTrees.trainRegressor">[docs]</a> <span class="k">def</span> <span class="nf">trainRegressor</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">loss</span><span class="o">=</span><span class="s2">"leastSquaresError"</span><span class="p">,</span> <span class="n">numIterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">learningRate</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">maxDepth</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
<span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -686,7 +686,7 @@
<span class="sd"> >>> model.predict(rdd).collect()</span>
<span class="sd"> [1.0, 0.0]</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"regression"</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_train</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"regression"</span><span class="p">,</span> <span class="n">categoricalFeaturesInfo</span><span class="p">,</span>
<span class="n">loss</span><span class="p">,</span> <span class="n">numIterations</span><span class="p">,</span> <span class="n">learningRate</span><span class="p">,</span> <span class="n">maxDepth</span><span class="p">,</span> <span class="n">maxBins</span><span class="p">)</span></div></div>
@@ -704,7 +704,7 @@
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/util.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/util.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/util.html
index 9d1af9a..5d0be6d 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/util.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/util.html
@@ -462,7 +462,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="Loader.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.Loader.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="Loader.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.Loader.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a model from the given path. The model should have been</span>
<span class="sd"> saved using py:meth:`Saveable.save`.</span>
@@ -485,21 +485,21 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_java_loader_class</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns the full class name of the Java loader. The default</span>
<span class="sd"> implementation replaces "pyspark" by "org.apache.spark" in</span>
<span class="sd"> the Python full class name.</span>
<span class="sd"> """</span>
- <span class="n">java_package</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">__module__</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"pyspark"</span><span class="p">,</span> <span class="s2">"org.apache.spark"</span><span class="p">)</span>
- <span class="k">return</span> <span class="s2">"."</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">java_package</span><span class="p">,</span> <span class="n">cls</span><span class="o">.</span><span class="n">__name__</span><span class="p">])</span>
+ <span class="n">java_package</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__module__</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"pyspark"</span><span class="p">,</span> <span class="s2">"org.apache.spark"</span><span class="p">)</span>
+ <span class="k">return</span> <span class="s2">"."</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">java_package</span><span class="p">,</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span><span class="p">])</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_load_java</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_load_java</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a Java model from the given path.</span>
<span class="sd"> """</span>
- <span class="n">java_class</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_java_loader_class</span><span class="p">()</span>
+ <span class="n">java_class</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_java_loader_class</span><span class="p">()</span>
<span class="n">java_obj</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span>
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">java_class</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"."</span><span class="p">):</span>
<span class="n">java_obj</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">java_obj</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
@@ -507,10 +507,10 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.3.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="JavaLoader.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.JavaLoader.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="JavaLoader.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.JavaLoader.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Load a model from the given path."""</span>
- <span class="n">java_model</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">cls</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span></div></div>
+ <span class="n">java_model</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="LinearDataGenerator"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.tree.LinearDataGenerator">[docs]</a><span class="k">class</span> <span class="nc">LinearDataGenerator</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
@@ -572,7 +572,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/profiler.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/profiler.html b/site/docs/2.1.2/api/python/_modules/pyspark/profiler.html
index ed21d4c..94f20d9 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/profiler.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/profiler.html
@@ -197,7 +197,7 @@
<span class="sd"> cProfile and Accumulator</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ctx</span><span class="p">):</span>
- <span class="n">Profiler</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ctx</span><span class="p">)</span>
+ <span class="n">Profiler</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ctx</span><span class="p">)</span>
<span class="c1"># Creates a new accumulator for combining the profiles of different</span>
<span class="c1"># partitions of a stage</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_accumulator</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">accumulator</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="n">PStatsParam</span><span class="p">)</span>
@@ -217,7 +217,7 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_accumulator</span><span class="o">.</span><span class="n">value</span></div></div>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">()</span>
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/rdd.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/rdd.html b/site/docs/2.1.2/api/python/_modules/pyspark/rdd.html
index 8bac5eb..60eac5a 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/rdd.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/rdd.html
@@ -142,8 +142,8 @@
<span class="sd"> >>> BoundedFloat(100.0, 0.95, 95.0, 105.0)</span>
<span class="sd"> 100.0</span>
<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__new__</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">mean</span><span class="p">,</span> <span class="n">confidence</span><span class="p">,</span> <span class="n">low</span><span class="p">,</span> <span class="n">high</span><span class="p">):</span>
- <span class="n">obj</span> <span class="o">=</span> <span class="nb">float</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">mean</span><span class="p">)</span>
+ <span class="k">def</span> <span class="nf">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">mean</span><span class="p">,</span> <span class="n">confidence</span><span class="p">,</span> <span class="n">low</span><span class="p">,</span> <span class="n">high</span><span class="p">):</span>
+ <span class="n">obj</span> <span class="o">=</span> <span class="nb">float</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">mean</span><span class="p">)</span>
<span class="n">obj</span><span class="o">.</span><span class="n">confidence</span> <span class="o">=</span> <span class="n">confidence</span>
<span class="n">obj</span><span class="o">.</span><span class="n">low</span> <span class="o">=</span> <span class="n">low</span>
<span class="n">obj</span><span class="o">.</span><span class="n">high</span> <span class="o">=</span> <span class="n">high</span>
@@ -198,8 +198,8 @@
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version</span> <span class="o">>=</span> <span class="s1">'3'</span><span class="p">:</span>
<span class="c1"># the representation of unicode string in Python 3 does not have prefix 'u',</span>
<span class="c1"># so remove the prefix 'u' for doc tests</span>
- <span class="n">literal_re</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s2">r"(\W|^)[uU](['])"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">UNICODE</span><span class="p">)</span>
- <span class="n">f</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">literal_re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s1">r'\1\2'</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">__doc__</span><span class="p">)</span>
+ <span class="n">literal_re</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(\W|^)[uU](['])"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">UNICODE</span><span class="p">)</span>
+ <span class="n">f</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">literal_re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\1\2'</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="vm">__doc__</span><span class="p">)</span>
<span class="k">return</span> <span class="n">f</span>
@@ -811,8 +811,8 @@
<span class="k">else</span><span class="p">:</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">):</span>
<span class="k">yield</span> <span class="n">i</span>
- <span class="k">return</span> <span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="n">b</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span>
- <span class="n">chain</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="n">pipe</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="n">b</span><span class="s1">''</span><span class="p">),</span> <span class="n">check_return_code</span><span class="p">()))</span>
+ <span class="k">return</span> <span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="sa">b</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span>
+ <span class="n">chain</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="n">pipe</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="sa">b</span><span class="s1">''</span><span class="p">),</span> <span class="n">check_return_code</span><span class="p">()))</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapPartitions</span><span class="p">(</span><span class="n">func</span><span class="p">)</span></div>
<div class="viewcode-block" id="RDD.foreach"><a class="viewcode-back" href="../../pyspark.html#pyspark.RDD.foreach">[docs]</a> <span class="k">def</span> <span class="nf">foreach</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">):</span>
@@ -1200,7 +1200,7 @@
<span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span>
<span class="k">pass</span> <span class="c1"># objects in buckets do not support '-'</span>
<span class="k">else</span><span class="p">:</span>
- <span class="k">if</span> <span class="nb">max</span><span class="p">(</span><span class="n">steps</span><span class="p">)</span> <span class="o">-</span> <span class="nb">min</span><span class="p">(</span><span class="n">steps</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">10</span><span class="p">:</span> <span class="c1"># handle precision errors</span>
+ <span class="k">if</span> <span class="nb">max</span><span class="p">(</span><span class="n">steps</span><span class="p">)</span> <span class="o">-</span> <span class="nb">min</span><span class="p">(</span><span class="n">steps</span><span class="p">)</span> <span class="o"><</span> <span class="mf">1e-10</span><span class="p">:</span> <span class="c1"># handle precision errors</span>
<span class="n">even</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">inc</span> <span class="o">=</span> <span class="p">(</span><span class="n">maxv</span> <span class="o">-</span> <span class="n">minv</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">buckets</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
@@ -2518,7 +2518,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[10/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/clustering.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/clustering.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/clustering.html
index 8aab284..eef793a 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/clustering.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/clustering.html
@@ -112,7 +112,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">java_model</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">BisectingKMeansModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BisectingKMeansModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">centers</span> <span class="o">=</span> <span class="p">[</span><span class="n">c</span><span class="o">.</span><span class="n">toArray</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="s2">"clusterCenters"</span><span class="p">)]</span>
<span class="nd">@property</span>
@@ -338,7 +338,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="KMeansModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.KMeansModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="KMeansModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.KMeansModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a model from the given path.</span>
<span class="sd"> """</span>
@@ -353,8 +353,8 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'0.9.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="KMeans.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.KMeans.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">rdd</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">runs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">initializationMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span>
- <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">initializationSteps</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">4</span><span class="p">,</span> <span class="n">initialModel</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+<div class="viewcode-block" id="KMeans.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.KMeans.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">rdd</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">runs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">initializationMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span>
+ <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">initializationSteps</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-4</span><span class="p">,</span> <span class="n">initialModel</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Train a k-means clustering model.</span>
@@ -540,7 +540,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.5.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="GaussianMixtureModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.GaussianMixtureModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="GaussianMixtureModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.GaussianMixtureModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Load the GaussianMixtureModel from disk.</span>
<span class="sd"> :param sc:</span>
@@ -548,9 +548,9 @@
<span class="sd"> :param path:</span>
<span class="sd"> Path to where the model is stored.</span>
<span class="sd"> """</span>
- <span class="n">model</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
+ <span class="n">model</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
<span class="n">wrapper</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">api</span><span class="o">.</span><span class="n">python</span><span class="o">.</span><span class="n">GaussianMixtureModelWrapper</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">cls</span><span class="p">(</span><span class="n">wrapper</span><span class="p">)</span></div></div>
+ <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">wrapper</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="GaussianMixture"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.GaussianMixture">[docs]</a><span class="k">class</span> <span class="nc">GaussianMixture</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
@@ -561,7 +561,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.3.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="GaussianMixture.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.GaussianMixture.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">rdd</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">convergenceTol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">initialModel</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+<div class="viewcode-block" id="GaussianMixture.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.GaussianMixture.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">rdd</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">convergenceTol</span><span class="o">=</span><span class="mf">1e-3</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">initialModel</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Train a Gaussian Mixture clustering model.</span>
@@ -671,11 +671,11 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.5.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="PowerIterationClusteringModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.PowerIterationClusteringModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="PowerIterationClusteringModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.PowerIterationClusteringModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a model from the given path.</span>
<span class="sd"> """</span>
- <span class="n">model</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
+ <span class="n">model</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
<span class="n">wrapper</span> <span class="o">=</span>\
<span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">api</span><span class="o">.</span><span class="n">python</span><span class="o">.</span><span class="n">PowerIterationClusteringModelWrapper</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
<span class="k">return</span> <span class="n">PowerIterationClusteringModel</span><span class="p">(</span><span class="n">wrapper</span><span class="p">)</span></div></div>
@@ -694,7 +694,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.5.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="PowerIterationClustering.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.PowerIterationClustering.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">rdd</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">initMode</span><span class="o">=</span><span class="s2">"random"</span><span class="p">):</span>
+<div class="viewcode-block" id="PowerIterationClustering.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.PowerIterationClustering.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">rdd</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">initMode</span><span class="o">=</span><span class="s2">"random"</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> :param rdd:</span>
<span class="sd"> An RDD of (i, j, s\ :sub:`ij`\) tuples representing the</span>
@@ -787,7 +787,7 @@
<span class="sd"> .. versionadded:: 1.5.0</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">clusterCenters</span><span class="p">,</span> <span class="n">clusterWeights</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">StreamingKMeansModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">centers</span><span class="o">=</span><span class="n">clusterCenters</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">StreamingKMeansModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">centers</span><span class="o">=</span><span class="n">clusterCenters</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_clusterWeights</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">clusterWeights</span><span class="p">)</span>
<span class="nd">@property</span>
@@ -1022,7 +1022,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.5.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="LDAModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.LDAModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="LDAModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.LDAModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Load the LDAModel from disk.</span>
<span class="sd"> :param sc:</span>
@@ -1045,7 +1045,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.5.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="LDA.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.LDA.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">rdd</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">docConcentration</span><span class="o">=-</span><span class="mf">1.0</span><span class="p">,</span>
+<div class="viewcode-block" id="LDA.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.clustering.LDA.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">rdd</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">maxIterations</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">docConcentration</span><span class="o">=-</span><span class="mf">1.0</span><span class="p">,</span>
<span class="n">topicConcentration</span><span class="o">=-</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">checkpointInterval</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s2">"em"</span><span class="p">):</span>
<span class="sd">"""Train a LDA model.</span>
@@ -1091,7 +1091,7 @@
<span class="k">def</span> <span class="nf">_test</span><span class="p">():</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">import</span> <span class="nn">pyspark.mllib.clustering</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">clustering</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">clustering</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'sc'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s1">'local[4]'</span><span class="p">,</span> <span class="s1">'PythonTest'</span><span class="p">,</span> <span class="n">batchSize</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">(</span><span class="n">globs</span><span class="o">=</span><span class="n">globs</span><span class="p">,</span> <span class="n">optionflags</span><span class="o">=</span><span class="n">doctest</span><span class="o">.</span><span class="n">ELLIPSIS</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'sc'</span><span class="p">]</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span>
@@ -1099,7 +1099,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/common.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/common.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/common.html
index 4e761b0..2f015d3 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/common.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/common.html
@@ -194,21 +194,21 @@
<span class="k">return</span> <span class="n">callJavaFunc</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_java_model</span><span class="p">,</span> <span class="n">name</span><span class="p">),</span> <span class="o">*</span><span class="n">a</span><span class="p">)</span>
-<span class="k">def</span> <span class="nf">inherit_doc</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+<span class="k">def</span> <span class="nf">inherit_doc</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> A decorator that makes a class inherit documentation from its parents.</span>
<span class="sd"> """</span>
- <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">func</span> <span class="ow">in</span> <span class="nb">vars</span><span class="p">(</span><span class="n">cls</span><span class="p">)</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">func</span> <span class="ow">in</span> <span class="nb">vars</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="c1"># only inherit docstring for public functions</span>
<span class="k">if</span> <span class="n">name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"_"</span><span class="p">):</span>
<span class="k">continue</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">func</span><span class="o">.</span><span class="n">__doc__</span><span class="p">:</span>
- <span class="k">for</span> <span class="n">parent</span> <span class="ow">in</span> <span class="n">cls</span><span class="o">.</span><span class="n">__bases__</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">func</span><span class="o">.</span><span class="vm">__doc__</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">parent</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__bases__</span><span class="p">:</span>
<span class="n">parent_func</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">parent</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">parent_func</span> <span class="ow">and</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">parent_func</span><span class="p">,</span> <span class="s2">"__doc__"</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
- <span class="n">func</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">parent_func</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">func</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">parent_func</span><span class="o">.</span><span class="vm">__doc__</span>
<span class="k">break</span>
- <span class="k">return</span> <span class="n">cls</span>
+ <span class="k">return</span> <span class="bp">cls</span>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/evaluation.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/evaluation.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/evaluation.html
index caec83b..28f6b38 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/evaluation.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/evaluation.html
@@ -100,7 +100,7 @@
<span class="n">StructField</span><span class="p">(</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">DoubleType</span><span class="p">(),</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)]))</span>
<span class="n">java_class</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">evaluation</span><span class="o">.</span><span class="n">BinaryClassificationMetrics</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">java_class</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">_jdf</span><span class="p">)</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">BinaryClassificationMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BinaryClassificationMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
@@ -159,7 +159,7 @@
<span class="n">StructField</span><span class="p">(</span><span class="s2">"observation"</span><span class="p">,</span> <span class="n">DoubleType</span><span class="p">(),</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)]))</span>
<span class="n">java_class</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">evaluation</span><span class="o">.</span><span class="n">RegressionMetrics</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">java_class</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">_jdf</span><span class="p">)</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">RegressionMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RegressionMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
@@ -251,7 +251,7 @@
<span class="n">StructField</span><span class="p">(</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">DoubleType</span><span class="p">(),</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)]))</span>
<span class="n">java_class</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">evaluation</span><span class="o">.</span><span class="n">MulticlassMetrics</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">java_class</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">_jdf</span><span class="p">)</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">MulticlassMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MulticlassMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
<div class="viewcode-block" id="MulticlassMetrics.confusionMatrix"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.evaluation.MulticlassMetrics.confusionMatrix">[docs]</a> <span class="k">def</span> <span class="nf">confusionMatrix</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -405,7 +405,7 @@
<span class="n">df</span> <span class="o">=</span> <span class="n">sql_ctx</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">predictionAndLabels</span><span class="p">,</span>
<span class="n">schema</span><span class="o">=</span><span class="n">sql_ctx</span><span class="o">.</span><span class="n">_inferSchema</span><span class="p">(</span><span class="n">predictionAndLabels</span><span class="p">))</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">callMLlibFunc</span><span class="p">(</span><span class="s2">"newRankingMetrics"</span><span class="p">,</span> <span class="n">df</span><span class="o">.</span><span class="n">_jdf</span><span class="p">)</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">RankingMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RankingMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
<div class="viewcode-block" id="RankingMetrics.precisionAt"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.evaluation.RankingMetrics.precisionAt">[docs]</a> <span class="k">def</span> <span class="nf">precisionAt</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">k</span><span class="p">):</span>
@@ -492,7 +492,7 @@
<span class="n">schema</span><span class="o">=</span><span class="n">sql_ctx</span><span class="o">.</span><span class="n">_inferSchema</span><span class="p">(</span><span class="n">predictionAndLabels</span><span class="p">))</span>
<span class="n">java_class</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">evaluation</span><span class="o">.</span><span class="n">MultilabelMetrics</span>
<span class="n">java_model</span> <span class="o">=</span> <span class="n">java_class</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">_jdf</span><span class="p">)</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">MultilabelMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MultilabelMetrics</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">java_model</span><span class="p">)</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.4.0'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">precision</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
@@ -581,7 +581,7 @@
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
<span class="kn">import</span> <span class="nn">pyspark.mllib.evaluation</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">evaluation</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">evaluation</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
<span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[4]"</span><span class="p">)</span>\
<span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"mllib.evaluation tests"</span><span class="p">)</span>\
@@ -593,7 +593,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/feature.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/feature.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/feature.html
index 9f02b2f..1f24be8 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/feature.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/feature.html
@@ -634,7 +634,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s1">'1.5.0'</span><span class="p">)</span>
-<div class="viewcode-block" id="Word2VecModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.feature.Word2VecModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="Word2VecModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.feature.Word2VecModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a model from the given path.</span>
<span class="sd"> """</span>
@@ -835,7 +835,7 @@
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/fpm.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/fpm.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/fpm.html
index 6fce6b1..0283937 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/fpm.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/fpm.html
@@ -105,11 +105,11 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="FPGrowthModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.fpm.FPGrowthModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="FPGrowthModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.fpm.FPGrowthModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Load a model from the given path.</span>
<span class="sd"> """</span>
- <span class="n">model</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
+ <span class="n">model</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
<span class="n">wrapper</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">api</span><span class="o">.</span><span class="n">python</span><span class="o">.</span><span class="n">FPGrowthModelWrapper</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
<span class="k">return</span> <span class="n">FPGrowthModel</span><span class="p">(</span><span class="n">wrapper</span><span class="p">)</span></div></div>
@@ -123,7 +123,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.4.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="FPGrowth.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.fpm.FPGrowth.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">minSupport</span><span class="o">=</span><span class="mf">0.3</span><span class="p">,</span> <span class="n">numPartitions</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
+<div class="viewcode-block" id="FPGrowth.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.fpm.FPGrowth.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">minSupport</span><span class="o">=</span><span class="mf">0.3</span><span class="p">,</span> <span class="n">numPartitions</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Computes an FP-Growth model that contains frequent itemsets.</span>
@@ -185,7 +185,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.6.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="PrefixSpan.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.fpm.PrefixSpan.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">minSupport</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">maxPatternLength</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">maxLocalProjDBSize</span><span class="o">=</span><span class="mi">32000000</span><span class="p">):</span>
+<div class="viewcode-block" id="PrefixSpan.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.fpm.PrefixSpan.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">minSupport</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">maxPatternLength</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">maxLocalProjDBSize</span><span class="o">=</span><span class="mi">32000000</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Finds the complete set of frequent sequential patterns in the</span>
<span class="sd"> input sequences of itemsets.</span>
@@ -225,7 +225,7 @@
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
<span class="kn">import</span> <span class="nn">pyspark.mllib.fpm</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">fpm</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">fpm</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
<span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[4]"</span><span class="p">)</span>\
<span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"mllib.fpm tests"</span><span class="p">)</span>\
@@ -248,7 +248,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg.html
index d5c7355..23db8c7 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg.html
@@ -190,7 +190,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">([</span>
<span class="n">StructField</span><span class="p">(</span><span class="s2">"type"</span><span class="p">,</span> <span class="n">ByteType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s2">"size"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="kc">True</span><span class="p">),</span>
@@ -198,11 +198,11 @@
<span class="n">StructField</span><span class="p">(</span><span class="s2">"values"</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">DoubleType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span> <span class="kc">True</span><span class="p">)])</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"pyspark.mllib.linalg"</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"org.apache.spark.mllib.linalg.VectorUDT"</span>
<span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
@@ -237,7 +237,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">([</span>
<span class="n">StructField</span><span class="p">(</span><span class="s2">"type"</span><span class="p">,</span> <span class="n">ByteType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s2">"numRows"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span>
@@ -248,11 +248,11 @@
<span class="n">StructField</span><span class="p">(</span><span class="s2">"isTransposed"</span><span class="p">,</span> <span class="n">BooleanType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">)])</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"pyspark.mllib.linalg"</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"org.apache.spark.mllib.linalg.MatrixUDT"</span>
<span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
@@ -535,19 +535,19 @@
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">op</span><span class="p">)(</span><span class="n">other</span><span class="p">))</span>
<span class="k">return</span> <span class="n">func</span>
- <span class="n">__neg__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__neg__"</span><span class="p">)</span>
- <span class="n">__add__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__add__"</span><span class="p">)</span>
- <span class="n">__sub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__sub__"</span><span class="p">)</span>
- <span class="n">__mul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__mul__"</span><span class="p">)</span>
+ <span class="fm">__neg__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__neg__"</span><span class="p">)</span>
+ <span class="fm">__add__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__add__"</span><span class="p">)</span>
+ <span class="fm">__sub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__sub__"</span><span class="p">)</span>
+ <span class="fm">__mul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__mul__"</span><span class="p">)</span>
<span class="n">__div__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__div__"</span><span class="p">)</span>
- <span class="n">__truediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__truediv__"</span><span class="p">)</span>
- <span class="n">__mod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__mod__"</span><span class="p">)</span>
- <span class="n">__radd__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__radd__"</span><span class="p">)</span>
- <span class="n">__rsub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rsub__"</span><span class="p">)</span>
- <span class="n">__rmul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rmul__"</span><span class="p">)</span>
+ <span class="fm">__truediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__truediv__"</span><span class="p">)</span>
+ <span class="fm">__mod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__mod__"</span><span class="p">)</span>
+ <span class="fm">__radd__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__radd__"</span><span class="p">)</span>
+ <span class="fm">__rsub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rsub__"</span><span class="p">)</span>
+ <span class="fm">__rmul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rmul__"</span><span class="p">)</span>
<span class="n">__rdiv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rdiv__"</span><span class="p">)</span>
- <span class="n">__rtruediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rtruediv__"</span><span class="p">)</span>
- <span class="n">__rmod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rmod__"</span><span class="p">)</span></div>
+ <span class="fm">__rtruediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rtruediv__"</span><span class="p">)</span>
+ <span class="fm">__rmod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rmod__"</span><span class="p">)</span></div>
<div class="viewcode-block" id="SparseVector"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseVector">[docs]</a><span class="k">class</span> <span class="nc">SparseVector</span><span class="p">(</span><span class="n">Vector</span><span class="p">):</span>
@@ -867,7 +867,7 @@
<span class="k">return</span> <span class="mf">0.</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+ <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">result</span> <span class="o">=</span> <span class="mi">31</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span>
@@ -1069,7 +1069,7 @@
<span class="sd"> Column-major dense matrix.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">values</span><span class="p">,</span> <span class="n">isTransposed</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
- <span class="n">Matrix</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
+ <span class="n">Matrix</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
<span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">)</span> <span class="o">==</span> <span class="n">numRows</span> <span class="o">*</span> <span class="n">numCols</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="n">values</span>
@@ -1193,7 +1193,7 @@
<span class="sd">"""Sparse Matrix stored in CSC format."""</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">colPtrs</span><span class="p">,</span> <span class="n">rowIndices</span><span class="p">,</span> <span class="n">values</span><span class="p">,</span>
<span class="n">isTransposed</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
- <span class="n">Matrix</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
+ <span class="n">Matrix</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">colPtrs</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">rowIndices</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
@@ -1420,7 +1420,7 @@
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg/distributed.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg/distributed.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg/distributed.html
index db4a288..7b935ad 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg/distributed.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/linalg/distributed.html
@@ -1225,7 +1225,7 @@
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
<span class="kn">from</span> <span class="nn">pyspark.mllib.linalg</span> <span class="k">import</span> <span class="n">Matrices</span>
<span class="kn">import</span> <span class="nn">pyspark.mllib.linalg.distributed</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">distributed</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">distributed</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
<span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[2]"</span><span class="p">)</span>\
<span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"mllib.linalg.distributed tests"</span><span class="p">)</span>\
@@ -1237,7 +1237,7 @@
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/random.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/random.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/random.html
index 99f1fea..c8ba6f3 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/random.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/random.html
@@ -472,7 +472,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/mllib/recommendation.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/recommendation.html b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/recommendation.html
index 8478220..5660ecf 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/mllib/recommendation.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/mllib/recommendation.html
@@ -252,9 +252,9 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.3.1"</span><span class="p">)</span>
-<div class="viewcode-block" id="MatrixFactorizationModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.recommendation.MatrixFactorizationModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="MatrixFactorizationModel.load"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.recommendation.MatrixFactorizationModel.load">[docs]</a> <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Load a model from the given path"""</span>
- <span class="n">model</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
+ <span class="n">model</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_load_java</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
<span class="n">wrapper</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">api</span><span class="o">.</span><span class="n">python</span><span class="o">.</span><span class="n">MatrixFactorizationModelWrapper</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
<span class="k">return</span> <span class="n">MatrixFactorizationModel</span><span class="p">(</span><span class="n">wrapper</span><span class="p">)</span></div></div>
@@ -266,7 +266,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_prepare</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">ratings</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_prepare</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">ratings</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ratings</span><span class="p">,</span> <span class="n">RDD</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ratings</span><span class="p">,</span> <span class="n">DataFrame</span><span class="p">):</span>
@@ -285,7 +285,7 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"0.9.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="ALS.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.recommendation.ALS.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">ratings</span><span class="p">,</span> <span class="n">rank</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">lambda_</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span> <span class="n">blocks</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">nonnegative</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+<div class="viewcode-block" id="ALS.train"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.recommendation.ALS.train">[docs]</a> <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">ratings</span><span class="p">,</span> <span class="n">rank</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">lambda_</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span> <span class="n">blocks</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">nonnegative</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Train a matrix factorization model given an RDD of ratings by users</span>
@@ -317,13 +317,13 @@
<span class="sd"> of None will use system time as the seed.</span>
<span class="sd"> (default: None)</span>
<span class="sd"> """</span>
- <span class="n">model</span> <span class="o">=</span> <span class="n">callMLlibFunc</span><span class="p">(</span><span class="s2">"trainALSModel"</span><span class="p">,</span> <span class="n">cls</span><span class="o">.</span><span class="n">_prepare</span><span class="p">(</span><span class="n">ratings</span><span class="p">),</span> <span class="n">rank</span><span class="p">,</span> <span class="n">iterations</span><span class="p">,</span>
+ <span class="n">model</span> <span class="o">=</span> <span class="n">callMLlibFunc</span><span class="p">(</span><span class="s2">"trainALSModel"</span><span class="p">,</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_prepare</span><span class="p">(</span><span class="n">ratings</span><span class="p">),</span> <span class="n">rank</span><span class="p">,</span> <span class="n">iterations</span><span class="p">,</span>
<span class="n">lambda_</span><span class="p">,</span> <span class="n">blocks</span><span class="p">,</span> <span class="n">nonnegative</span><span class="p">,</span> <span class="n">seed</span><span class="p">)</span>
<span class="k">return</span> <span class="n">MatrixFactorizationModel</span><span class="p">(</span><span class="n">model</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"0.9.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="ALS.trainImplicit"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.recommendation.ALS.trainImplicit">[docs]</a> <span class="k">def</span> <span class="nf">trainImplicit</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">ratings</span><span class="p">,</span> <span class="n">rank</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">lambda_</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span> <span class="n">blocks</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
+<div class="viewcode-block" id="ALS.trainImplicit"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.recommendation.ALS.trainImplicit">[docs]</a> <span class="k">def</span> <span class="nf">trainImplicit</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">ratings</span><span class="p">,</span> <span class="n">rank</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">lambda_</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span> <span class="n">blocks</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
<span class="n">nonnegative</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Train a matrix factorization model given an RDD of 'implicit</span>
@@ -358,7 +358,7 @@
<span class="sd"> of None will use system time as the seed.</span>
<span class="sd"> (default: None)</span>
<span class="sd"> """</span>
- <span class="n">model</span> <span class="o">=</span> <span class="n">callMLlibFunc</span><span class="p">(</span><span class="s2">"trainImplicitALSModel"</span><span class="p">,</span> <span class="n">cls</span><span class="o">.</span><span class="n">_prepare</span><span class="p">(</span><span class="n">ratings</span><span class="p">),</span> <span class="n">rank</span><span class="p">,</span>
+ <span class="n">model</span> <span class="o">=</span> <span class="n">callMLlibFunc</span><span class="p">(</span><span class="s2">"trainImplicitALSModel"</span><span class="p">,</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_prepare</span><span class="p">(</span><span class="n">ratings</span><span class="p">),</span> <span class="n">rank</span><span class="p">,</span>
<span class="n">iterations</span><span class="p">,</span> <span class="n">lambda_</span><span class="p">,</span> <span class="n">blocks</span><span class="p">,</span> <span class="n">alpha</span><span class="p">,</span> <span class="n">nonnegative</span><span class="p">,</span> <span class="n">seed</span><span class="p">)</span>
<span class="k">return</span> <span class="n">MatrixFactorizationModel</span><span class="p">(</span><span class="n">model</span><span class="p">)</span></div></div>
@@ -367,7 +367,7 @@
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">import</span> <span class="nn">pyspark.mllib.recommendation</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SQLContext</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">recommendation</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">mllib</span><span class="o">.</span><span class="n">recommendation</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s1">'local[4]'</span><span class="p">,</span> <span class="s1">'PythonTest'</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'sc'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'sqlContext'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SQLContext</span><span class="p">(</span><span class="n">sc</span><span class="p">)</span>
@@ -377,7 +377,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[14/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
Fix signature description broken in PySpark API documentation in 2.1.2
Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/6bbac496
Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/6bbac496
Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/6bbac496
Branch: refs/heads/asf-site
Commit: 6bbac4966e79156af1e57d6c1da013ee9e828ea6
Parents: 7751277
Author: hyukjinkwon <gu...@apache.org>
Authored: Tue Jul 3 01:32:55 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Tue Jul 3 01:32:55 2018 +0800
----------------------------------------------------------------------
.../python/_modules/pyspark/accumulators.html | 2 +-
.../api/python/_modules/pyspark/broadcast.html | 6 +-
.../2.1.2/api/python/_modules/pyspark/conf.html | 4 +-
.../api/python/_modules/pyspark/context.html | 8 +-
.../api/python/_modules/pyspark/files.html | 10 +-
.../_modules/pyspark/ml/classification.html | 54 ++--
.../python/_modules/pyspark/ml/clustering.html | 24 +-
.../python/_modules/pyspark/ml/evaluation.html | 10 +-
.../api/python/_modules/pyspark/ml/feature.html | 60 ++---
.../api/python/_modules/pyspark/ml/linalg.html | 42 +--
.../api/python/_modules/pyspark/ml/param.html | 8 +-
.../_modules/pyspark/ml/param/shared.html | 52 ++--
.../python/_modules/pyspark/ml/pipeline.html | 28 +-
.../_modules/pyspark/ml/recommendation.html | 6 +-
.../python/_modules/pyspark/ml/regression.html | 46 ++--
.../api/python/_modules/pyspark/ml/tuning.html | 12 +-
.../api/python/_modules/pyspark/ml/util.html | 30 +--
.../api/python/_modules/pyspark/ml/wrapper.html | 8 +-
.../_modules/pyspark/mllib/classification.html | 28 +-
.../_modules/pyspark/mllib/clustering.html | 32 +--
.../python/_modules/pyspark/mllib/common.html | 12 +-
.../_modules/pyspark/mllib/evaluation.html | 14 +-
.../python/_modules/pyspark/mllib/feature.html | 4 +-
.../api/python/_modules/pyspark/mllib/fpm.html | 12 +-
.../python/_modules/pyspark/mllib/linalg.html | 42 +--
.../pyspark/mllib/linalg/distributed.html | 4 +-
.../python/_modules/pyspark/mllib/random.html | 2 +-
.../_modules/pyspark/mllib/recommendation.html | 18 +-
.../_modules/pyspark/mllib/regression.html | 22 +-
.../api/python/_modules/pyspark/mllib/tree.html | 40 +--
.../api/python/_modules/pyspark/mllib/util.html | 20 +-
.../api/python/_modules/pyspark/profiler.html | 4 +-
.../2.1.2/api/python/_modules/pyspark/rdd.html | 16 +-
.../python/_modules/pyspark/serializers.html | 70 ++---
.../api/python/_modules/pyspark/sql/column.html | 68 ++---
.../python/_modules/pyspark/sql/context.html | 22 +-
.../python/_modules/pyspark/sql/dataframe.html | 26 +-
.../python/_modules/pyspark/sql/functions.html | 18 +-
.../api/python/_modules/pyspark/sql/group.html | 16 +-
.../python/_modules/pyspark/sql/readwriter.html | 4 +-
.../python/_modules/pyspark/sql/session.html | 6 +-
.../python/_modules/pyspark/sql/streaming.html | 4 +-
.../api/python/_modules/pyspark/sql/types.html | 74 ++---
.../api/python/_modules/pyspark/sql/window.html | 2 +-
.../_modules/pyspark/streaming/context.html | 28 +-
.../_modules/pyspark/streaming/dstream.html | 10 +-
.../_modules/pyspark/streaming/kafka.html | 24 +-
site/docs/2.1.2/api/python/_static/pygments.css | 4 +
site/docs/2.1.2/api/python/pyspark.html | 47 ++--
site/docs/2.1.2/api/python/pyspark.ml.html | 150 +++++------
site/docs/2.1.2/api/python/pyspark.mllib.html | 63 ++---
site/docs/2.1.2/api/python/pyspark.sql.html | 267 +++++++++----------
.../2.1.2/api/python/pyspark.streaming.html | 3 +-
site/docs/2.1.2/api/python/searchindex.js | 2 +-
54 files changed, 788 insertions(+), 800 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/accumulators.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/accumulators.html b/site/docs/2.1.2/api/python/_modules/pyspark/accumulators.html
index b440fa0..5b1c432 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/accumulators.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/accumulators.html
@@ -310,7 +310,7 @@
<span class="n">thread</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
<span class="k">return</span> <span class="n">server</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">()</span>
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/broadcast.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/broadcast.html b/site/docs/2.1.2/api/python/_modules/pyspark/broadcast.html
index fa67a7b..477d268 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/broadcast.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/broadcast.html
@@ -131,7 +131,7 @@
<span class="k">except</span> <span class="n">pickle</span><span class="o">.</span><span class="n">PickleError</span><span class="p">:</span>
<span class="k">raise</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="n">msg</span> <span class="o">=</span> <span class="s2">"Could not serialize broadcast: "</span> <span class="o">+</span> <span class="n">e</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span> <span class="o">+</span> <span class="s2">": "</span> <span class="o">+</span> <span class="n">e</span><span class="o">.</span><span class="n">message</span>
+ <span class="n">msg</span> <span class="o">=</span> <span class="s2">"Could not serialize broadcast: "</span> <span class="o">+</span> <span class="n">e</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">+</span> <span class="s2">": "</span> <span class="o">+</span> <span class="n">e</span><span class="o">.</span><span class="n">message</span>
<span class="n">print_exec</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">stderr</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">pickle</span><span class="o">.</span><span class="n">PicklingError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
@@ -191,7 +191,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s2">"_registry"</span><span class="p">,</span> <span class="nb">set</span><span class="p">())</span>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s2">"_registry"</span><span class="p">,</span> <span class="nb">set</span><span class="p">())</span>
<span class="k">def</span> <span class="nf">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">for</span> <span class="n">bcast</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_registry</span><span class="p">:</span>
@@ -204,7 +204,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">_registry</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">()</span>
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/conf.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/conf.html b/site/docs/2.1.2/api/python/_modules/pyspark/conf.html
index 4fb090b..b470d58 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/conf.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/conf.html
@@ -117,7 +117,7 @@
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version</span> <span class="o">></span> <span class="s1">'3'</span><span class="p">:</span>
<span class="n">unicode</span> <span class="o">=</span> <span class="nb">str</span>
- <span class="n">__doc__</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">r"(\W|^)[uU](['])"</span><span class="p">,</span> <span class="s1">r'\1\2'</span><span class="p">,</span> <span class="n">__doc__</span><span class="p">)</span>
+ <span class="vm">__doc__</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(\W|^)[uU](['])"</span><span class="p">,</span> <span class="sa">r</span><span class="s1">'\1\2'</span><span class="p">,</span> <span class="vm">__doc__</span><span class="p">)</span>
<div class="viewcode-block" id="SparkConf"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkConf">[docs]</a><span class="k">class</span> <span class="nc">SparkConf</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
@@ -268,7 +268,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/context.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/context.html b/site/docs/2.1.2/api/python/_modules/pyspark/context.html
index 529ef29..e3516be 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/context.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/context.html
@@ -297,7 +297,7 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">JavaSparkContext</span><span class="p">(</span><span class="n">jconf</span><span class="p">)</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_ensure_initialized</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">instance</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">gateway</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_ensure_initialized</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">instance</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">gateway</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Checks whether a SparkContext is initialized or not.</span>
<span class="sd"> Throws error if a SparkContext is already running.</span>
@@ -347,7 +347,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="SparkContext.getOrCreate"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkContext.getOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getOrCreate</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+<div class="viewcode-block" id="SparkContext.getOrCreate"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkContext.getOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getOrCreate</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Get or instantiate a SparkContext and register it as a singleton object.</span>
@@ -366,7 +366,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">_jsc</span><span class="o">.</span><span class="n">setLogLevel</span><span class="p">(</span><span class="n">logLevel</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="SparkContext.setSystemProperty"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkContext.setSystemProperty">[docs]</a> <span class="k">def</span> <span class="nf">setSystemProperty</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+<div class="viewcode-block" id="SparkContext.setSystemProperty"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkContext.setSystemProperty">[docs]</a> <span class="k">def</span> <span class="nf">setSystemProperty</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Set a Java system property, such as spark.executor.memory. This must</span>
<span class="sd"> must be invoked before instantiating SparkContext.</span>
@@ -1053,7 +1053,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/files.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/files.html b/site/docs/2.1.2/api/python/_modules/pyspark/files.html
index 25f4f0a..f983621 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/files.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/files.html
@@ -87,7 +87,7 @@
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"Do not construct SparkFiles objects"</span><span class="p">)</span>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="SparkFiles.get"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkFiles.get">[docs]</a> <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">filename</span><span class="p">):</span>
+<div class="viewcode-block" id="SparkFiles.get"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkFiles.get">[docs]</a> <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">filename</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Get the absolute path of a file added through C{SparkContext.addFile()}.</span>
<span class="sd"> """</span>
@@ -95,16 +95,16 @@
<span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
-<div class="viewcode-block" id="SparkFiles.getRootDirectory"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkFiles.getRootDirectory">[docs]</a> <span class="k">def</span> <span class="nf">getRootDirectory</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+<div class="viewcode-block" id="SparkFiles.getRootDirectory"><a class="viewcode-back" href="../../pyspark.html#pyspark.SparkFiles.getRootDirectory">[docs]</a> <span class="k">def</span> <span class="nf">getRootDirectory</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Get the root directory that contains files added through</span>
<span class="sd"> C{SparkContext.addFile()}.</span>
<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">cls</span><span class="o">.</span><span class="n">_is_running_on_worker</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_root_directory</span>
+ <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_is_running_on_worker</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_root_directory</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># This will have to change if we support multiple SparkContexts:</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">SparkFiles</span><span class="o">.</span><span class="n">getRootDirectory</span><span class="p">()</span></div></div>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">SparkFiles</span><span class="o">.</span><span class="n">getRootDirectory</span><span class="p">()</span></div></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/classification.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/classification.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/classification.html
index ab909b8..1ed4a57 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/classification.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/classification.html
@@ -184,7 +184,7 @@
<span class="nd">@keyword_only</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">thresholds</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">probabilityCol</span><span class="o">=</span><span class="s2">"probability"</span><span class="p">,</span>
<span class="n">rawPredictionCol</span><span class="o">=</span><span class="s2">"rawPrediction"</span><span class="p">,</span> <span class="n">standardization</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">weightCol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">aggregationDepth</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">family</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">):</span>
@@ -196,10 +196,10 @@
<span class="sd"> aggregationDepth=2, family="auto")</span>
<span class="sd"> If the threshold and thresholds Params are both set, they must be equivalent.</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">LogisticRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">LogisticRegression</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.classification.LogisticRegression"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">E</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">family</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1E-6</span><span class="p">,</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">family</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_checkThresholdConsistency</span><span class="p">()</span>
@@ -207,7 +207,7 @@
<span class="nd">@keyword_only</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.3.0"</span><span class="p">)</span>
<div class="viewcode-block" id="LogisticRegression.setParams"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.LogisticRegression.setParams">[docs]</a> <span class="k">def</span> <span class="nf">setParams</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">regParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">elasticNetParam</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">fitIntercept</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">thresholds</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">probabilityCol</span><span class="o">=</span><span class="s2">"probability"</span><span class="p">,</span>
<span class="n">rawPredictionCol</span><span class="o">=</span><span class="s2">"rawPrediction"</span><span class="p">,</span> <span class="n">standardization</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">weightCol</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">aggregationDepth</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">family</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">):</span>
@@ -293,7 +293,7 @@
<span class="s2">" thresholds: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">ts</span><span class="p">)))</span>
<span class="n">t</span> <span class="o">=</span> <span class="mf">1.0</span><span class="o">/</span><span class="p">(</span><span class="mf">1.0</span> <span class="o">+</span> <span class="n">ts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">/</span><span class="n">ts</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
<span class="n">t2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">getOrDefault</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">threshold</span><span class="p">)</span>
- <span class="k">if</span> <span class="nb">abs</span><span class="p">(</span><span class="n">t2</span> <span class="o">-</span> <span class="n">t</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">1</span><span class="n">E</span><span class="o">-</span><span class="mi">5</span><span class="p">:</span>
+ <span class="k">if</span> <span class="nb">abs</span><span class="p">(</span><span class="n">t2</span> <span class="o">-</span> <span class="n">t</span><span class="p">)</span> <span class="o">>=</span> <span class="mf">1E-5</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Logistic Regression getThreshold found inconsistent values for"</span> <span class="o">+</span>
<span class="s2">" threshold (</span><span class="si">%g</span><span class="s2">) and thresholds (equivalent to </span><span class="si">%g</span><span class="s2">)"</span> <span class="o">%</span> <span class="p">(</span><span class="n">t2</span><span class="p">,</span> <span class="n">t</span><span class="p">))</span>
@@ -366,7 +366,7 @@
<span class="k">return</span> <span class="n">BinaryLogisticRegressionTrainingSummary</span><span class="p">(</span><span class="n">java_blrt_summary</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"No training summary available for this </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
@@ -590,7 +590,7 @@
<span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">supportedImpurities</span><span class="p">),</span> <span class="n">typeConverter</span><span class="o">=</span><span class="n">TypeConverters</span><span class="o">.</span><span class="n">toString</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">TreeClassifierParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">TreeClassifierParams</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.6.0"</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">setImpurity</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -688,7 +688,7 @@
<span class="sd"> maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \</span>
<span class="sd"> seed=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">DecisionTreeClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DecisionTreeClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.classification.DecisionTreeClassifier"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
@@ -817,7 +817,7 @@
<span class="sd"> maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini", \</span>
<span class="sd"> numTrees=20, featureSubsetStrategy="auto", seed=None, subsamplingRate=1.0)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">RandomForestClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RandomForestClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.classification.RandomForestClassifier"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
@@ -959,7 +959,7 @@
<span class="sd"> maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, \</span>
<span class="sd"> lossType="logistic", maxIter=20, stepSize=0.1, seed=None, subsamplingRate=1.0)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">GBTClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GBTClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.classification.GBTClassifier"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxDepth</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxBins</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">minInstancesPerNode</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">minInfoGain</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
@@ -1104,7 +1104,7 @@
<span class="sd"> probabilityCol="probability", rawPredictionCol="rawPrediction", smoothing=1.0, \</span>
<span class="sd"> modelType="multinomial", thresholds=None, weightCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">NaiveBayes</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">NaiveBayes</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.classification.NaiveBayes"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">smoothing</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">modelType</span><span class="o">=</span><span class="s2">"multinomial"</span><span class="p">)</span>
@@ -1252,24 +1252,24 @@
<span class="nd">@keyword_only</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">layers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">blockSize</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span> <span class="n">stepSize</span><span class="o">=</span><span class="mf">0.03</span><span class="p">,</span>
+ <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">layers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">blockSize</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span> <span class="n">stepSize</span><span class="o">=</span><span class="mf">0.03</span><span class="p">,</span>
<span class="n">solver</span><span class="o">=</span><span class="s2">"l-bfgs"</span><span class="p">,</span> <span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \</span>
<span class="sd"> maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, stepSize=0.03, \</span>
<span class="sd"> solver="l-bfgs", initialWeights=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">MultilayerPerceptronClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MultilayerPerceptronClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.classification.MultilayerPerceptronClassifier"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">E</span><span class="o">-</span><span class="mi">4</span><span class="p">,</span> <span class="n">blockSize</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span> <span class="n">stepSize</span><span class="o">=</span><span class="mf">0.03</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"l-bfgs"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1E-4</span><span class="p">,</span> <span class="n">blockSize</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span> <span class="n">stepSize</span><span class="o">=</span><span class="mf">0.03</span><span class="p">,</span> <span class="n">solver</span><span class="o">=</span><span class="s2">"l-bfgs"</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="nd">@keyword_only</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.6.0"</span><span class="p">)</span>
<div class="viewcode-block" id="MultilayerPerceptronClassifier.setParams"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.MultilayerPerceptronClassifier.setParams">[docs]</a> <span class="k">def</span> <span class="nf">setParams</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span>
- <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">6</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">layers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">blockSize</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span> <span class="n">stepSize</span><span class="o">=</span><span class="mf">0.03</span><span class="p">,</span>
+ <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">layers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">blockSize</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span> <span class="n">stepSize</span><span class="o">=</span><span class="mf">0.03</span><span class="p">,</span>
<span class="n">solver</span><span class="o">=</span><span class="s2">"l-bfgs"</span><span class="p">,</span> <span class="n">initialWeights</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \</span>
@@ -1447,7 +1447,7 @@
<span class="sd"> __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \</span>
<span class="sd"> classifier=None, weightCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">OneVsRest</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">OneVsRest</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_set</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -1541,12 +1541,12 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="OneVsRest.read"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.OneVsRest.read">[docs]</a> <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+<div class="viewcode-block" id="OneVsRest.read"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.OneVsRest.read">[docs]</a> <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""Returns an MLReader instance for this class."""</span>
- <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="n">cls</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_from_java</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">java_stage</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_from_java</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">java_stage</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Given a Java OneVsRest, create and return a Python wrapper of it.</span>
<span class="sd"> Used for ML persistence.</span>
@@ -1555,7 +1555,7 @@
<span class="n">labelCol</span> <span class="o">=</span> <span class="n">java_stage</span><span class="o">.</span><span class="n">getLabelCol</span><span class="p">()</span>
<span class="n">predictionCol</span> <span class="o">=</span> <span class="n">java_stage</span><span class="o">.</span><span class="n">getPredictionCol</span><span class="p">()</span>
<span class="n">classifier</span> <span class="o">=</span> <span class="n">JavaParams</span><span class="o">.</span><span class="n">_from_java</span><span class="p">(</span><span class="n">java_stage</span><span class="o">.</span><span class="n">getClassifier</span><span class="p">())</span>
- <span class="n">py_stage</span> <span class="o">=</span> <span class="n">cls</span><span class="p">(</span><span class="n">featuresCol</span><span class="o">=</span><span class="n">featuresCol</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="n">labelCol</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="n">predictionCol</span><span class="p">,</span>
+ <span class="n">py_stage</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">featuresCol</span><span class="o">=</span><span class="n">featuresCol</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="n">labelCol</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="n">predictionCol</span><span class="p">,</span>
<span class="n">classifier</span><span class="o">=</span><span class="n">classifier</span><span class="p">)</span>
<span class="n">py_stage</span><span class="o">.</span><span class="n">_resetUid</span><span class="p">(</span><span class="n">java_stage</span><span class="o">.</span><span class="n">uid</span><span class="p">())</span>
<span class="k">return</span> <span class="n">py_stage</span>
@@ -1588,7 +1588,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">models</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">OneVsRestModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">OneVsRestModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">models</span> <span class="o">=</span> <span class="n">models</span>
<span class="k">def</span> <span class="nf">_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">):</span>
@@ -1667,12 +1667,12 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"2.0.0"</span><span class="p">)</span>
-<div class="viewcode-block" id="OneVsRestModel.read"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.OneVsRestModel.read">[docs]</a> <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+<div class="viewcode-block" id="OneVsRestModel.read"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.classification.OneVsRestModel.read">[docs]</a> <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">"""Returns an MLReader instance for this class."""</span>
- <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="n">cls</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">JavaMLReader</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_from_java</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">java_stage</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_from_java</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">java_stage</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Given a Java OneVsRestModel, create and return a Python wrapper of it.</span>
<span class="sd"> Used for ML persistence.</span>
@@ -1682,7 +1682,7 @@
<span class="n">predictionCol</span> <span class="o">=</span> <span class="n">java_stage</span><span class="o">.</span><span class="n">getPredictionCol</span><span class="p">()</span>
<span class="n">classifier</span> <span class="o">=</span> <span class="n">JavaParams</span><span class="o">.</span><span class="n">_from_java</span><span class="p">(</span><span class="n">java_stage</span><span class="o">.</span><span class="n">getClassifier</span><span class="p">())</span>
<span class="n">models</span> <span class="o">=</span> <span class="p">[</span><span class="n">JavaParams</span><span class="o">.</span><span class="n">_from_java</span><span class="p">(</span><span class="n">model</span><span class="p">)</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="n">java_stage</span><span class="o">.</span><span class="n">models</span><span class="p">()]</span>
- <span class="n">py_stage</span> <span class="o">=</span> <span class="n">cls</span><span class="p">(</span><span class="n">models</span><span class="o">=</span><span class="n">models</span><span class="p">)</span><span class="o">.</span><span class="n">setPredictionCol</span><span class="p">(</span><span class="n">predictionCol</span><span class="p">)</span><span class="o">.</span><span class="n">setLabelCol</span><span class="p">(</span><span class="n">labelCol</span><span class="p">)</span>\
+ <span class="n">py_stage</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">models</span><span class="o">=</span><span class="n">models</span><span class="p">)</span><span class="o">.</span><span class="n">setPredictionCol</span><span class="p">(</span><span class="n">predictionCol</span><span class="p">)</span><span class="o">.</span><span class="n">setLabelCol</span><span class="p">(</span><span class="n">labelCol</span><span class="p">)</span>\
<span class="o">.</span><span class="n">setFeaturesCol</span><span class="p">(</span><span class="n">featuresCol</span><span class="p">)</span><span class="o">.</span><span class="n">setClassifier</span><span class="p">(</span><span class="n">classifier</span><span class="p">)</span>
<span class="n">py_stage</span><span class="o">.</span><span class="n">_resetUid</span><span class="p">(</span><span class="n">java_stage</span><span class="o">.</span><span class="n">uid</span><span class="p">())</span>
<span class="k">return</span> <span class="n">py_stage</span>
@@ -1703,11 +1703,11 @@
<span class="k">return</span> <span class="n">_java_obj</span></div>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">import</span> <span class="nn">pyspark.ml.classification</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">classification</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">classification</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="c1"># The small batch size here ensures that we see multiple batches,</span>
<span class="c1"># even in these small test examples:</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/clustering.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/clustering.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/clustering.html
index 016d76e..e3cb508 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/clustering.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/clustering.html
@@ -180,7 +180,7 @@
<span class="k">return</span> <span class="n">GaussianMixtureSummary</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_call_java</span><span class="p">(</span><span class="s2">"summary"</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"No training summary available for this </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span></div>
<span class="nd">@inherit_doc</span>
@@ -276,7 +276,7 @@
<span class="sd"> __init__(self, featuresCol="features", predictionCol="prediction", k=2, \</span>
<span class="sd"> probabilityCol="probability", tol=0.01, maxIter=100, seed=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">GaussianMixture</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GaussianMixture</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.clustering.GaussianMixture"</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">k</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
@@ -390,7 +390,7 @@
<span class="k">return</span> <span class="n">KMeansSummary</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_call_java</span><span class="p">(</span><span class="s2">"summary"</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"No training summary available for this </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span></div>
<span class="nd">@inherit_doc</span>
@@ -454,14 +454,14 @@
<span class="nd">@keyword_only</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
- <span class="n">initMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span> <span class="n">initSteps</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">4</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">initMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span> <span class="n">initSteps</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-4</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> __init__(self, featuresCol="features", predictionCol="prediction", k=2, \</span>
<span class="sd"> initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">KMeans</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">KMeans</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.clustering.KMeans"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">k</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">initMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span> <span class="n">initSteps</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">4</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">k</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">initMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span> <span class="n">initSteps</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-4</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -471,7 +471,7 @@
<span class="nd">@keyword_only</span>
<span class="nd">@since</span><span class="p">(</span><span class="s2">"1.5.0"</span><span class="p">)</span>
<div class="viewcode-block" id="KMeans.setParams"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.clustering.KMeans.setParams">[docs]</a> <span class="k">def</span> <span class="nf">setParams</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">featuresCol</span><span class="o">=</span><span class="s2">"features"</span><span class="p">,</span> <span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
- <span class="n">initMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span> <span class="n">initSteps</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mi">1</span><span class="n">e</span><span class="o">-</span><span class="mi">4</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">initMode</span><span class="o">=</span><span class="s2">"k-means||"</span><span class="p">,</span> <span class="n">initSteps</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-4</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> setParams(self, featuresCol="features", predictionCol="prediction", k=2, \</span>
<span class="sd"> initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, seed=None)</span>
@@ -563,7 +563,7 @@
<span class="k">return</span> <span class="n">BisectingKMeansSummary</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_call_java</span><span class="p">(</span><span class="s2">"summary"</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"No training summary available for this </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span></div>
<span class="nd">@inherit_doc</span>
@@ -635,7 +635,7 @@
<span class="sd"> __init__(self, featuresCol="features", predictionCol="prediction", maxIter=20, \</span>
<span class="sd"> seed=None, k=4, minDivisibleClusterSize=1.0)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">BisectingKMeans</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BisectingKMeans</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.clustering.BisectingKMeans"</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">minDivisibleClusterSize</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
@@ -958,7 +958,7 @@
<span class="sd"> docConcentration=None, topicConcentration=None,\</span>
<span class="sd"> topicDistributionCol="topicDistribution", keepLastCheckpoint=True):</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">LDA</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">LDA</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.clustering.LDA"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">checkpointInterval</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
<span class="n">k</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s2">"online"</span><span class="p">,</span> <span class="n">learningOffset</span><span class="o">=</span><span class="mf">1024.0</span><span class="p">,</span> <span class="n">learningDecay</span><span class="o">=</span><span class="mf">0.51</span><span class="p">,</span>
@@ -1174,11 +1174,11 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">getOrDefault</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keepLastCheckpoint</span><span class="p">)</span></div></div>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">import</span> <span class="nn">pyspark.ml.clustering</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">clustering</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">clustering</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="c1"># The small batch size here ensures that we see multiple batches,</span>
<span class="c1"># even in these small test examples:</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/evaluation.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/evaluation.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/evaluation.html
index 4fbdf54..ddaf4cb 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/evaluation.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/evaluation.html
@@ -191,7 +191,7 @@
<span class="sd"> __init__(self, rawPredictionCol="rawPrediction", labelCol="label", \</span>
<span class="sd"> metricName="areaUnderROC")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">BinaryClassificationEvaluator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BinaryClassificationEvaluator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.evaluation.BinaryClassificationEvaluator"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">rawPredictionCol</span><span class="o">=</span><span class="s2">"rawPrediction"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span>
@@ -269,7 +269,7 @@
<span class="sd"> __init__(self, predictionCol="prediction", labelCol="label", \</span>
<span class="sd"> metricName="rmse")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">RegressionEvaluator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RegressionEvaluator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.evaluation.RegressionEvaluator"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span>
@@ -342,7 +342,7 @@
<span class="sd"> __init__(self, predictionCol="prediction", labelCol="label", \</span>
<span class="sd"> metricName="f1")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">MulticlassClassificationEvaluator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MulticlassClassificationEvaluator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">predictionCol</span><span class="o">=</span><span class="s2">"prediction"</span><span class="p">,</span> <span class="n">labelCol</span><span class="o">=</span><span class="s2">"label"</span><span class="p">,</span>
@@ -376,12 +376,12 @@
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div></div>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">import</span> <span class="nn">tempfile</span>
<span class="kn">import</span> <span class="nn">pyspark.ml.evaluation</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">evaluation</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">evaluation</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="c1"># The small batch size here ensures that we see multiple batches,</span>
<span class="c1"># even in these small test examples:</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[13/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/feature.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/feature.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/feature.html
index 776d399..3a2c2a1 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/feature.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/feature.html
@@ -137,7 +137,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, threshold=0.0, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">Binarizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">Binarizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.Binarizer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">threshold</span><span class="o">=</span><span class="mf">0.0</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -223,7 +223,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, splits=None, inputCol=None, outputCol=None, handleInvalid="error")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">Bucketizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">Bucketizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.Bucketizer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">handleInvalid</span><span class="o">=</span><span class="s2">"error"</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -336,7 +336,7 @@
<span class="sd"> __init__(self, minTF=1.0, minDF=1.0, vocabSize=1 << 18, binary=False, inputCol=None,\</span>
<span class="sd"> outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">CountVectorizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CountVectorizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.CountVectorizer"</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">minTF</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">minDF</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">vocabSize</span><span class="o">=</span><span class="mi">1</span> <span class="o"><<</span> <span class="mi">18</span><span class="p">,</span> <span class="n">binary</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
@@ -469,7 +469,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, inverse=False, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">DCT</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DCT</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.DCT"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">inverse</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -533,7 +533,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, scalingVec=None, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">ElementwiseProduct</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">ElementwiseProduct</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.ElementwiseProduct"</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -603,7 +603,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, numFeatures=1 << 18, binary=False, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HashingTF</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HashingTF</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.HashingTF"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">numFeatures</span><span class="o">=</span><span class="mi">1</span> <span class="o"><<</span> <span class="mi">18</span><span class="p">,</span> <span class="n">binary</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -676,7 +676,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, minDocFreq=0, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">IDF</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">IDF</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.IDF"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">minDocFreq</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -766,7 +766,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">MaxAbsScaler</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MaxAbsScaler</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.MaxAbsScaler"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">()</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -860,7 +860,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, min=0.0, max=1.0, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">MinMaxScaler</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MinMaxScaler</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.MinMaxScaler"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="nb">min</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="nb">max</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -978,7 +978,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, n=2, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">NGram</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">NGram</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.NGram"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">n</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -1042,7 +1042,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, p=2.0, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">Normalizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">Normalizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.Normalizer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">2.0</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -1122,7 +1122,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, includeFirst=True, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">OneHotEncoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">OneHotEncoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.OneHotEncoder"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">dropLast</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -1187,7 +1187,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, degree=2, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">PolynomialExpansion</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PolynomialExpansion</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span>
<span class="s2">"org.apache.spark.ml.feature.PolynomialExpansion"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">degree</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
@@ -1283,7 +1283,7 @@
<span class="sd"> __init__(self, numBuckets=2, inputCol=None, outputCol=None, relativeError=0.001, \</span>
<span class="sd"> handleInvalid="error")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">QuantileDiscretizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">QuantileDiscretizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.QuantileDiscretizer"</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">numBuckets</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">relativeError</span><span class="o">=</span><span class="mf">0.001</span><span class="p">,</span> <span class="n">handleInvalid</span><span class="o">=</span><span class="s2">"error"</span><span class="p">)</span>
@@ -1409,7 +1409,7 @@
<span class="sd"> __init__(self, minTokenLength=1, gaps=True, pattern="\\s+", inputCol=None, \</span>
<span class="sd"> outputCol=None, toLowercase=True)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">RegexTokenizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RegexTokenizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.RegexTokenizer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">minTokenLength</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">gaps</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">pattern</span><span class="o">=</span><span class="s2">"</span><span class="se">\\</span><span class="s2">s+"</span><span class="p">,</span> <span class="n">toLowercase</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -1513,7 +1513,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, statement=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">SQLTransformer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SQLTransformer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.SQLTransformer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -1591,7 +1591,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, withMean=False, withStd=True, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">StandardScaler</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">StandardScaler</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.StandardScaler"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">withMean</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">withStd</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -1707,7 +1707,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, inputCol=None, outputCol=None, handleInvalid="error")</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">StringIndexer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">StringIndexer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.StringIndexer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">handleInvalid</span><span class="o">=</span><span class="s2">"error"</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -1765,7 +1765,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, inputCol=None, outputCol=None, labels=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">IndexToString</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">IndexToString</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.IndexToString"</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -1827,7 +1827,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">StopWordsRemover</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">StopWordsRemover</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.StopWordsRemover"</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">stopWords</span><span class="o">=</span><span class="n">StopWordsRemover</span><span class="o">.</span><span class="n">loadDefaultStopWords</span><span class="p">(</span><span class="s2">"english"</span><span class="p">),</span>
@@ -1923,7 +1923,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">Tokenizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">Tokenizer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.Tokenizer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -1967,7 +1967,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, inputCols=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">VectorAssembler</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">VectorAssembler</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.VectorAssembler"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -2064,7 +2064,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, maxCategories=20, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">VectorIndexer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">VectorIndexer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.VectorIndexer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">maxCategories</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -2179,7 +2179,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, inputCol=None, outputCol=None, indices=None, names=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">VectorSlicer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">VectorSlicer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.VectorSlicer"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">indices</span><span class="o">=</span><span class="p">[],</span> <span class="n">names</span><span class="o">=</span><span class="p">[])</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -2301,7 +2301,7 @@
<span class="sd"> __init__(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1, \</span>
<span class="sd"> seed=None, inputCol=None, outputCol=None, windowSize=5, maxSentenceLength=1000)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">Word2Vec</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">Word2Vec</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.Word2Vec"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">vectorSize</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">minCount</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">numPartitions</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">stepSize</span><span class="o">=</span><span class="mf">0.025</span><span class="p">,</span> <span class="n">maxIter</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="n">windowSize</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">maxSentenceLength</span><span class="o">=</span><span class="mi">1000</span><span class="p">)</span>
@@ -2463,7 +2463,7 @@
<span class="sd">"""</span>
<span class="sd"> __init__(self, k=None, inputCol=None, outputCol=None)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">PCA</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PCA</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.PCA"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">setParams</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -2602,7 +2602,7 @@
<span class="sd"> __init__(self, formula=None, featuresCol="features", labelCol="label", \</span>
<span class="sd"> forceIndexLabel=False)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">RFormula</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RFormula</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.RFormula"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">forceIndexLabel</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_kwargs</span>
@@ -2731,7 +2731,7 @@
<span class="sd"> __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None, \</span>
<span class="sd"> labelCol="label", selectorType="numTopFeatures", percentile=0.1, fpr=0.05)</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">ChiSqSelector</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">ChiSqSelector</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_java_obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_java_obj</span><span class="p">(</span><span class="s2">"org.apache.spark.ml.feature.ChiSqSelector"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">uid</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_setDefault</span><span class="p">(</span><span class="n">numTopFeatures</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span> <span class="n">selectorType</span><span class="o">=</span><span class="s2">"numTopFeatures"</span><span class="p">,</span> <span class="n">percentile</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
<span class="n">fpr</span><span class="o">=</span><span class="mf">0.05</span><span class="p">)</span>
@@ -2831,7 +2831,7 @@
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call_java</span><span class="p">(</span><span class="s2">"selectedFeatures"</span><span class="p">)</span></div>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">import</span> <span class="nn">tempfile</span>
@@ -2839,7 +2839,7 @@
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span><span class="p">,</span> <span class="n">SparkSession</span>
<span class="n">globs</span> <span class="o">=</span> <span class="nb">globals</span><span class="p">()</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
- <span class="n">features</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">feature</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">features</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">ml</span><span class="o">.</span><span class="n">feature</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">globs</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">features</span><span class="p">)</span>
<span class="c1"># The small batch size here ensures that we see multiple batches,</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/linalg.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/linalg.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/linalg.html
index 69f389f..587bb99 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/linalg.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/linalg.html
@@ -188,7 +188,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">([</span>
<span class="n">StructField</span><span class="p">(</span><span class="s2">"type"</span><span class="p">,</span> <span class="n">ByteType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s2">"size"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="kc">True</span><span class="p">),</span>
@@ -196,11 +196,11 @@
<span class="n">StructField</span><span class="p">(</span><span class="s2">"values"</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">DoubleType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span> <span class="kc">True</span><span class="p">)])</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"pyspark.ml.linalg"</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"org.apache.spark.ml.linalg.VectorUDT"</span>
<span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
@@ -235,7 +235,7 @@
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">([</span>
<span class="n">StructField</span><span class="p">(</span><span class="s2">"type"</span><span class="p">,</span> <span class="n">ByteType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s2">"numRows"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">),</span>
@@ -246,11 +246,11 @@
<span class="n">StructField</span><span class="p">(</span><span class="s2">"isTransposed"</span><span class="p">,</span> <span class="n">BooleanType</span><span class="p">(),</span> <span class="kc">False</span><span class="p">)])</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"pyspark.ml.linalg"</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"org.apache.spark.ml.linalg.MatrixUDT"</span>
<span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
@@ -491,19 +491,19 @@
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">op</span><span class="p">)(</span><span class="n">other</span><span class="p">))</span>
<span class="k">return</span> <span class="n">func</span>
- <span class="n">__neg__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__neg__"</span><span class="p">)</span>
- <span class="n">__add__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__add__"</span><span class="p">)</span>
- <span class="n">__sub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__sub__"</span><span class="p">)</span>
- <span class="n">__mul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__mul__"</span><span class="p">)</span>
+ <span class="fm">__neg__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__neg__"</span><span class="p">)</span>
+ <span class="fm">__add__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__add__"</span><span class="p">)</span>
+ <span class="fm">__sub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__sub__"</span><span class="p">)</span>
+ <span class="fm">__mul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__mul__"</span><span class="p">)</span>
<span class="n">__div__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__div__"</span><span class="p">)</span>
- <span class="n">__truediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__truediv__"</span><span class="p">)</span>
- <span class="n">__mod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__mod__"</span><span class="p">)</span>
- <span class="n">__radd__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__radd__"</span><span class="p">)</span>
- <span class="n">__rsub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rsub__"</span><span class="p">)</span>
- <span class="n">__rmul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rmul__"</span><span class="p">)</span>
+ <span class="fm">__truediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__truediv__"</span><span class="p">)</span>
+ <span class="fm">__mod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__mod__"</span><span class="p">)</span>
+ <span class="fm">__radd__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__radd__"</span><span class="p">)</span>
+ <span class="fm">__rsub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rsub__"</span><span class="p">)</span>
+ <span class="fm">__rmul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rmul__"</span><span class="p">)</span>
<span class="n">__rdiv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rdiv__"</span><span class="p">)</span>
- <span class="n">__rtruediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rtruediv__"</span><span class="p">)</span>
- <span class="n">__rmod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rmod__"</span><span class="p">)</span></div>
+ <span class="fm">__rtruediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rtruediv__"</span><span class="p">)</span>
+ <span class="fm">__rmod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s2">"__rmod__"</span><span class="p">)</span></div>
<div class="viewcode-block" id="SparseVector"><a class="viewcode-back" href="../../../pyspark.ml.html#pyspark.ml.linalg.SparseVector">[docs]</a><span class="k">class</span> <span class="nc">SparseVector</span><span class="p">(</span><span class="n">Vector</span><span class="p">):</span>
@@ -778,7 +778,7 @@
<span class="k">return</span> <span class="mf">0.</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+ <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">result</span> <span class="o">=</span> <span class="mi">31</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span>
@@ -925,7 +925,7 @@
<span class="sd"> Column-major dense matrix.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">values</span><span class="p">,</span> <span class="n">isTransposed</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
- <span class="n">Matrix</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
+ <span class="n">Matrix</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
<span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">)</span> <span class="o">==</span> <span class="n">numRows</span> <span class="o">*</span> <span class="n">numCols</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="n">values</span>
@@ -1038,7 +1038,7 @@
<span class="sd">"""Sparse Matrix stored in CSC format."""</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">colPtrs</span><span class="p">,</span> <span class="n">rowIndices</span><span class="p">,</span> <span class="n">values</span><span class="p">,</span>
<span class="n">isTransposed</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
- <span class="n">Matrix</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
+ <span class="n">Matrix</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">colPtrs</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">rowIndices</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
@@ -1208,7 +1208,7 @@
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/ml/param.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/ml/param.html b/site/docs/2.1.2/api/python/_modules/pyspark/ml/param.html
index 6ccd4df..d5f7fd5 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/ml/param.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/ml/param.html
@@ -136,7 +136,7 @@
<span class="nd">@staticmethod</span>
<span class="k">def</span> <span class="nf">_is_numeric</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
<span class="n">vtype</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">vtype</span> <span class="ow">in</span> <span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]</span> <span class="ow">or</span> <span class="n">vtype</span><span class="o">.</span><span class="n">__name__</span> <span class="o">==</span> <span class="s1">'long'</span>
+ <span class="k">return</span> <span class="n">vtype</span> <span class="ow">in</span> <span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]</span> <span class="ow">or</span> <span class="n">vtype</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'long'</span>
<span class="nd">@staticmethod</span>
<span class="k">def</span> <span class="nf">_is_integer</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
@@ -275,7 +275,7 @@
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">ABCMeta</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">Params</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">Params</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="c1">#: internal param map for user-supplied values param map</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_paramMap</span> <span class="o">=</span> <span class="p">{}</span>
@@ -292,8 +292,8 @@
<span class="sd">"""</span>
<span class="sd"> Copy all params defined on the class to current object.</span>
<span class="sd"> """</span>
- <span class="n">cls</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
- <span class="n">src_name_attrs</span> <span class="o">=</span> <span class="p">[(</span><span class="n">x</span><span class="p">,</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">x</span><span class="p">))</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">dir</span><span class="p">(</span><span class="n">cls</span><span class="p">)]</span>
+ <span class="bp">cls</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
+ <span class="n">src_name_attrs</span> <span class="o">=</span> <span class="p">[(</span><span class="n">x</span><span class="p">,</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">x</span><span class="p">))</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">dir</span><span class="p">(</span><span class="bp">cls</span><span class="p">)]</span>
<span class="n">src_params</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">nameAttr</span><span class="p">:</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">nameAttr</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">Param</span><span class="p">),</span> <span class="n">src_name_attrs</span><span class="p">))</span>
<span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">src_params</span><span class="p">:</span>
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span><span class="o">.</span><span class="n">_copy_new_parent</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[02/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/pyspark.streaming.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/pyspark.streaming.html b/site/docs/2.1.2/api/python/pyspark.streaming.html
index 8a2002d..5c1817d 100644
--- a/site/docs/2.1.2/api/python/pyspark.streaming.html
+++ b/site/docs/2.1.2/api/python/pyspark.streaming.html
@@ -766,7 +766,8 @@ DStream’s batching interval</li>
<dl class="class">
<dt id="pyspark.streaming.StreamingListener.Java">
<em class="property">class </em><code class="descname">Java</code><a class="reference internal" href="_modules/pyspark/streaming/listener.html#StreamingListener.Java"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.streaming.StreamingListener.Java" title="Permalink to this definition">¶</a></dt>
-<dd><dl class="attribute">
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">object</span></code></p>
+<dl class="attribute">
<dt id="pyspark.streaming.StreamingListener.Java.implements">
<code class="descname">implements</code><em class="property"> = ['org.apache.spark.streaming.api.java.PythonStreamingListener']</em><a class="headerlink" href="#pyspark.streaming.StreamingListener.Java.implements" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[08/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/serializers.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/serializers.html b/site/docs/2.1.2/api/python/_modules/pyspark/serializers.html
index 7d555b9..d432c3f 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/serializers.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/serializers.html
@@ -158,13 +158,13 @@
<span class="c1"># subclasses should override __eq__ as appropriate.</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">__dict__</span>
+ <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="vm">__dict__</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+ <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">()"</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span>
+ <span class="k">return</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">()"</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span>
<span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
@@ -285,7 +285,7 @@
<span class="sd"> have similar sizes.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">serializer</span><span class="p">,</span> <span class="n">batchSize</span><span class="o">=</span><span class="mi">10</span><span class="p">):</span>
- <span class="n">BatchedSerializer</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">serializer</span><span class="p">,</span> <span class="n">batchSize</span><span class="p">)</span>
+ <span class="n">BatchedSerializer</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">serializer</span><span class="p">,</span> <span class="n">batchSize</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_batched</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">iterator</span><span class="p">):</span>
<span class="n">n</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">batchSize</span>
@@ -306,7 +306,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">serializer</span><span class="p">,</span> <span class="n">bestSize</span><span class="o">=</span><span class="mi">1</span> <span class="o"><<</span> <span class="mi">16</span><span class="p">):</span>
- <span class="n">BatchedSerializer</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">serializer</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">UNKNOWN_BATCH_SIZE</span><span class="p">)</span>
+ <span class="n">BatchedSerializer</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">serializer</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">UNKNOWN_BATCH_SIZE</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bestSize</span> <span class="o">=</span> <span class="n">bestSize</span>
<span class="k">def</span> <span class="nf">dump_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">iterator</span><span class="p">,</span> <span class="n">stream</span><span class="p">):</span>
@@ -408,23 +408,23 @@
<span class="k">def</span> <span class="nf">_restore</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">fields</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="sd">""" Restore an object of namedtuple"""</span>
<span class="n">k</span> <span class="o">=</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">fields</span><span class="p">)</span>
- <span class="n">cls</span> <span class="o">=</span> <span class="n">__cls</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">cls</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">cls</span> <span class="o">=</span> <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">fields</span><span class="p">)</span>
- <span class="n">__cls</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">cls</span>
- <span class="k">return</span> <span class="n">cls</span><span class="p">(</span><span class="o">*</span><span class="n">value</span><span class="p">)</span>
+ <span class="bp">cls</span> <span class="o">=</span> <span class="n">__cls</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">cls</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">cls</span> <span class="o">=</span> <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">fields</span><span class="p">)</span>
+ <span class="n">__cls</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="bp">cls</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="o">*</span><span class="n">value</span><span class="p">)</span>
-<span class="k">def</span> <span class="nf">_hack_namedtuple</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+<span class="k">def</span> <span class="nf">_hack_namedtuple</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="sd">""" Make class generated by namedtuple picklable """</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">__name__</span>
- <span class="n">fields</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_fields</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span>
+ <span class="n">fields</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_fields</span>
<span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span><span class="n">_restore</span><span class="p">,</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">fields</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">(</span><span class="bp">self</span><span class="p">)))</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">__reduce__</span> <span class="o">=</span> <span class="n">__reduce__</span>
- <span class="n">cls</span><span class="o">.</span><span class="n">_is_namedtuple_</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="k">return</span> <span class="n">cls</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">__reduce__</span> <span class="o">=</span> <span class="n">__reduce__</span>
+ <span class="bp">cls</span><span class="o">.</span><span class="n">_is_namedtuple_</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="k">return</span> <span class="bp">cls</span>
<span class="k">def</span> <span class="nf">_hijack_namedtuple</span><span class="p">():</span>
@@ -437,8 +437,8 @@
<span class="k">global</span> <span class="n">_old_namedtuple_kwdefaults</span> <span class="c1"># or it will put in closure too</span>
<span class="k">def</span> <span class="nf">_copy_func</span><span class="p">(</span><span class="n">f</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">types</span><span class="o">.</span><span class="n">FunctionType</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">__code__</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">__globals__</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span><span class="p">,</span>
- <span class="n">f</span><span class="o">.</span><span class="n">__defaults__</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">__closure__</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">types</span><span class="o">.</span><span class="n">FunctionType</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="vm">__code__</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="vm">__globals__</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span>
+ <span class="n">f</span><span class="o">.</span><span class="vm">__defaults__</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="vm">__closure__</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_kwdefaults</span><span class="p">(</span><span class="n">f</span><span class="p">):</span>
<span class="c1"># __kwdefaults__ contains the default values of keyword-only arguments which are</span>
@@ -461,23 +461,23 @@
<span class="k">def</span> <span class="nf">namedtuple</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">_old_namedtuple_kwdefaults</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">kwargs</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
- <span class="n">cls</span> <span class="o">=</span> <span class="n">_old_namedtuple</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">_hack_namedtuple</span><span class="p">(</span><span class="n">cls</span><span class="p">)</span>
+ <span class="bp">cls</span> <span class="o">=</span> <span class="n">_old_namedtuple</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">_hack_namedtuple</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span>
<span class="c1"># replace namedtuple with new one</span>
- <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="n">__globals__</span><span class="p">[</span><span class="s2">"_old_namedtuple_kwdefaults"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_old_namedtuple_kwdefaults</span>
- <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="n">__globals__</span><span class="p">[</span><span class="s2">"_old_namedtuple"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_old_namedtuple</span>
- <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="n">__globals__</span><span class="p">[</span><span class="s2">"_hack_namedtuple"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_hack_namedtuple</span>
- <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="n">__code__</span> <span class="o">=</span> <span class="n">namedtuple</span><span class="o">.</span><span class="n">__code__</span>
+ <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="vm">__globals__</span><span class="p">[</span><span class="s2">"_old_namedtuple_kwdefaults"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_old_namedtuple_kwdefaults</span>
+ <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="vm">__globals__</span><span class="p">[</span><span class="s2">"_old_namedtuple"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_old_namedtuple</span>
+ <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="vm">__globals__</span><span class="p">[</span><span class="s2">"_hack_namedtuple"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_hack_namedtuple</span>
+ <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="vm">__code__</span> <span class="o">=</span> <span class="n">namedtuple</span><span class="o">.</span><span class="vm">__code__</span>
<span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="o">.</span><span class="n">__hijack</span> <span class="o">=</span> <span class="mi">1</span>
<span class="c1"># hack the cls already generated by namedtuple</span>
<span class="c1"># those created in other module can be pickled as normal,</span>
<span class="c1"># so only hack those in __main__ module</span>
- <span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="s2">"__main__"</span><span class="p">]</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="s2">"__main__"</span><span class="p">]</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">o</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">type</span> <span class="ow">and</span> <span class="n">o</span><span class="o">.</span><span class="n">__base__</span> <span class="ow">is</span> <span class="nb">tuple</span>
<span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="s2">"_fields"</span><span class="p">)</span>
- <span class="ow">and</span> <span class="s2">"__reduce__"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">o</span><span class="o">.</span><span class="n">__dict__</span><span class="p">):</span>
+ <span class="ow">and</span> <span class="s2">"__reduce__"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">o</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">):</span>
<span class="n">_hack_namedtuple</span><span class="p">(</span><span class="n">o</span><span class="p">)</span> <span class="c1"># hack inplace</span>
@@ -536,23 +536,23 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="n">FramedSerializer</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
+ <span class="n">FramedSerializer</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_type</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">def</span> <span class="nf">dumps</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_type</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">b</span><span class="s1">'P'</span> <span class="o">+</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+ <span class="k">return</span> <span class="sa">b</span><span class="s1">'P'</span> <span class="o">+</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">b</span><span class="s1">'M'</span> <span class="o">+</span> <span class="n">marshal</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
+ <span class="k">return</span> <span class="sa">b</span><span class="s1">'M'</span> <span class="o">+</span> <span class="n">marshal</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_type</span> <span class="o">=</span> <span class="n">b</span><span class="s1">'P'</span>
- <span class="k">return</span> <span class="n">b</span><span class="s1">'P'</span> <span class="o">+</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_type</span> <span class="o">=</span> <span class="sa">b</span><span class="s1">'P'</span>
+ <span class="k">return</span> <span class="sa">b</span><span class="s1">'P'</span> <span class="o">+</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">loads</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="n">_type</span> <span class="o">=</span> <span class="n">obj</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
- <span class="k">if</span> <span class="n">_type</span> <span class="o">==</span> <span class="n">b</span><span class="s1">'M'</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">_type</span> <span class="o">==</span> <span class="sa">b</span><span class="s1">'M'</span><span class="p">:</span>
<span class="k">return</span> <span class="n">marshal</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span>
- <span class="k">elif</span> <span class="n">_type</span> <span class="o">==</span> <span class="n">b</span><span class="s1">'P'</span><span class="p">:</span>
+ <span class="k">elif</span> <span class="n">_type</span> <span class="o">==</span> <span class="sa">b</span><span class="s1">'P'</span><span class="p">:</span>
<span class="k">return</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"invalid sevialization type: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">_type</span><span class="p">)</span>
@@ -563,7 +563,7 @@
<span class="sd"> Compress the serialized data</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">serializer</span><span class="p">):</span>
- <span class="n">FramedSerializer</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
+ <span class="n">FramedSerializer</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">serializer</span><span class="p">,</span> <span class="n">FramedSerializer</span><span class="p">),</span> <span class="s2">"serializer must be a FramedSerializer"</span>
<span class="bp">self</span><span class="o">.</span><span class="n">serializer</span> <span class="o">=</span> <span class="n">serializer</span>
@@ -639,7 +639,7 @@
<span class="n">stream</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">()</span>
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/column.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/column.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/column.html
index 0afc7c6..c825510 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/column.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/column.html
@@ -125,7 +125,7 @@
<span class="k">def</span> <span class="nf">_</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jc</span><span class="p">,</span> <span class="n">name</span><span class="p">)()</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
<span class="k">return</span> <span class="n">_</span>
@@ -134,7 +134,7 @@
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span>
<span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">functions</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jc</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
<span class="k">return</span> <span class="n">_</span>
@@ -145,7 +145,7 @@
<span class="n">jc</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_jc</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">Column</span><span class="p">)</span> <span class="k">else</span> <span class="n">_create_column_from_literal</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="n">njc</span> <span class="o">=</span> <span class="n">fn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jc</span><span class="p">,</span> <span class="n">jc</span><span class="p">)</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">reverse</span> <span class="k">else</span> <span class="n">fn</span><span class="p">(</span><span class="n">jc</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jc</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">njc</span><span class="p">)</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
<span class="k">return</span> <span class="n">_</span>
@@ -156,7 +156,7 @@
<span class="n">jc</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_jc</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">Column</span><span class="p">)</span> <span class="k">else</span> <span class="n">other</span>
<span class="n">njc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jc</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="n">jc</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">njc</span><span class="p">)</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
<span class="k">return</span> <span class="n">_</span>
@@ -167,7 +167,7 @@
<span class="n">jother</span> <span class="o">=</span> <span class="n">_create_column_from_literal</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">jother</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jc</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
- <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
+ <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span>
<span class="k">return</span> <span class="n">_</span>
@@ -194,41 +194,41 @@
<span class="bp">self</span><span class="o">.</span><span class="n">_jc</span> <span class="o">=</span> <span class="n">jc</span>
<span class="c1"># arithmetic operators</span>
- <span class="n">__neg__</span> <span class="o">=</span> <span class="n">_func_op</span><span class="p">(</span><span class="s2">"negate"</span><span class="p">)</span>
- <span class="n">__add__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"plus"</span><span class="p">)</span>
- <span class="n">__sub__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"minus"</span><span class="p">)</span>
- <span class="n">__mul__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"multiply"</span><span class="p">)</span>
+ <span class="fm">__neg__</span> <span class="o">=</span> <span class="n">_func_op</span><span class="p">(</span><span class="s2">"negate"</span><span class="p">)</span>
+ <span class="fm">__add__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"plus"</span><span class="p">)</span>
+ <span class="fm">__sub__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"minus"</span><span class="p">)</span>
+ <span class="fm">__mul__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"multiply"</span><span class="p">)</span>
<span class="n">__div__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"divide"</span><span class="p">)</span>
- <span class="n">__truediv__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"divide"</span><span class="p">)</span>
- <span class="n">__mod__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"mod"</span><span class="p">)</span>
- <span class="n">__radd__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"plus"</span><span class="p">)</span>
- <span class="n">__rsub__</span> <span class="o">=</span> <span class="n">_reverse_op</span><span class="p">(</span><span class="s2">"minus"</span><span class="p">)</span>
- <span class="n">__rmul__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"multiply"</span><span class="p">)</span>
+ <span class="fm">__truediv__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"divide"</span><span class="p">)</span>
+ <span class="fm">__mod__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"mod"</span><span class="p">)</span>
+ <span class="fm">__radd__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"plus"</span><span class="p">)</span>
+ <span class="fm">__rsub__</span> <span class="o">=</span> <span class="n">_reverse_op</span><span class="p">(</span><span class="s2">"minus"</span><span class="p">)</span>
+ <span class="fm">__rmul__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"multiply"</span><span class="p">)</span>
<span class="n">__rdiv__</span> <span class="o">=</span> <span class="n">_reverse_op</span><span class="p">(</span><span class="s2">"divide"</span><span class="p">)</span>
- <span class="n">__rtruediv__</span> <span class="o">=</span> <span class="n">_reverse_op</span><span class="p">(</span><span class="s2">"divide"</span><span class="p">)</span>
- <span class="n">__rmod__</span> <span class="o">=</span> <span class="n">_reverse_op</span><span class="p">(</span><span class="s2">"mod"</span><span class="p">)</span>
- <span class="n">__pow__</span> <span class="o">=</span> <span class="n">_bin_func_op</span><span class="p">(</span><span class="s2">"pow"</span><span class="p">)</span>
- <span class="n">__rpow__</span> <span class="o">=</span> <span class="n">_bin_func_op</span><span class="p">(</span><span class="s2">"pow"</span><span class="p">,</span> <span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="fm">__rtruediv__</span> <span class="o">=</span> <span class="n">_reverse_op</span><span class="p">(</span><span class="s2">"divide"</span><span class="p">)</span>
+ <span class="fm">__rmod__</span> <span class="o">=</span> <span class="n">_reverse_op</span><span class="p">(</span><span class="s2">"mod"</span><span class="p">)</span>
+ <span class="fm">__pow__</span> <span class="o">=</span> <span class="n">_bin_func_op</span><span class="p">(</span><span class="s2">"pow"</span><span class="p">)</span>
+ <span class="fm">__rpow__</span> <span class="o">=</span> <span class="n">_bin_func_op</span><span class="p">(</span><span class="s2">"pow"</span><span class="p">,</span> <span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="c1"># logistic operators</span>
- <span class="n">__eq__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"equalTo"</span><span class="p">)</span>
- <span class="n">__ne__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"notEqual"</span><span class="p">)</span>
- <span class="n">__lt__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"lt"</span><span class="p">)</span>
- <span class="n">__le__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"leq"</span><span class="p">)</span>
- <span class="n">__ge__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"geq"</span><span class="p">)</span>
- <span class="n">__gt__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"gt"</span><span class="p">)</span>
+ <span class="fm">__eq__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"equalTo"</span><span class="p">)</span>
+ <span class="fm">__ne__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"notEqual"</span><span class="p">)</span>
+ <span class="fm">__lt__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"lt"</span><span class="p">)</span>
+ <span class="fm">__le__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"leq"</span><span class="p">)</span>
+ <span class="fm">__ge__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"geq"</span><span class="p">)</span>
+ <span class="fm">__gt__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"gt"</span><span class="p">)</span>
<span class="c1"># `and`, `or`, `not` cannot be overloaded in Python,</span>
<span class="c1"># so use bitwise operators as boolean operators</span>
- <span class="n">__and__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s1">'and'</span><span class="p">)</span>
- <span class="n">__or__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s1">'or'</span><span class="p">)</span>
- <span class="n">__invert__</span> <span class="o">=</span> <span class="n">_func_op</span><span class="p">(</span><span class="s1">'not'</span><span class="p">)</span>
- <span class="n">__rand__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"and"</span><span class="p">)</span>
- <span class="n">__ror__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"or"</span><span class="p">)</span>
+ <span class="fm">__and__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s1">'and'</span><span class="p">)</span>
+ <span class="fm">__or__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s1">'or'</span><span class="p">)</span>
+ <span class="fm">__invert__</span> <span class="o">=</span> <span class="n">_func_op</span><span class="p">(</span><span class="s1">'not'</span><span class="p">)</span>
+ <span class="fm">__rand__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"and"</span><span class="p">)</span>
+ <span class="fm">__ror__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"or"</span><span class="p">)</span>
<span class="c1"># container operators</span>
- <span class="n">__contains__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"contains"</span><span class="p">)</span>
- <span class="n">__getitem__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"apply"</span><span class="p">)</span>
+ <span class="fm">__contains__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"contains"</span><span class="p">)</span>
+ <span class="fm">__getitem__</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"apply"</span><span class="p">)</span>
<span class="c1"># bitwise operators</span>
<span class="n">bitwiseOR</span> <span class="o">=</span> <span class="n">_bin_op</span><span class="p">(</span><span class="s2">"bitwiseOR"</span><span class="p">)</span>
@@ -473,7 +473,7 @@
<span class="k">def</span> <span class="nf">__nonzero__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Cannot convert column into bool: please use '&' for 'and', '|' for 'or', "</span>
<span class="s2">"'~' for 'not' when building DataFrame boolean expressions."</span><span class="p">)</span>
- <span class="n">__bool__</span> <span class="o">=</span> <span class="n">__nonzero__</span>
+ <span class="fm">__bool__</span> <span class="o">=</span> <span class="n">__nonzero__</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s1">'Column<</span><span class="si">%s</span><span class="s1">>'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jc</span><span class="o">.</span><span class="n">toString</span><span class="p">()</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf8'</span><span class="p">)</span></div>
@@ -483,7 +483,7 @@
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">SparkSession</span>
<span class="kn">import</span> <span class="nn">pyspark.sql.column</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">column</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">column</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\
<span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[4]"</span><span class="p">)</span>\
<span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"sql.column tests"</span><span class="p">)</span>\
@@ -502,7 +502,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/context.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/context.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/context.html
index 545f759..f01a437 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/context.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/context.html
@@ -142,17 +142,17 @@
<span class="nd">@classmethod</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.6</span><span class="p">)</span>
-<div class="viewcode-block" id="SQLContext.getOrCreate"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.SQLContext.getOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getOrCreate</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">):</span>
+<div class="viewcode-block" id="SQLContext.getOrCreate"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.SQLContext.getOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getOrCreate</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sc</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Get the existing SQLContext or create a new one with given SparkContext.</span>
<span class="sd"> :param sc: SparkContext</span>
<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">cls</span><span class="o">.</span><span class="n">_instantiatedContext</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instantiatedContext</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">jsqlContext</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">SQLContext</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">_jsc</span><span class="o">.</span><span class="n">sc</span><span class="p">())</span>
<span class="n">sparkSession</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">jsqlContext</span><span class="o">.</span><span class="n">sparkSession</span><span class="p">())</span>
- <span class="n">cls</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">sparkSession</span><span class="p">,</span> <span class="n">jsqlContext</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_instantiatedContext</span></div>
+ <span class="bp">cls</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">sparkSession</span><span class="p">,</span> <span class="n">jsqlContext</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instantiatedContext</span></div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.6</span><span class="p">)</span>
<div class="viewcode-block" id="SQLContext.newSession"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.SQLContext.newSession">[docs]</a> <span class="k">def</span> <span class="nf">newSession</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -161,7 +161,7 @@
<span class="sd"> registered temporary views and UDFs, but shared SparkContext and</span>
<span class="sd"> table cache.</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sparkSession</span><span class="o">.</span><span class="n">newSession</span><span class="p">())</span></div>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sparkSession</span><span class="o">.</span><span class="n">newSession</span><span class="p">())</span></div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="SQLContext.setConf"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.SQLContext.setConf">[docs]</a> <span class="k">def</span> <span class="nf">setConf</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
@@ -562,10 +562,10 @@
<span class="n">sparkSession</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span><span class="o">.</span><span class="n">enableHiveSupport</span><span class="p">()</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">sparkSession</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="p">(</span><span class="n">sparkContext</span><span class="p">,</span> <span class="n">jhiveContext</span><span class="o">.</span><span class="n">sparkSession</span><span class="p">())</span>
- <span class="n">SQLContext</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sparkContext</span><span class="p">,</span> <span class="n">sparkSession</span><span class="p">,</span> <span class="n">jhiveContext</span><span class="p">)</span>
+ <span class="n">SQLContext</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sparkContext</span><span class="p">,</span> <span class="n">sparkSession</span><span class="p">,</span> <span class="n">jhiveContext</span><span class="p">)</span>
<span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">_createForTesting</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">sparkContext</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_createForTesting</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">sparkContext</span><span class="p">):</span>
<span class="sd">"""(Internal use only) Create a new HiveContext for testing.</span>
<span class="sd"> All test code that touches HiveContext *must* go through this method. Otherwise,</span>
@@ -574,7 +574,7 @@
<span class="sd"> """</span>
<span class="n">jsc</span> <span class="o">=</span> <span class="n">sparkContext</span><span class="o">.</span><span class="n">_jsc</span><span class="o">.</span><span class="n">sc</span><span class="p">()</span>
<span class="n">jtestHive</span> <span class="o">=</span> <span class="n">sparkContext</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">hive</span><span class="o">.</span><span class="n">test</span><span class="o">.</span><span class="n">TestHiveContext</span><span class="p">(</span><span class="n">jsc</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">cls</span><span class="p">(</span><span class="n">sparkContext</span><span class="p">,</span> <span class="n">jtestHive</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">sparkContext</span><span class="p">,</span> <span class="n">jtestHive</span><span class="p">)</span>
<div class="viewcode-block" id="HiveContext.refreshTable"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.HiveContext.refreshTable">[docs]</a> <span class="k">def</span> <span class="nf">refreshTable</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tableName</span><span class="p">):</span>
<span class="sd">"""Invalidate and refresh all the cached the metadata of the given</span>
@@ -595,7 +595,7 @@
<div class="viewcode-block" id="UDFRegistration.register"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.UDFRegistration.register">[docs]</a> <span class="k">def</span> <span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">returnType</span><span class="o">=</span><span class="n">StringType</span><span class="p">()):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqlContext</span><span class="o">.</span><span class="n">registerFunction</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">returnType</span><span class="p">)</span></div>
- <span class="n">register</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">SQLContext</span><span class="o">.</span><span class="n">registerFunction</span><span class="o">.</span><span class="n">__doc__</span></div>
+ <span class="n">register</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">SQLContext</span><span class="o">.</span><span class="n">registerFunction</span><span class="o">.</span><span class="vm">__doc__</span></div>
<span class="k">def</span> <span class="nf">_test</span><span class="p">():</span>
@@ -608,7 +608,7 @@
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"SPARK_HOME"</span><span class="p">])</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">context</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">context</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s1">'local[4]'</span><span class="p">,</span> <span class="s1">'PythonTest'</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'tempfile'</span><span class="p">]</span> <span class="o">=</span> <span class="n">tempfile</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'os'</span><span class="p">]</span> <span class="o">=</span> <span class="n">os</span>
@@ -637,7 +637,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/dataframe.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/dataframe.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/dataframe.html
index 6abb745..51a796a 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/dataframe.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/dataframe.html
@@ -817,7 +817,7 @@
<span class="n">on</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">on</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">on</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">Column</span><span class="p">),</span> <span class="s2">"on should be Column or list of Column"</span>
- <span class="n">on</span> <span class="o">=</span> <span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">__and__</span><span class="p">(</span><span class="n">y</span><span class="p">),</span> <span class="n">on</span><span class="p">)</span>
+ <span class="n">on</span> <span class="o">=</span> <span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="fm">__and__</span><span class="p">(</span><span class="n">y</span><span class="p">),</span> <span class="n">on</span><span class="p">)</span>
<span class="n">on</span> <span class="o">=</span> <span class="n">on</span><span class="o">.</span><span class="n">_jc</span>
<span class="k">if</span> <span class="n">on</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">how</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
@@ -1018,7 +1018,7 @@
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
- <span class="s2">"'</span><span class="si">%s</span><span class="s2">' object has no attribute '</span><span class="si">%s</span><span class="s2">'"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">,</span> <span class="n">name</span><span class="p">))</span>
+ <span class="s2">"'</span><span class="si">%s</span><span class="s2">' object has no attribute '</span><span class="si">%s</span><span class="s2">'"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="n">name</span><span class="p">))</span>
<span class="n">jc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
@@ -1671,17 +1671,17 @@
<div class="viewcode-block" id="DataFrameNaFunctions.drop"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameNaFunctions.drop">[docs]</a> <span class="k">def</span> <span class="nf">drop</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s1">'any'</span><span class="p">,</span> <span class="n">thresh</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">dropna</span><span class="p">(</span><span class="n">how</span><span class="o">=</span><span class="n">how</span><span class="p">,</span> <span class="n">thresh</span><span class="o">=</span><span class="n">thresh</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="n">subset</span><span class="p">)</span></div>
- <span class="n">drop</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">dropna</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">drop</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">dropna</span><span class="o">.</span><span class="vm">__doc__</span>
<div class="viewcode-block" id="DataFrameNaFunctions.fill"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameNaFunctions.fill">[docs]</a> <span class="k">def</span> <span class="nf">fill</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="n">subset</span><span class="p">)</span></div>
- <span class="n">fill</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">fillna</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">fill</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">fillna</span><span class="o">.</span><span class="vm">__doc__</span>
<div class="viewcode-block" id="DataFrameNaFunctions.replace"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameNaFunctions.replace">[docs]</a> <span class="k">def</span> <span class="nf">replace</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">to_replace</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="p">)</span></div>
- <span class="n">replace</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">replace</span><span class="o">.</span><span class="n">__doc__</span></div>
+ <span class="n">replace</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">replace</span><span class="o">.</span><span class="vm">__doc__</span></div>
<div class="viewcode-block" id="DataFrameStatFunctions"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions">[docs]</a><span class="k">class</span> <span class="nc">DataFrameStatFunctions</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
@@ -1696,32 +1696,32 @@
<div class="viewcode-block" id="DataFrameStatFunctions.approxQuantile"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.approxQuantile">[docs]</a> <span class="k">def</span> <span class="nf">approxQuantile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">probabilities</span><span class="p">,</span> <span class="n">relativeError</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">approxQuantile</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">probabilities</span><span class="p">,</span> <span class="n">relativeError</span><span class="p">)</span></div>
- <span class="n">approxQuantile</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">approxQuantile</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">approxQuantile</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">approxQuantile</span><span class="o">.</span><span class="vm">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.corr"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.corr">[docs]</a> <span class="k">def</span> <span class="nf">corr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">,</span> <span class="n">method</span><span class="p">)</span></div>
- <span class="n">corr</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">corr</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">corr</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">corr</span><span class="o">.</span><span class="vm">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.cov"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.cov">[docs]</a> <span class="k">def</span> <span class="nf">cov</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">cov</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">)</span></div>
- <span class="n">cov</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">cov</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">cov</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">cov</span><span class="o">.</span><span class="vm">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.crosstab"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.crosstab">[docs]</a> <span class="k">def</span> <span class="nf">crosstab</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">crosstab</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">)</span></div>
- <span class="n">crosstab</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">crosstab</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">crosstab</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">crosstab</span><span class="o">.</span><span class="vm">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.freqItems"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.freqItems">[docs]</a> <span class="k">def</span> <span class="nf">freqItems</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cols</span><span class="p">,</span> <span class="n">support</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">freqItems</span><span class="p">(</span><span class="n">cols</span><span class="p">,</span> <span class="n">support</span><span class="p">)</span></div>
- <span class="n">freqItems</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">freqItems</span><span class="o">.</span><span class="n">__doc__</span>
+ <span class="n">freqItems</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">freqItems</span><span class="o">.</span><span class="vm">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.sampleBy"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.sampleBy">[docs]</a> <span class="k">def</span> <span class="nf">sampleBy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">fractions</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">sampleBy</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">fractions</span><span class="p">,</span> <span class="n">seed</span><span class="p">)</span></div>
- <span class="n">sampleBy</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">sampleBy</span><span class="o">.</span><span class="n">__doc__</span></div>
+ <span class="n">sampleBy</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">sampleBy</span><span class="o">.</span><span class="vm">__doc__</span></div>
<span class="k">def</span> <span class="nf">_test</span><span class="p">():</span>
@@ -1730,7 +1730,7 @@
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span><span class="p">,</span> <span class="n">SQLContext</span><span class="p">,</span> <span class="n">SparkSession</span>
<span class="kn">import</span> <span class="nn">pyspark.sql.dataframe</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.functions</span> <span class="k">import</span> <span class="n">from_unixtime</span>
- <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">dataframe</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">dataframe</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s1">'local[4]'</span><span class="p">,</span> <span class="s1">'PythonTest'</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'sc'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span>
<span class="n">globs</span><span class="p">[</span><span class="s1">'sqlContext'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SQLContext</span><span class="p">(</span><span class="n">sc</span><span class="p">)</span>
@@ -1756,7 +1756,7 @@
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
+<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[06/14] spark-website git commit: Fix signature description broken in
PySpark API documentation in 2.1.2
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/streaming/dstream.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/streaming/dstream.html b/site/docs/2.1.2/api/python/_modules/pyspark/streaming/dstream.html
index 48eb5cc..853b230 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/streaming/dstream.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/streaming/dstream.html
@@ -202,7 +202,7 @@
<span class="sd">"""</span>
<span class="sd"> Apply a function to each RDD in this DStream.</span>
<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="n">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">old_func</span> <span class="o">=</span> <span class="n">func</span>
<span class="n">func</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">t</span><span class="p">,</span> <span class="n">rdd</span><span class="p">:</span> <span class="n">old_func</span><span class="p">(</span><span class="n">rdd</span><span class="p">)</span>
<span class="n">jfunc</span> <span class="o">=</span> <span class="n">TransformFunction</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jrdd_deserializer</span><span class="p">)</span>
@@ -338,10 +338,10 @@
<span class="sd"> `func` can have one argument of `rdd`, or have two arguments of</span>
<span class="sd"> (`time`, `rdd`)</span>
<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="n">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">oldfunc</span> <span class="o">=</span> <span class="n">func</span>
<span class="n">func</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">t</span><span class="p">,</span> <span class="n">rdd</span><span class="p">:</span> <span class="n">oldfunc</span><span class="p">(</span><span class="n">rdd</span><span class="p">)</span>
- <span class="k">assert</span> <span class="n">func</span><span class="o">.</span><span class="n">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">2</span><span class="p">,</span> <span class="s2">"func should take one or two arguments"</span>
+ <span class="k">assert</span> <span class="n">func</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">2</span><span class="p">,</span> <span class="s2">"func should take one or two arguments"</span>
<span class="k">return</span> <span class="n">TransformedDStream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">func</span><span class="p">)</span></div>
<div class="viewcode-block" id="DStream.transformWith"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.DStream.transformWith">[docs]</a> <span class="k">def</span> <span class="nf">transformWith</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">other</span><span class="p">,</span> <span class="n">keepSerializer</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
@@ -352,10 +352,10 @@
<span class="sd"> `func` can have two arguments of (`rdd_a`, `rdd_b`) or have three</span>
<span class="sd"> arguments of (`time`, `rdd_a`, `rdd_b`)</span>
<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="n">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
<span class="n">oldfunc</span> <span class="o">=</span> <span class="n">func</span>
<span class="n">func</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">t</span><span class="p">,</span> <span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">:</span> <span class="n">oldfunc</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span>
- <span class="k">assert</span> <span class="n">func</span><span class="o">.</span><span class="n">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">3</span><span class="p">,</span> <span class="s2">"func should take two or three arguments"</span>
+ <span class="k">assert</span> <span class="n">func</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">3</span><span class="p">,</span> <span class="s2">"func should take two or three arguments"</span>
<span class="n">jfunc</span> <span class="o">=</span> <span class="n">TransformFunction</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jrdd_deserializer</span><span class="p">,</span> <span class="n">other</span><span class="o">.</span><span class="n">_jrdd_deserializer</span><span class="p">)</span>
<span class="n">dstream</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">PythonTransformed2DStream</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdstream</span><span class="o">.</span><span class="n">dstream</span><span class="p">(),</span>
<span class="n">other</span><span class="o">.</span><span class="n">_jdstream</span><span class="o">.</span><span class="n">dstream</span><span class="p">(),</span> <span class="n">jfunc</span><span class="p">)</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/streaming/kafka.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/streaming/kafka.html b/site/docs/2.1.2/api/python/_modules/pyspark/streaming/kafka.html
index 52f3960..c7e8fbf 100644
--- a/site/docs/2.1.2/api/python/_modules/pyspark/streaming/kafka.html
+++ b/site/docs/2.1.2/api/python/_modules/pyspark/streaming/kafka.html
@@ -288,7 +288,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">untilOffset</span> <span class="o">=</span> <span class="n">untilOffset</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">topic</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">topic</span>
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">partition</span>
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">fromOffset</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">fromOffset</span>
@@ -297,7 +297,7 @@
<span class="k">return</span> <span class="kc">False</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+ <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">"OffsetRange(topic: </span><span class="si">%s</span><span class="s2">, partition: </span><span class="si">%d</span><span class="s2">, range: [</span><span class="si">%d</span><span class="s2"> -> </span><span class="si">%d</span><span class="s2">]"</span> \
@@ -326,17 +326,17 @@
<span class="k">return</span> <span class="n">helper</span><span class="o">.</span><span class="n">createTopicAndPartition</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_topic</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_partition</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_topic</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">_topic</span>
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_partition</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">_partition</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
- <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+ <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_topic</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_partition</span><span class="p">)</span><span class="o">.</span><span class="n">__hash__</span><span class="p">()</span></div>
+ <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_topic</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_partition</span><span class="p">)</span><span class="o">.</span><span class="fm">__hash__</span><span class="p">()</span></div>
<div class="viewcode-block" id="Broker"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.kafka.Broker">[docs]</a><span class="k">class</span> <span class="nc">Broker</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
@@ -363,7 +363,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jrdd</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">jrdd_deserializer</span><span class="p">):</span>
- <span class="n">RDD</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jrdd</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">jrdd_deserializer</span><span class="p">)</span>
+ <span class="n">RDD</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jrdd</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">jrdd_deserializer</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">offsetRanges</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -383,13 +383,13 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jdstream</span><span class="p">,</span> <span class="n">ssc</span><span class="p">,</span> <span class="n">jrdd_deserializer</span><span class="p">):</span>
- <span class="n">DStream</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jdstream</span><span class="p">,</span> <span class="n">ssc</span><span class="p">,</span> <span class="n">jrdd_deserializer</span><span class="p">)</span>
+ <span class="n">DStream</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jdstream</span><span class="p">,</span> <span class="n">ssc</span><span class="p">,</span> <span class="n">jrdd_deserializer</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">foreachRDD</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">func</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Apply a function to each RDD in this DStream.</span>
<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="n">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">old_func</span> <span class="o">=</span> <span class="n">func</span>
<span class="n">func</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">r</span><span class="p">,</span> <span class="n">rdd</span><span class="p">:</span> <span class="n">old_func</span><span class="p">(</span><span class="n">rdd</span><span class="p">)</span>
<span class="n">jfunc</span> <span class="o">=</span> <span class="n">TransformFunction</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jrdd_deserializer</span><span class="p">)</span> \
@@ -405,10 +405,10 @@
<span class="sd"> `func` can have one argument of `rdd`, or have two arguments of</span>
<span class="sd"> (`time`, `rdd`)</span>
<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="n">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">func</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">oldfunc</span> <span class="o">=</span> <span class="n">func</span>
<span class="n">func</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">t</span><span class="p">,</span> <span class="n">rdd</span><span class="p">:</span> <span class="n">oldfunc</span><span class="p">(</span><span class="n">rdd</span><span class="p">)</span>
- <span class="k">assert</span> <span class="n">func</span><span class="o">.</span><span class="n">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">2</span><span class="p">,</span> <span class="s2">"func should take one or two arguments"</span>
+ <span class="k">assert</span> <span class="n">func</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_argcount</span> <span class="o">==</span> <span class="mi">2</span><span class="p">,</span> <span class="s2">"func should take one or two arguments"</span>
<span class="k">return</span> <span class="n">KafkaTransformedDStream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">func</span><span class="p">)</span>
@@ -419,7 +419,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prev</span><span class="p">,</span> <span class="n">func</span><span class="p">):</span>
- <span class="n">TransformedDStream</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prev</span><span class="p">,</span> <span class="n">func</span><span class="p">)</span>
+ <span class="n">TransformedDStream</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prev</span><span class="p">,</span> <span class="n">func</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="k">def</span> <span class="nf">_jdstream</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -462,7 +462,7 @@
<span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">topic</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">offset</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__str__</span><span class="p">()</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span><span class="n">KafkaMessageAndMetadata</span><span class="p">,</span>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_static/pygments.css
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/_static/pygments.css b/site/docs/2.1.2/api/python/_static/pygments.css
index 8213e90..20c4814 100644
--- a/site/docs/2.1.2/api/python/_static/pygments.css
+++ b/site/docs/2.1.2/api/python/_static/pygments.css
@@ -47,8 +47,10 @@
.highlight .mh { color: #208050 } /* Literal.Number.Hex */
.highlight .mi { color: #208050 } /* Literal.Number.Integer */
.highlight .mo { color: #208050 } /* Literal.Number.Oct */
+.highlight .sa { color: #4070a0 } /* Literal.String.Affix */
.highlight .sb { color: #4070a0 } /* Literal.String.Backtick */
.highlight .sc { color: #4070a0 } /* Literal.String.Char */
+.highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */
.highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */
.highlight .s2 { color: #4070a0 } /* Literal.String.Double */
.highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */
@@ -59,7 +61,9 @@
.highlight .s1 { color: #4070a0 } /* Literal.String.Single */
.highlight .ss { color: #517918 } /* Literal.String.Symbol */
.highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */
+.highlight .fm { color: #06287e } /* Name.Function.Magic */
.highlight .vc { color: #bb60d5 } /* Name.Variable.Class */
.highlight .vg { color: #bb60d5 } /* Name.Variable.Global */
.highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */
+.highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */
.highlight .il { color: #208050 } /* Literal.Number.Integer.Long */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/pyspark.html
----------------------------------------------------------------------
diff --git a/site/docs/2.1.2/api/python/pyspark.html b/site/docs/2.1.2/api/python/pyspark.html
index cce60da..2f32994 100644
--- a/site/docs/2.1.2/api/python/pyspark.html
+++ b/site/docs/2.1.2/api/python/pyspark.html
@@ -69,47 +69,40 @@
<p>PySpark is the Python API for Spark.</p>
<p>Public classes:</p>
<blockquote>
-<div><ul>
+<div><ul class="simple">
<li><dl class="first docutils">
<dt><a class="reference internal" href="#pyspark.SparkContext" title="pyspark.SparkContext"><code class="xref py py-class docutils literal"><span class="pre">SparkContext</span></code></a>:</dt>
-<dd><p class="first last">Main entry point for Spark functionality.</p>
-</dd>
+<dd>Main entry point for Spark functionality.</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><a class="reference internal" href="#pyspark.RDD" title="pyspark.RDD"><code class="xref py py-class docutils literal"><span class="pre">RDD</span></code></a>:</dt>
-<dd><p class="first last">A Resilient Distributed Dataset (RDD), the basic abstraction in Spark.</p>
-</dd>
+<dd>A Resilient Distributed Dataset (RDD), the basic abstraction in Spark.</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><a class="reference internal" href="#pyspark.Broadcast" title="pyspark.Broadcast"><code class="xref py py-class docutils literal"><span class="pre">Broadcast</span></code></a>:</dt>
-<dd><p class="first last">A broadcast variable that gets reused across tasks.</p>
-</dd>
+<dd>A broadcast variable that gets reused across tasks.</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><a class="reference internal" href="#pyspark.Accumulator" title="pyspark.Accumulator"><code class="xref py py-class docutils literal"><span class="pre">Accumulator</span></code></a>:</dt>
-<dd><p class="first last">An “add-only” shared variable that tasks can only add values to.</p>
-</dd>
+<dd>An “add-only” shared variable that tasks can only add values to.</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><a class="reference internal" href="#pyspark.SparkConf" title="pyspark.SparkConf"><code class="xref py py-class docutils literal"><span class="pre">SparkConf</span></code></a>:</dt>
-<dd><p class="first last">For configuring Spark.</p>
-</dd>
+<dd>For configuring Spark.</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><a class="reference internal" href="#pyspark.SparkFiles" title="pyspark.SparkFiles"><code class="xref py py-class docutils literal"><span class="pre">SparkFiles</span></code></a>:</dt>
-<dd><p class="first last">Access files shipped with jobs.</p>
-</dd>
+<dd>Access files shipped with jobs.</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><a class="reference internal" href="#pyspark.StorageLevel" title="pyspark.StorageLevel"><code class="xref py py-class docutils literal"><span class="pre">StorageLevel</span></code></a>:</dt>
-<dd><p class="first last">Finer-grained cache persistence levels.</p>
-</dd>
+<dd>Finer-grained cache persistence levels.</dd>
</dl>
</li>
</ul>
@@ -277,7 +270,7 @@ Its format depends on the scheduler implementation.</p>
<li>in case of YARN something like ‘application_1433865536131_34483’</li>
</ul>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sc</span><span class="o">.</span><span class="n">applicationId</span>
-<span class="go">u'local-...'</span>
+<span class="go">'local-...'</span>
</pre></div>
</div>
</dd></dl>
@@ -756,7 +749,7 @@ Spark 1.2)</p>
<span class="gp">... </span> <span class="n">_</span> <span class="o">=</span> <span class="n">testFile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">"Hello world!"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">textFile</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">textFile</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">textFile</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[u'Hello world!']</span>
+<span class="go">['Hello world!']</span>
</pre></div>
</div>
</dd></dl>
@@ -779,10 +772,10 @@ serializer:</p>
<span class="gp">... </span> <span class="n">_</span> <span class="o">=</span> <span class="n">testFile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">"Hello"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">textFile</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">textFile</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">textFile</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[u'Hello']</span>
+<span class="go">['Hello']</span>
<span class="gp">>>> </span><span class="n">parallelized</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="s2">"World!"</span><span class="p">])</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">union</span><span class="p">([</span><span class="n">textFile</span><span class="p">,</span> <span class="n">parallelized</span><span class="p">])</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[u'Hello', 'World!']</span>
+<span class="go">['Hello', 'World!']</span>
</pre></div>
</div>
</dd></dl>
@@ -832,7 +825,7 @@ fully in memory.</p>
<span class="gp">... </span> <span class="n">_</span> <span class="o">=</span> <span class="n">file2</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">"2"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">textFiles</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">wholeTextFiles</span><span class="p">(</span><span class="n">dirPath</span><span class="p">)</span>
<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">textFiles</span><span class="o">.</span><span class="n">collect</span><span class="p">())</span>
-<span class="go">[(u'.../1.txt', u'1'), (u'.../2.txt', u'2')]</span>
+<span class="go">[('.../1.txt', '1'), ('.../2.txt', '2')]</span>
</pre></div>
</div>
</dd></dl>
@@ -1684,7 +1677,7 @@ If no storage level is specified defaults to (<code class="xref py py-class docu
<code class="descname">pipe</code><span class="sig-paren">(</span><em>command</em>, <em>env=None</em>, <em>checkCode=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/rdd.html#RDD.pipe"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.RDD.pipe" title="Permalink to this definition">¶</a></dt>
<dd><p>Return an RDD created by piping elements to a forked external process.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="s1">'1'</span><span class="p">,</span> <span class="s1">'2'</span><span class="p">,</span> <span class="s1">''</span><span class="p">,</span> <span class="s1">'3'</span><span class="p">])</span><span class="o">.</span><span class="n">pipe</span><span class="p">(</span><span class="s1">'cat'</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-<span class="go">[u'1', u'2', u'', u'3']</span>
+<span class="go">['1', '2', '', '3']</span>
</pre></div>
</div>
<table class="docutils field-list" frame="void" rules="none">
@@ -1799,7 +1792,7 @@ using <cite>coalesce</cite>, which can avoid performing a shuffle.</p>
<dl class="method">
<dt id="pyspark.RDD.repartitionAndSortWithinPartitions">
-<code class="descname">repartitionAndSortWithinPartitions</code><span class="sig-paren">(</span><em>numPartitions=None</em>, <em>partitionFunc=<function portable_hash></em>, <em>ascending=True</em>, <em>keyfunc=<function <lambda>></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/rdd.html#RDD.repartitionAndSortWithinPartitions"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.RDD.repartitionAndSortWithinPartitions" title="Permalink to this definition">¶</a></dt>
+<code class="descname">repartitionAndSortWithinPartitions</code><span class="sig-paren">(</span><em>numPartitions=None</em>, <em>partitionFunc=<function portable_hash></em>, <em>ascending=True</em>, <em>keyfunc=<function RDD.<lambda>></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/rdd.html#RDD.repartitionAndSortWithinPartitions"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.RDD.repartitionAndSortWithinPartitions" title="Permalink to this definition">¶</a></dt>
<dd><p>Repartition the RDD according to the given partitioner and, within each resulting partition,
sort records by their keys.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">rdd</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">6</span><span class="p">),</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">)])</span>
@@ -2088,8 +2081,8 @@ RDD’s key and value types. The mechanism is as follows:</p>
<span class="gp">>>> </span><span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">])</span><span class="o">.</span><span class="n">saveAsTextFile</span><span class="p">(</span><span class="n">tempFile3</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">codec</span><span class="p">)</span>
<span class="gp">>>> </span><span class="kn">from</span> <span class="nn">fileinput</span> <span class="k">import</span> <span class="nb">input</span><span class="p">,</span> <span class="n">hook_compressed</span>
<span class="gp">>>> </span><span class="n">result</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="nb">input</span><span class="p">(</span><span class="n">glob</span><span class="p">(</span><span class="n">tempFile3</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s2">"/part*.gz"</span><span class="p">),</span> <span class="n">openhook</span><span class="o">=</span><span class="n">hook_compressed</span><span class="p">))</span>
-<span class="gp">>>> </span><span class="n">b</span><span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
-<span class="go">u'bar\nfoo\n'</span>
+<span class="gp">>>> </span><span class="sa">b</span><span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+<span class="go">'bar\nfoo\n'</span>
</pre></div>
</div>
</dd></dl>
@@ -2100,7 +2093,7 @@ RDD’s key and value types. The mechanism is as follows:</p>
<dd><p>Assign a name to this RDD.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">rdd1</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">rdd1</span><span class="o">.</span><span class="n">setName</span><span class="p">(</span><span class="s1">'RDD1'</span><span class="p">)</span><span class="o">.</span><span class="n">name</span><span class="p">()</span>
-<span class="go">u'RDD1'</span>
+<span class="go">'RDD1'</span>
</pre></div>
</div>
</dd></dl>
@@ -2120,7 +2113,7 @@ RDD’s key and value types. The mechanism is as follows:</p>
<dl class="method">
<dt id="pyspark.RDD.sortByKey">
-<code class="descname">sortByKey</code><span class="sig-paren">(</span><em>ascending=True</em>, <em>numPartitions=None</em>, <em>keyfunc=<function <lambda>></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/rdd.html#RDD.sortByKey"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.RDD.sortByKey" title="Permalink to this definition">¶</a></dt>
+<code class="descname">sortByKey</code><span class="sig-paren">(</span><em>ascending=True</em>, <em>numPartitions=None</em>, <em>keyfunc=<function RDD.<lambda>></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/rdd.html#RDD.sortByKey"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.RDD.sortByKey" title="Permalink to this definition">¶</a></dt>
<dd><p>Sorts this RDD, which is assumed to consist of (key, value) pairs.
# noqa</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">tmp</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">'a'</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="p">(</span><span class="s1">'b'</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="p">(</span><span class="s1">'1'</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span> <span class="p">(</span><span class="s1">'d'</span><span class="p">,</span> <span class="mi">4</span><span class="p">),</span> <span class="p">(</span><span class="s1">'2'</span><span class="p">,</span> <span class="mi">5</span><span class="p">)]</span>
@@ -2659,7 +2652,7 @@ not be as fast as more specialized serializers.</p>
<dl class="method">
<dt id="pyspark.PickleSerializer.loads">
-<code class="descname">loads</code><span class="sig-paren">(</span><em>obj</em>, <em>encoding=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/serializers.html#PickleSerializer.loads"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.PickleSerializer.loads" title="Permalink to this definition">¶</a></dt>
+<code class="descname">loads</code><span class="sig-paren">(</span><em>obj</em>, <em>encoding='bytes'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/pyspark/serializers.html#PickleSerializer.loads"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#pyspark.PickleSerializer.loads" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>
</dd></dl>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org