[jira] [Updated] (SPARK-15404) pyspark sql bug ,here is the testcase

["郭同 (JIRA)" <ji...@apache.org>]

     [ https://issues.apache.org/jira/browse/SPARK-15404?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

郭同 updated SPARK-15404:
-----------------------
    Description: 
from __future__ import print_function

import os
import sys

from pyspark import SparkContext
from pyspark.sql import SQLContext
from pyspark.sql.types import Row, StructField, StructType, StringType, IntegerType


if __name__ == "__main__":
    sc = SparkContext(appName="PythonSQL")
    sqlContext = SQLContext(sc)
    schema = StructType([StructField("person_name", StringType(), False),
                     StructField("person_age", IntegerType(), False)])

    some_rdd = sc.parallelize([Row(person_name="John", person_age=19),
                               Row(person_name="Smith", person_age=23),
                               Row(person_name="Sarah", person_age=18)])

    some_df = sqlContext.createDataFrame(some_rdd, schema)
    some_df.printSchema()

    some_df.registerAsTable("people")

    teenagers = sqlContext.sql("SELECT * FROM people ")

    for each in teenagers.collect():
        print(each)

    sc.stop()


  was:
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function

import os
import sys

from pyspark import SparkContext
from pyspark.sql import SQLContext
from pyspark.sql.types import Row, StructField, StructType, StringType, IntegerType


if __name__ == "__main__":
    sc = SparkContext(appName="PythonSQL")
    sqlContext = SQLContext(sc)
    schema = StructType([StructField("person_name", StringType(), False),
                     StructField("person_age", IntegerType(), False)])
    # RDD is created from a list of rows
    some_rdd = sc.parallelize([Row(person_name="John", person_age=19),
                               Row(person_name="Smith", person_age=23),
                               Row(person_name="Sarah", person_age=18)])
    # Infer schema from the first row, create a DataFrame and print the schema
    some_df = sqlContext.createDataFrame(some_rdd, schema)
    some_df.printSchema()

    # Another RDD is created from a list of tuples
    another_rdd = sc.parallelize([("John", 19), ("Smith", 23), ("Sarah", 18)])
    # Schema with two fields - person_name and person_age

    # Create a DataFrame by applying the schema to the RDD and print the schema
    another_df = sqlContext.createDataFrame(another_rdd, schema)
    another_df.printSchema()

  #  print(some_df.filter(some_df.age > 20).collect())

    # root
    #  |-- age: integer (nullable = true)
    #  |-- name: string (nullable = true)

    # A JSON dataset is pointed to by path.
    # The path can be either a single text file or a directory storing text files.
    if len(sys.argv) < 2:
        path = "file://" + \
            os.path.join(os.environ['SPARK_HOME'], "examples/src/main/resources/people.json")
    else:
        path = sys.argv[1]
    # Create a DataFrame from the file(s) pointed to by path
    people = sqlContext.jsonFile(path)
    # root
    #  |-- person_name: string (nullable = false)
    #  |-- person_age: integer (nullable = false)

    # The inferred schema can be visualized using the printSchema() method.
    people.printSchema()
    # root
    #  |-- age: IntegerType
    #  |-- name: StringType

    # Register this DataFrame as a table.
    people.registerAsTable("people")
    some_df.registerAsTable("people2")
    # SQL statements can be run by using the sql methods provided by sqlContext
    teenagers = sqlContext.sql("SELECT * FROM people2 ")
    print("!!!!!---------------\n")
    for each in teenagers.collect():
        print(each)
        print("---------------\n")
    print("!!!!!-------------\n")
    sc.stop()



> pyspark sql bug ,here is the testcase
> -------------------------------------
>
>                 Key: SPARK-15404
>                 URL: https://issues.apache.org/jira/browse/SPARK-15404
>             Project: Spark
>          Issue Type: Bug
>         Environment: 1.6
>            Reporter: 郭同
>
> from __future__ import print_function
> import os
> import sys
> from pyspark import SparkContext
> from pyspark.sql import SQLContext
> from pyspark.sql.types import Row, StructField, StructType, StringType, IntegerType
> if __name__ == "__main__":
>     sc = SparkContext(appName="PythonSQL")
>     sqlContext = SQLContext(sc)
>     schema = StructType([StructField("person_name", StringType(), False),
>                      StructField("person_age", IntegerType(), False)])
>     some_rdd = sc.parallelize([Row(person_name="John", person_age=19),
>                                Row(person_name="Smith", person_age=23),
>                                Row(person_name="Sarah", person_age=18)])
>     some_df = sqlContext.createDataFrame(some_rdd, schema)
>     some_df.printSchema()
>     some_df.registerAsTable("people")
>     teenagers = sqlContext.sql("SELECT * FROM people ")
>     for each in teenagers.collect():
>         print(each)
>     sc.stop()



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org