[GitHub] [iceberg] TGooch44 commented on a change in pull request #1727: [python] Adding parquet package and the classes for reading parquet

[GitBox <gi...@apache.org>]

TGooch44 commented on a change in pull request #1727:
URL: https://github.com/apache/iceberg/pull/1727#discussion_r535781402



##########
File path: python/iceberg/parquet/parquet_reader.py
##########
@@ -0,0 +1,264 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+from collections import namedtuple
+from datetime import datetime
+import decimal
+import logging
+import typing
+
+from iceberg.api import Schema
+from iceberg.api.expressions import Expression
+from iceberg.api.io import InputFile
+from iceberg.api.types import NestedField, Type, TypeID
+from iceberg.core.filesystem import FileSystem, LocalFileSystem, S3FileSystem
+from iceberg.core.util.profile import profile
+from iceberg.exceptions import FileSystemNotFound, InvalidCastException
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+from pyarrow import fs
+import pyarrow.dataset as ds
+import pyarrow.parquet as pq
+
+from .dataset_utils import get_dataset_filter
+from .parquet_schema_utils import prune_columns
+from .parquet_to_iceberg import convert_parquet_to_iceberg
+
+_logger = logging.getLogger(__name__)
+
+DTYPE_MAP: typing.Dict[TypeID,
+                       typing.Callable[[NestedField], typing.Tuple[pa.Field, typing.Any]]] = \
+    {TypeID.BINARY: lambda field: pa.binary(),
+     TypeID.BOOLEAN: lambda field: (pa.bool_(), False),
+     TypeID.DATE: lambda field: (pa.date32(), datetime.now()),
+     TypeID.DECIMAL: lambda field: (pa.decimal128(field.type.precision, field.type.scale),
+                                    decimal.Decimal()),
+     TypeID.DOUBLE: lambda field: (pa.float64(), np.nan),
+     TypeID.FIXED: lambda field: pa.binary(field.length),
+     TypeID.FLOAT: lambda field: (pa.float32(), np.nan),
+     TypeID.INTEGER: lambda field: (pa.int32(), np.nan),
+     TypeID.LIST: lambda field: (pa.list_(pa.field("element",
+                                                   DTYPE_MAP[field.type.element_type.type_id](field.type)[0])),
+                                 None),
+     TypeID.LONG: lambda field: (pa.int64(), np.nan),
+     # To-Do: update to support reading map fields
+     # TypeID.MAP: lambda field: (,),
+     TypeID.STRING: lambda field: (pa.string(), ""),
+     TypeID.STRUCT: lambda field: (pa.struct([(nested_field.name,
+                                               DTYPE_MAP[nested_field.type.type_id](nested_field.type)[0])
+                                              for nested_field in field.type.fields]), {}),
+     TypeID.TIMESTAMP: lambda field: (pa.timestamp("us"), datetime.now()),
+     # not used in SPARK, so not implementing for now
+     # TypeID.TIME: pa.time64(None)
+     }
+
+FS_MAP: typing.Dict[typing.Type[FileSystem], typing.Type[fs.FileSystem]] = {LocalFileSystem: fs.LocalFileSystem}
+
+try:
+    FS_MAP[S3FileSystem] = fs.S3FileSystem

Review comment:
       what happens is the import succeeds for `fs`, but when you actually do fs.S3FileSystem it throws the import error. It might be clearer to do it all in the `try...except` though...let me just change that




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org