You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/01/16 22:31:30 UTC
[2/3] incubator-kudu git commit: [python client] - Expand C++ API
coverage and improve usability and documentation
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/libkudu_client.pxd
----------------------------------------------------------------------
diff --git a/python/kudu/libkudu_client.pxd b/python/kudu/libkudu_client.pxd
new file mode 100644
index 0000000..a44027e
--- /dev/null
+++ b/python/kudu/libkudu_client.pxd
@@ -0,0 +1,606 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from libc.stdint cimport *
+from libcpp cimport bool as c_bool
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+
+# This must be included for cerr and other things to work
+cdef extern from "<iostream>":
+ pass
+
+#----------------------------------------------------------------------
+# Smart pointers and such
+
+cdef extern from "<tr1/memory>" namespace "std::tr1" nogil:
+
+ cdef cppclass shared_ptr[T]:
+ T* get()
+ void reset()
+ void reset(T* p)
+
+cdef extern from "kudu/util/status.h" namespace "kudu" nogil:
+
+ # We can later add more of the common status factory methods as needed
+ cdef Status Status_OK "Status::OK"()
+
+ cdef cppclass Status:
+ Status()
+
+ string ToString()
+
+ Slice message()
+
+ c_bool ok()
+ c_bool IsNotFound()
+ c_bool IsCorruption()
+ c_bool IsNotSupported()
+ c_bool IsIOError()
+ c_bool IsInvalidArgument()
+ c_bool IsAlreadyPresent()
+ c_bool IsRuntimeError()
+ c_bool IsNetworkError()
+ c_bool IsIllegalState()
+ c_bool IsNotAuthorized()
+ c_bool IsAborted()
+
+
+cdef extern from "kudu/util/monotime.h" namespace "kudu" nogil:
+
+ # These classes are not yet needed directly but will need to be completed
+ # from the C++ API
+ cdef cppclass MonoDelta:
+ MonoDelta()
+
+ @staticmethod
+ MonoDelta FromSeconds(double seconds)
+
+ @staticmethod
+ MonoDelta FromMilliseconds(int64_t ms)
+
+ @staticmethod
+ MonoDelta FromMicroseconds(int64_t us)
+
+ @staticmethod
+ MonoDelta FromNanoseconds(int64_t ns)
+
+ c_bool Initialized()
+ c_bool LessThan(const MonoDelta& other)
+ c_bool MoreThan(const MonoDelta& other)
+ c_bool Equals(const MonoDelta& other)
+
+ string ToString()
+
+ double ToSeconds()
+ int64_t ToMilliseconds()
+ int64_t ToMicroseconds()
+ int64_t ToNanoseconds()
+
+ # TODO, when needed
+ # void ToTimeVal(struct timeval *tv)
+ # void ToTimeSpec(struct timespec *ts)
+
+ # @staticmethod
+ # void NanosToTimeSpec(int64_t nanos, struct timespec* ts);
+
+
+ cdef cppclass MonoTime:
+ pass
+
+
+cdef extern from "kudu/client/schema.h" namespace "kudu::client" nogil:
+
+ enum DataType" kudu::client::KuduColumnSchema::DataType":
+ KUDU_INT8 " kudu::client::KuduColumnSchema::INT8"
+ KUDU_INT16 " kudu::client::KuduColumnSchema::INT16"
+ KUDU_INT32 " kudu::client::KuduColumnSchema::INT32"
+ KUDU_INT64 " kudu::client::KuduColumnSchema::INT64"
+ KUDU_STRING " kudu::client::KuduColumnSchema::STRING"
+ KUDU_BOOL " kudu::client::KuduColumnSchema::BOOL"
+ KUDU_FLOAT " kudu::client::KuduColumnSchema::FLOAT"
+ KUDU_DOUBLE " kudu::client::KuduColumnSchema::DOUBLE"
+ KUDU_BINARY " kudu::client::KuduColumnSchema::BINARY"
+ KUDU_TIMESTAMP " kudu::client::KuduColumnSchema::TIMESTAMP"
+
+ enum EncodingType" kudu::client::KuduColumnStorageAttributes::EncodingType":
+ EncodingType_AUTO " kudu::client::KuduColumnStorageAttributes::AUTO_ENCODING"
+ EncodingType_PLAIN " kudu::client::KuduColumnStorageAttributes::PLAIN_ENCODING"
+ EncodingType_PREFIX " kudu::client::KuduColumnStorageAttributes::PREFIX_ENCODING"
+ EncodingType_GROUP_VARINT " kudu::client::KuduColumnStorageAttributes::GROUP_VARINT"
+ EncodingType_RLE " kudu::client::KuduColumnStorageAttributes::RLE"
+
+ enum CompressionType" kudu::client::KuduColumnStorageAttributes::CompressionType":
+ CompressionType_DEFAULT " kudu::client::KuduColumnStorageAttributes::DEFAULT_COMPRESSION"
+ CompressionType_NONE " kudu::client::KuduColumnStorageAttributes::NO_COMPRESSION"
+ CompressionType_SNAPPY " kudu::client::KuduColumnStorageAttributes::SNAPPY"
+ CompressionType_LZ4 " kudu::client::KuduColumnStorageAttributes::LZ4"
+ CompressionType_ZLIB " kudu::client::KuduColumnStorageAttributes::ZLIB"
+
+ cdef struct KuduColumnStorageAttributes:
+ KuduColumnStorageAttributes()
+
+ EncodingType encoding
+ CompressionType compression
+ string ToString()
+
+ cdef cppclass KuduColumnSchema:
+ KuduColumnSchema(const KuduColumnSchema& other)
+ KuduColumnSchema(const string& name, DataType type)
+ KuduColumnSchema(const string& name, DataType type, c_bool is_nullable)
+ KuduColumnSchema(const string& name, DataType type, c_bool is_nullable,
+ const void* default_value)
+
+ string& name()
+ c_bool is_nullable()
+ DataType type()
+
+ c_bool Equals(KuduColumnSchema& other)
+ void CopyFrom(KuduColumnSchema& other)
+
+ cdef cppclass KuduSchema:
+ KuduSchema()
+ KuduSchema(vector[KuduColumnSchema]& columns, int key_columns)
+
+ c_bool Equals(const KuduSchema& other)
+ KuduColumnSchema Column(size_t idx)
+ size_t num_columns()
+
+ void GetPrimaryKeyColumnIndexes(vector[int]* indexes)
+
+ KuduPartialRow* NewRow()
+
+ cdef cppclass KuduColumnSpec:
+
+ KuduColumnSpec* Default(KuduValue* value)
+ KuduColumnSpec* RemoveDefault()
+
+ KuduColumnSpec* Compression(CompressionType compression)
+ KuduColumnSpec* Encoding(EncodingType encoding)
+ KuduColumnSpec* BlockSize(int32_t block_size)
+
+ KuduColumnSpec* PrimaryKey()
+ KuduColumnSpec* NotNull()
+ KuduColumnSpec* Nullable()
+ KuduColumnSpec* Type(DataType type_)
+
+ KuduColumnSpec* RenameTo(string& new_name)
+
+
+ cdef cppclass KuduSchemaBuilder:
+
+ KuduColumnSpec* AddColumn(string& name)
+ KuduSchemaBuilder* SetPrimaryKey(vector[string]& key_col_names);
+
+ Status Build(KuduSchema* schema)
+
+
+cdef extern from "kudu/client/row_result.h" namespace "kudu::client" nogil:
+
+ cdef cppclass KuduRowResult:
+ c_bool IsNull(Slice& col_name)
+ c_bool IsNull(int col_idx)
+
+ # These getters return a bad Status if the type does not match,
+ # the value is unset, or the value is NULL. Otherwise they return
+ # the current set value in *val.
+ Status GetBool(Slice& col_name, c_bool* val)
+
+ Status GetInt8(Slice& col_name, int8_t* val)
+ Status GetInt16(Slice& col_name, int16_t* val)
+ Status GetInt32(Slice& col_name, int32_t* val)
+ Status GetInt64(Slice& col_name, int64_t* val)
+
+ Status GetTimestamp(const Slice& col_name,
+ int64_t* micros_since_utc_epoch)
+
+ Status GetBool(int col_idx, c_bool* val)
+
+ Status GetInt8(int col_idx, int8_t* val)
+ Status GetInt16(int col_idx, int16_t* val)
+ Status GetInt32(int col_idx, int32_t* val)
+ Status GetInt64(int col_idx, int64_t* val)
+
+ Status GetString(Slice& col_name, Slice* val)
+ Status GetString(int col_idx, Slice* val)
+
+ Status GetFloat(Slice& col_name, float* val)
+ Status GetFloat(int col_idx, float* val)
+
+ Status GetDouble(Slice& col_name, double* val)
+ Status GetDouble(int col_idx, double* val)
+
+ Status GetBinary(const Slice& col_name, Slice* val)
+ Status GetBinary(int col_idx, Slice* val)
+
+ const void* cell(int col_idx)
+ string ToString()
+
+
+cdef extern from "kudu/util/slice.h" namespace "kudu" nogil:
+
+ cdef cppclass Slice:
+ Slice()
+ Slice(const uint8_t* data, size_t n)
+ Slice(const char* data, size_t n)
+
+ Slice(string& s)
+ Slice(const char* s)
+
+ # Many other constructors have been omitted; we can return and add them
+ # as needed for the code generation.
+
+ const uint8_t* data()
+ uint8_t* mutable_data()
+ size_t size()
+ c_bool empty()
+
+ uint8_t operator[](size_t n)
+
+ void clear()
+ void remove_prefix(size_t n)
+ void truncate(size_t n)
+
+ Status check_size(size_t expected_size)
+
+ string ToString()
+
+ string ToDebugString()
+ string ToDebugString(size_t max_len)
+
+ int compare(Slice& b)
+
+ c_bool starts_with(Slice& x)
+
+ void relocate(uint8_t* d)
+
+ # Many other API methods omitted
+
+
+cdef extern from "kudu/common/partial_row.h" namespace "kudu" nogil:
+
+ cdef cppclass KuduPartialRow:
+ # Schema must not be garbage-collected
+ # KuduPartialRow(const Schema* schema)
+
+ #----------------------------------------------------------------------
+ # Setters
+
+ # Slice setters
+ Status SetBool(Slice& col_name, c_bool val)
+
+ Status SetInt8(Slice& col_name, int8_t val)
+ Status SetInt16(Slice& col_name, int16_t val)
+ Status SetInt32(Slice& col_name, int32_t val)
+ Status SetInt64(Slice& col_name, int64_t val)
+
+ Status SetTimestamp(const Slice& col_name,
+ int64_t micros_since_utc_epoch)
+ Status SetTimestamp(int col_idx, int64_t micros_since_utc_epoch)
+
+ Status SetDouble(Slice& col_name, double val)
+ Status SetFloat(Slice& col_name, float val)
+
+ # Integer setters
+ Status SetBool(int col_idx, c_bool val)
+
+ Status SetInt8(int col_idx, int8_t val)
+ Status SetInt16(int col_idx, int16_t val)
+ Status SetInt32(int col_idx, int32_t val)
+ Status SetInt64(int col_idx, int64_t val)
+
+ Status SetDouble(int col_idx, double val)
+ Status SetFloat(int col_idx, float val)
+
+ # Set, but does not copy string
+ Status SetString(Slice& col_name, Slice& val)
+ Status SetString(int col_idx, Slice& val)
+
+ Status SetStringCopy(Slice& col_name, Slice& val)
+ Status SetStringCopy(int col_idx, Slice& val)
+
+ Status SetBinaryCopy(const Slice& col_name, const Slice& val)
+ Status SetBinaryCopy(int col_idx, const Slice& val)
+
+ Status SetNull(Slice& col_name)
+ Status SetNull(int col_idx)
+
+ Status Unset(Slice& col_name)
+ Status Unset(int col_idx)
+
+ #----------------------------------------------------------------------
+ # Getters
+
+ c_bool IsColumnSet(Slice& col_name)
+ c_bool IsColumnSet(int col_idx)
+
+ c_bool IsNull(Slice& col_name)
+ c_bool IsNull(int col_idx)
+
+ Status GetBool(Slice& col_name, c_bool* val)
+ Status GetBool(int col_idx, c_bool* val)
+
+ Status GetInt8(Slice& col_name, int8_t* val)
+ Status GetInt8(int col_idx, int8_t* val)
+
+ Status GetInt16(Slice& col_name, int16_t* val)
+ Status GetInt16(int col_idx, int16_t* val)
+
+ Status GetInt32(Slice& col_name, int32_t* val)
+ Status GetInt32(int col_idx, int32_t* val)
+
+ Status GetInt64(Slice& col_name, int64_t* val)
+ Status GetInt64(int col_idx, int64_t* val)
+
+ Status GetTimestamp(const Slice& col_name,
+ int64_t* micros_since_utc_epoch)
+ Status GetTimestamp(int col_idx, int64_t* micros_since_utc_epoch)
+
+ Status GetDouble(Slice& col_name, double* val)
+ Status GetDouble(int col_idx, double* val)
+
+ Status GetFloat(Slice& col_name, float* val)
+ Status GetFloat(int col_idx, float* val)
+
+ # Gets the string but does not copy the value. Callers should
+ # copy the resulting Slice if necessary.
+ Status GetString(Slice& col_name, Slice* val)
+ Status GetString(int col_idx, Slice* val)
+
+ Status GetBinary(const Slice& col_name, Slice* val)
+ Status GetBinary(int col_idx, Slice* val)
+
+ Status EncodeRowKey(string* encoded_key)
+ string ToEncodedRowKeyOrDie()
+
+ # Return true if all of the key columns have been specified
+ # for this mutation.
+ c_bool IsKeySet()
+
+ # Return true if all columns have been specified.
+ c_bool AllColumnsSet()
+ string ToString()
+
+ # const Schema* schema()
+
+
+cdef extern from "kudu/client/write_op.h" namespace "kudu::client" nogil:
+
+ enum WriteType" kudu::client::KuduWriteOperation::Type":
+ INSERT " kudu::client::KuduWriteOperation::INSERT"
+ UPDATE " kudu::client::KuduWriteOperation::UPDATE"
+ DELETE " kudu::client::KuduWriteOperation::DELETE"
+
+ cdef cppclass KuduWriteOperation:
+ KuduPartialRow& row()
+ KuduPartialRow* mutable_row()
+
+ # This is a pure virtual function implemented on each of the cppclass
+ # subclasses
+ string ToString()
+
+ # Also a pure virtual
+ WriteType type()
+
+ cdef cppclass KuduInsert(KuduWriteOperation):
+ pass
+
+ cdef cppclass KuduDelete(KuduWriteOperation):
+ pass
+
+ cdef cppclass KuduUpdate(KuduWriteOperation):
+ pass
+
+
+cdef extern from "kudu/client/scan_predicate.h" namespace "kudu::client" nogil:
+ enum ComparisonOp" kudu::client::KuduPredicate::ComparisonOp":
+ KUDU_LESS_EQUAL " kudu::client::KuduPredicate::LESS_EQUAL"
+ KUDU_GREATER_EQUAL " kudu::client::KuduPredicate::GREATER_EQUAL"
+ KUDU_EQUAL " kudu::client::KuduPredicate::EQUAL"
+
+ cdef cppclass KuduPredicate:
+ KuduPredicate* Clone()
+
+
+cdef extern from "kudu/client/value.h" namespace "kudu::client" nogil:
+
+ cdef cppclass KuduValue:
+ @staticmethod
+ KuduValue* FromInt(int64_t val);
+
+ @staticmethod
+ KuduValue* FromFloat(float val);
+
+ @staticmethod
+ KuduValue* FromDouble(double val);
+
+ @staticmethod
+ KuduValue* FromBool(c_bool val);
+
+ @staticmethod
+ KuduValue* CopyString(const Slice& s);
+
+
+cdef extern from "kudu/client/client.h" namespace "kudu::client" nogil:
+
+ # Omitted KuduClient::ReplicaSelection enum
+
+ cdef cppclass KuduClient:
+
+ Status DeleteTable(const string& table_name)
+ Status OpenTable(const string& table_name,
+ shared_ptr[KuduTable]* table)
+ Status GetTableSchema(const string& table_name, KuduSchema* schema)
+
+ KuduTableCreator* NewTableCreator()
+ Status IsCreateTableInProgress(const string& table_name,
+ c_bool* create_in_progress)
+
+ c_bool IsMultiMaster()
+
+ Status ListTables(vector[string]* tables)
+ Status ListTables(vector[string]* tables, const string& filter)
+
+ Status TableExists(const string& table_name, c_bool* exists)
+
+ KuduTableAlterer* NewTableAlterer()
+ Status IsAlterTableInProgress(const string& table_name,
+ c_bool* alter_in_progress)
+
+ shared_ptr[KuduSession] NewSession()
+
+ cdef cppclass KuduClientBuilder:
+ KuduClientBuilder()
+ KuduClientBuilder& master_server_addrs(const vector[string]& addrs)
+ KuduClientBuilder& add_master_server_addr(const string& addr)
+
+ KuduClientBuilder& default_admin_operation_timeout(
+ const MonoDelta& timeout)
+
+ KuduClientBuilder& default_rpc_timeout(const MonoDelta& timeout)
+
+ Status Build(shared_ptr[KuduClient]* client)
+
+ cdef cppclass KuduTableCreator:
+ KuduTableCreator& table_name(string& name)
+ KuduTableCreator& schema(KuduSchema* schema)
+ KuduTableCreator& split_keys(vector[string]& keys)
+ KuduTableCreator& num_replicas(int n_replicas)
+ KuduTableCreator& wait(c_bool wait)
+
+ Status Create()
+
+ cdef cppclass KuduTableAlterer:
+ # The name of the existing table to alter
+ KuduTableAlterer& table_name(string& name)
+
+ KuduTableAlterer& rename_table(string& name)
+
+ KuduTableAlterer& add_column(string& name, DataType type,
+ const void *default_value)
+ KuduTableAlterer& add_column(string& name, DataType type,
+ const void *default_value,
+ KuduColumnStorageAttributes attr)
+
+ KuduTableAlterer& add_nullable_column(string& name, DataType type)
+
+ KuduTableAlterer& drop_column(string& name)
+
+ KuduTableAlterer& rename_column(string& old_name, string& new_name)
+
+ KuduTableAlterer& wait(c_bool wait)
+
+ Status Alter()
+
+ # Instances of KuduTable are not directly instantiated by users of the
+ # client.
+ cdef cppclass KuduTable:
+
+ string& name()
+ KuduSchema& schema()
+
+ KuduInsert* NewInsert()
+ KuduUpdate* NewUpdate()
+ KuduDelete* NewDelete()
+
+ KuduPredicate* NewComparisonPredicate(const Slice& col_name,
+ ComparisonOp op,
+ KuduValue* value);
+
+ KuduClient* client()
+ # const PartitionSchema& partition_schema()
+
+ enum FlushMode" kudu::client::KuduSession::FlushMode":
+ FlushMode_AutoSync " kudu::client::KuduSession::AUTO_FLUSH_SYNC"
+ FlushMode_AutoBackground " kudu::client::KuduSession::AUTO_FLUSH_BACKGROUND"
+ FlushMode_Manual " kudu::client::KuduSession::MANUAL_FLUSH"
+
+ cdef cppclass KuduSession:
+
+ Status SetFlushMode(FlushMode m)
+
+ void SetMutationBufferSpace(size_t size)
+ void SetTimeoutMillis(int millis)
+
+ void SetPriority(int priority)
+
+ Status Apply(KuduWriteOperation* write_op)
+ Status Apply(KuduInsert* write_op)
+ Status Apply(KuduUpdate* write_op)
+ Status Apply(KuduDelete* write_op)
+
+ # This is thread-safe
+ Status Flush()
+
+ # TODO: Will need to decide on a strategy for exposing the session's
+ # async API to Python
+
+ # Status ApplyAsync(KuduWriteOperation* write_op,
+ # KuduStatusCallback cb)
+ # Status ApplyAsync(KuduInsert* write_op,
+ # KuduStatusCallback cb)
+ # Status ApplyAsync(KuduUpdate* write_op,
+ # KuduStatusCallback cb)
+ # Status ApplyAsync(KuduDelete* write_op,
+ # KuduStatusCallback cb)
+ # void FlushAsync(KuduStatusCallback& cb)
+
+
+ Status Close()
+ c_bool HasPendingOperations()
+ int CountBufferedOperations()
+
+ int CountPendingErrors()
+ void GetPendingErrors(vector[C_KuduError*]* errors, c_bool* overflowed)
+
+ KuduClient* client()
+
+ enum ReadMode" kudu::client::KuduScanner::ReadMode":
+ READ_LATEST " kudu::client::KuduScanner::READ_LATEST"
+ READ_AT_SNAPSHOT " kudu::client::KuduScanner::READ_AT_SNAPSHOT"
+
+ cdef cppclass KuduScanner:
+ KuduScanner(KuduTable* table)
+
+ Status AddConjunctPredicate(KuduPredicate* pred)
+
+ Status Open()
+ void Close()
+
+ c_bool HasMoreRows()
+ Status NextBatch(vector[KuduRowResult]* rows)
+ Status SetBatchSizeBytes(uint32_t batch_size)
+
+ # Pending definition of ReplicaSelection enum
+ # Status SetSelection(ReplicaSelection selection)
+
+ Status SetReadMode(ReadMode read_mode)
+ Status SetSnapshot(uint64_t snapshot_timestamp_micros)
+ Status SetTimeoutMillis(int millis)
+
+ string ToString()
+
+ cdef cppclass C_KuduError " kudu::client::KuduError":
+
+ Status& status()
+
+ KuduWriteOperation& failed_op()
+ KuduWriteOperation* release_failed_op()
+
+ c_bool was_possibly_successful()
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/schema.pxd
----------------------------------------------------------------------
diff --git a/python/kudu/schema.pxd b/python/kudu/schema.pxd
new file mode 100644
index 0000000..b70f8ad
--- /dev/null
+++ b/python/kudu/schema.pxd
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from libcpp.map cimport map
+
+from libkudu_client cimport *
+
+
+cdef class KuduType(object):
+ cdef readonly:
+ DataType type
+
+
+cdef class ColumnSchema:
+ """
+ Wraps a Kudu client ColumnSchema object
+ """
+ cdef:
+ KuduColumnSchema* schema
+ KuduType _type
+
+
+cdef class ColumnSpec:
+ cdef:
+ KuduColumnSpec* spec
+
+
+cdef class SchemaBuilder:
+ cdef:
+ KuduSchemaBuilder builder
+
+
+cdef class Schema:
+ cdef:
+ const KuduSchema* schema
+ object parent
+ bint own_schema
+ map[string, int] _col_mapping
+ bint _mapping_initialized
+
+ cdef int get_loc(self, name) except -1
+
+ cdef inline DataType loc_type(self, int i):
+ return self.schema.Column(i).type()
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/schema.pyx
----------------------------------------------------------------------
diff --git a/python/kudu/schema.pyx b/python/kudu/schema.pyx
new file mode 100644
index 0000000..f02d0e4
--- /dev/null
+++ b/python/kudu/schema.pyx
@@ -0,0 +1,545 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+# cython: embedsignature = True
+
+from cython.operator cimport dereference as deref
+
+from kudu.compat import tobytes, frombytes
+from kudu.schema cimport *
+from kudu.errors cimport check_status
+
+import six
+
+from . import util
+
+BOOL = KUDU_BOOL
+STRING = KUDU_STRING
+
+INT8 = KUDU_INT8
+INT16 = KUDU_INT16
+INT32 = KUDU_INT32
+INT64 = KUDU_INT64
+
+FLOAT = KUDU_FLOAT
+DOUBLE = KUDU_DOUBLE
+
+TIMESTAMP = KUDU_TIMESTAMP
+BINARY = KUDU_BINARY
+
+
+cdef dict _reverse_dict(d):
+ return dict((v, k) for k, v in d.items())
+
+
+# CompressionType enums
+COMPRESSION_DEFAULT = CompressionType_DEFAULT
+COMPRESSION_NONE = CompressionType_NONE
+COMPRESSION_SNAPPY = CompressionType_SNAPPY
+COMPRESSION_LZ4 = CompressionType_LZ4
+COMPRESSION_ZLIB = CompressionType_ZLIB
+
+cdef dict _compression_types = {
+ 'default': COMPRESSION_DEFAULT,
+ 'none': COMPRESSION_NONE,
+ 'snappy': COMPRESSION_SNAPPY,
+ 'lz4': COMPRESSION_LZ4,
+ 'zlib': COMPRESSION_ZLIB,
+}
+
+cdef dict _compression_type_to_name = _reverse_dict(_compression_types)
+
+
+# EncodingType enums
+ENCODING_AUTO = EncodingType_AUTO
+ENCODING_PLAIN = EncodingType_PLAIN
+ENCODING_PREFIX = EncodingType_PREFIX
+ENCODING_GROUP_VARINT = EncodingType_GROUP_VARINT
+ENCODING_RLE = EncodingType_RLE
+
+cdef dict _encoding_types = {
+ 'auto': ENCODING_AUTO,
+ 'plain': ENCODING_PLAIN,
+ 'prefix': ENCODING_PREFIX,
+ 'group_varint': ENCODING_GROUP_VARINT,
+ 'rle': ENCODING_RLE,
+}
+
+cdef dict _encoding_type_to_name = _reverse_dict(_encoding_types)
+
+
+cdef class KuduType(object):
+
+ """
+ Usability wrapper for Kudu data type enum
+ """
+
+ def __cinit__(self, DataType type):
+ self.type = type
+
+ property name:
+
+ def __get__(self):
+ return _type_names[self.type]
+
+ def __repr__(self):
+ return 'KuduType({0})'.format(self.name)
+
+
+int8 = KuduType(KUDU_INT8)
+int16 = KuduType(KUDU_INT16)
+int32 = KuduType(KUDU_INT32)
+int64 = KuduType(KUDU_INT64)
+string_ = KuduType(KUDU_STRING)
+bool_ = KuduType(KUDU_BOOL)
+float_ = KuduType(KUDU_FLOAT)
+double_ = KuduType(KUDU_DOUBLE)
+binary = KuduType(KUDU_BINARY)
+timestamp = KuduType(KUDU_TIMESTAMP)
+
+
+cdef dict _type_names = {
+ INT8: 'int8',
+ INT16: 'int16',
+ INT32: 'int32',
+ INT64: 'int64',
+ STRING: 'string',
+ BOOL: 'bool',
+ FLOAT: 'float',
+ DOUBLE: 'double',
+ BINARY: 'binary',
+ TIMESTAMP: 'timestamp'
+}
+
+
+cdef dict _type_name_to_number = _reverse_dict(_type_names)
+
+cdef dict _type_to_obj = {
+ INT8: int8,
+ INT16: int16,
+ INT32: int32,
+ INT64: int64,
+ STRING: string_,
+ BOOL: bool_,
+ FLOAT: float_,
+ DOUBLE: double_,
+ BINARY: binary,
+ TIMESTAMP: timestamp
+}
+
+
+cdef KuduType to_data_type(object obj):
+ if isinstance(obj, KuduType):
+ return obj
+ elif isinstance(obj, six.string_types):
+ return _type_to_obj[_type_name_to_number[obj]]
+ elif obj in _type_to_obj:
+ return _type_to_obj[obj]
+ else:
+ raise ValueError('Invalid type: {0}'.format(obj))
+
+
+cdef class ColumnSchema:
+ """
+ Wraps a Kudu client ColumnSchema object. Use schema.at(i) or schema[i] to
+ construct one.
+ """
+
+ def __cinit__(self):
+ self.schema = NULL
+ self._type = None
+
+ def __dealloc__(self):
+ if self.schema is not NULL:
+ del self.schema
+
+ property name:
+ def __get__(self):
+ return frombytes(self.schema.name())
+
+ property type:
+ def __get__(self):
+ if self._type is None:
+ self._type = _type_to_obj[self.schema.type()]
+ return self._type
+
+ property nullable:
+ def __get__(self):
+ return self.schema.is_nullable()
+
+ def equals(self, other):
+ if not isinstance(other, ColumnSchema):
+ return False
+ return self.schema.Equals(deref((<ColumnSchema> other).schema))
+
+ def __repr__(self):
+ return ('ColumnSchema(name=%s, type=%s, nullable=%s)'
+ % (self.name, self.type.name,
+ self.nullable))
+
+
+#----------------------------------------------------------------------
+
+cdef class ColumnSpec:
+
+ """
+ Helper class for configuring a column's settings while using the
+ SchemaBuilder.
+ """
+
+ def type(self, type_):
+ self.spec.Type(to_data_type(type_).type)
+ return self
+
+ def default(self, value):
+ """
+ Set a default value for the column
+ """
+ raise NotImplementedError
+
+ def clear_default(self):
+ """
+ Remove a default value set.
+ """
+ raise NotImplementedError
+
+ def compression(self, compression):
+ """
+ Set the compression type
+
+ Parameters
+ ----------
+ compression : string or int
+ One of {'default', 'none', 'snappy', 'lz4', 'zlib'}
+ Or see kudu.COMPRESSION_* constants
+
+ Returns
+ -------
+ self
+ """
+ cdef CompressionType type
+ if isinstance(compression, int):
+ # todo: validation
+ type = <CompressionType> compression
+ else:
+ if compression is None:
+ type = CompressionType_NONE
+ else:
+ try:
+ type = _compression_types[compression.lower()]
+ except KeyError:
+ raise ValueError('Invalid compression type: {0}'
+ .format(compression))
+
+ self.spec.Compression(type)
+ return self
+
+ def encoding(self, encoding):
+ """
+ Set the encoding type
+
+ Parameters
+ ----------
+ encoding : string or int
+ One of {'auto', 'plain', 'prefix', 'group_varint', 'rle'}
+ Or see kudu.ENCODING_* constants
+
+ Returns
+ -------
+ self
+ """
+ cdef EncodingType type
+ if isinstance(encoding, six.string_types):
+ try:
+ type = _encoding_types[encoding.lower()]
+ except KeyError:
+ raise ValueError('Invalid encoding type: {0}'
+ .format(encoding))
+ else:
+ # todo: validation
+ type = <EncodingType> encoding
+
+ self.spec.Encoding(type)
+ return self
+
+ def primary_key(self):
+ """
+ Make this column a primary key. If you use this method, it will be the
+ only primary key. Otherwise see set_primary_keys method on
+ SchemaBuilder.
+
+ Returns
+ -------
+ self
+ """
+ self.spec.PrimaryKey()
+ return self
+
+ def nullable(self, bint is_nullable=True):
+ """
+ Set nullable (True) or not nullable (False)
+
+ Parameters
+ ----------
+ is_nullable : boolean, default True
+
+ Returns
+ -------
+ self
+ """
+ if is_nullable:
+ self.spec.Nullable()
+ else:
+ self.spec.NotNull()
+ return self
+
+ def rename(self, new_name):
+ """
+ Change the column name.
+
+ TODO: Not implemented for table creation
+ """
+ self.spec.RenameTo(new_name)
+ return self
+
+
+cdef class SchemaBuilder:
+
+ def add_column(self, name, type_=None, nullable=None, compression=None,
+ encoding=None, primary_key=False):
+ """
+ Add a new column to the schema. Returns a ColumnSpec object for further
+ configuration and use in a fluid programming style.
+
+ Parameters
+ ----------
+ name : string
+ type_ : string or KuduType
+ Data type e.g. 'int32' or kudu.int32
+ nullable : boolean, default None
+ New columns are nullable by default. Set boolean value for explicit
+ nullable / not-nullable
+ compression : string or int
+ One of {'default', 'none', 'snappy', 'lz4', 'zlib'}
+ Or see kudu.COMPRESSION_* constants
+ encoding : string or int
+ One of {'auto', 'plain', 'prefix', 'group_varint', 'rle'}
+ Or see kudu.ENCODING_* constants
+ primary_key : boolean, default False
+ Use this column as the table primary key
+
+ Examples
+ --------
+ (builder.add_column('foo')
+ .nullable(True)
+ .compression('lz4'))
+
+ Returns
+ -------
+ spec : ColumnSpec
+ """
+ cdef:
+ ColumnSpec result = ColumnSpec()
+ string c_name = tobytes(name)
+
+ result.spec = self.builder.AddColumn(c_name)
+
+ if type_ is not None:
+ result.type(type_)
+
+ if nullable is not None:
+ result.nullable(nullable)
+
+ if compression is not None:
+ result.compression(compression)
+
+ if encoding is not None:
+ result.encoding(encoding)
+
+ if primary_key:
+ result.primary_key()
+
+ return result
+
+ def set_primary_keys(self, key_names):
+ """
+ Set indicated columns (by name) to be the primary keys of the table
+ schema
+
+ Parameters
+ ----------
+ key_names : list of Python strings
+
+ Returns
+ -------
+ None
+ """
+ cdef:
+ vector[string] key_col_names
+
+ for name in key_names:
+ key_col_names.push_back(tobytes(name))
+
+ self.builder.SetPrimaryKey(key_col_names)
+
+ def build(self):
+ """
+ Creates an immutable Schema object after the user has finished adding
+ and onfiguring columns
+
+ Returns
+ -------
+ schema : Schema
+ """
+ cdef Schema result = Schema()
+ cdef KuduSchema* schema = new KuduSchema()
+ check_status(self.builder.Build(schema))
+
+ result.schema = schema
+ return result
+
+
+cdef class Schema:
+
+ """
+ Container for a Kudu table schema. Obtain from Table instances or create
+ new ones using kudu.SchemaBuilder
+ """
+
+ def __cinit__(self):
+ # Users should not call this directly
+ self.schema = NULL
+ self.own_schema = 1
+ self._col_mapping.clear()
+ self._mapping_initialized = 0
+
+ def __dealloc__(self):
+ if self.schema is not NULL and self.own_schema:
+ del self.schema
+
+ property names:
+
+ def __get__(self):
+ result = []
+ for i in range(self.schema.num_columns()):
+ name = frombytes(self.schema.Column(i).name())
+ result.append(name)
+
+ return result
+
+ def __repr__(self):
+ # Got to be careful with huge schemas, maybe some kind of summary repr
+ # when more than 20-30 columns?
+ buf = six.StringIO()
+
+ col_names = self.names
+ space = 2 + max(len(x) for x in col_names)
+
+ for i in range(len(self)):
+ col = self.at(i)
+ not_null = '' if col.nullable else ' NOT NULL'
+
+ buf.write('\n{0}{1}{2}'
+ .format(col.name.ljust(space),
+ col.type.name, not_null))
+
+ pk_string = ', '.join(col_names[i] for i in self.primary_key_indices())
+ buf.write('\nPRIMARY KEY ({0})'.format(pk_string))
+
+ return "kudu.Schema {{{0}\n}}".format(util.indent(buf.getvalue(), 2))
+
+ def __len__(self):
+ return self.schema.num_columns()
+
+ def __getitem__(self, key):
+ if isinstance(key, six.string_types):
+ key = self.get_loc(key)
+
+ if key < 0:
+ key += len(self)
+ return self.at(key)
+
+ def equals(self, Schema other):
+ """
+ Returns True if the table schemas are equal
+ """
+ return self.schema.Equals(deref(other.schema))
+
+ cdef int get_loc(self, name) except -1:
+ if not self._mapping_initialized:
+ for i in range(self.schema.num_columns()):
+ self._col_mapping[self.schema.Column(i).name()] = i
+ self._mapping_initialized = 1
+
+ name = tobytes(name)
+
+ # TODO: std::map is slightly verbose and inefficient here (O(lg n)
+ # lookups), may consider replacing with a better / different hash table
+ # should it become a performance bottleneck
+ cdef map[string, int].iterator it = self._col_mapping.find(name)
+ if it == self._col_mapping.end():
+ raise KeyError(name)
+ return self._col_mapping[name]
+
+ def at(self, size_t i):
+ """
+ Return the ColumnSchema for a column index. Analogous to schema[i].
+
+ Returns
+ -------
+ col_schema : ColumnSchema
+ """
+ cdef ColumnSchema result = ColumnSchema()
+
+ if i < 0 or i >= self.schema.num_columns():
+ raise IndexError('Column index {0} is not in range'
+ .format(i))
+
+ result.schema = new KuduColumnSchema(self.schema.Column(i))
+
+ return result
+
+ def primary_key_indices(self):
+ """
+ Return the indices of the columns used as primary keys
+
+ Returns
+ -------
+ key_indices : list[int]
+ """
+ cdef:
+ vector[int] indices
+ size_t i
+
+ self.schema.GetPrimaryKeyColumnIndexes(&indices)
+
+ result = []
+ for i in range(indices.size()):
+ result.append(indices[i])
+ return result
+
+ def primary_keys(self):
+ """
+ Return the names of the columns used as primary keys
+
+ Returns
+ -------
+ key_names : list[str]
+ """
+ indices = self.primary_key_indices()
+ return [self.at(i).name for i in indices]
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/tests/common.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/common.py b/python/kudu/tests/common.py
new file mode 100644
index 0000000..a944d7d
--- /dev/null
+++ b/python/kudu/tests/common.py
@@ -0,0 +1,147 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import division
+
+import json
+import fnmatch
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+
+import kudu
+
+
+class KuduTestBase(object):
+
+ """
+ Base test class that will start a configurable number of master and
+ tablet servers.
+ """
+
+ BASE_PORT = 37000
+ NUM_TABLET_SERVERS = 3
+
+ @classmethod
+ def start_cluster(cls):
+ local_path = tempfile.mkdtemp(dir=os.getenv("TEST_TMPDIR", None))
+ bin_path = "{0}/build/latest".format(os.getenv("KUDU_HOME"))
+
+ os.makedirs("{0}/master/".format(local_path))
+ os.makedirs("{0}/master/data".format(local_path))
+ os.makedirs("{0}/master/logs".format(local_path))
+
+ path = [
+ "{0}/kudu-master".format(bin_path),
+ "-rpc_server_allow_ephemeral_ports",
+ "-rpc_bind_addresses=0.0.0.0:0",
+ "-fs_wal_dir={0}/master/data".format(local_path),
+ "-fs_data_dirs={0}/master/data".format(local_path),
+ "-log_dir={0}/master/logs".format(local_path),
+ "-logtostderr",
+ "-webserver_port=0",
+ "-server_dump_info_path={0}/master/config.json".format(local_path)
+ ]
+
+ p = subprocess.Popen(path, shell=False)
+ fid = open("{0}/master/kudu-master.pid".format(local_path), "w+")
+ fid.write("{0}".format(p.pid))
+ fid.close()
+
+ # We have to wait for the master to settle before the config file
+ # appears
+ config_file = "{0}/master/config.json".format(local_path)
+ for i in range(30):
+ if os.path.exists(config_file):
+ break
+ time.sleep(0.1 * (i + 1))
+ else:
+ raise Exception("Could not find kudu-master config file")
+
+ # If the server was started get the bind port from the config dump
+ master_config = json.load(open("{0}/master/config.json"
+ .format(local_path), "r"))
+ # One master bound on local host
+ master_port = master_config["bound_rpc_addresses"][0]["port"]
+
+ for m in range(cls.NUM_TABLET_SERVERS):
+ os.makedirs("{0}/ts/{1}".format(local_path, m))
+ os.makedirs("{0}/ts/{1}/logs".format(local_path, m))
+
+ path = [
+ "{0}/kudu-tserver".format(bin_path),
+ "-rpc_server_allow_ephemeral_ports",
+ "-rpc_bind_addresses=0.0.0.0:0",
+ "-tserver_master_addrs=127.0.0.1:{0}".format(master_port),
+ "-webserver_port=0",
+ "-log_dir={0}/master/logs".format(local_path),
+ "-logtostderr",
+ "-fs_data_dirs={0}/ts/{1}/data".format(local_path, m),
+ "-fs_wal_dir={0}/ts/{1}/data".format(local_path, m),
+ ]
+ p = subprocess.Popen(path, shell=False)
+ tserver_pid = "{0}/ts/{1}/kudu-tserver.pid".format(local_path, m)
+ fid = open(tserver_pid, "w+")
+ fid.write("{0}".format(p.pid))
+ fid.close()
+
+ return local_path, master_port
+
+ @classmethod
+ def stop_cluster(cls, path):
+ for root, dirnames, filenames in os.walk('{0}/..'.format(path)):
+ for filename in fnmatch.filter(filenames, '*.pid'):
+ with open(os.path.join(root, filename)) as fid:
+ a = fid.read()
+ r = subprocess.Popen(["kill", "{0}".format(a)])
+ r.wait()
+ os.remove(os.path.join(root, filename))
+ shutil.rmtree(path, True)
+
+ @classmethod
+ def setUpClass(cls):
+ cls.cluster_path, master_port = cls.start_cluster()
+ time.sleep(1)
+
+ cls.master_host = '127.0.0.1'
+ cls.master_port = master_port
+
+ cls.client = kudu.connect(cls.master_host, cls.master_port)
+
+ cls.schema = cls.example_schema()
+
+ cls.ex_table = 'example-table'
+ if cls.client.table_exists(cls.ex_table):
+ cls.client.delete_table(cls.ex_table)
+ cls.client.create_table(cls.ex_table, cls.schema)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.stop_cluster(cls.cluster_path)
+
+ @classmethod
+ def example_schema(cls):
+ builder = kudu.schema_builder()
+ builder.add_column('key', kudu.int32, nullable=False)
+ builder.add_column('int_val', kudu.int32)
+ builder.add_column('string_val', kudu.string)
+ builder.set_primary_keys(['key'])
+
+ return builder.build()
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/tests/test_client.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/test_client.py b/python/kudu/tests/test_client.py
new file mode 100644
index 0000000..4636b3f
--- /dev/null
+++ b/python/kudu/tests/test_client.py
@@ -0,0 +1,189 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from kudu.compat import unittest, long
+from kudu.tests.common import KuduTestBase
+import kudu
+
+
+class TestClient(KuduTestBase, unittest.TestCase):
+
+ def setUp(self):
+ pass
+
+ def test_table_basics(self):
+ table = self.client.table(self.ex_table)
+
+ self.assertEqual(table.name, self.ex_table)
+ self.assertEqual(table.num_columns, len(self.schema))
+
+ def test_table_column(self):
+ table = self.client.table(self.ex_table)
+ col = table['key']
+
+ assert col.name == b'key'
+ assert col.parent is table
+
+ result_repr = repr(col)
+ expected_repr = ('Column(key, parent={0}, type=int32)'
+ .format(self.ex_table))
+ assert result_repr == expected_repr
+
+ def test_table_schema_retains_reference(self):
+ import gc
+
+ table = self.client.table(self.ex_table)
+ schema = table.schema
+ table = None
+
+ gc.collect()
+ repr(schema)
+
+ def test_table_exists(self):
+ self.assertFalse(self.client.table_exists('nonexistent-table'))
+ self.assertTrue(self.client.table_exists(self.ex_table))
+
+ def test_list_tables(self):
+ schema = self.example_schema()
+
+ to_create = ['foo1', 'foo2', 'foo3']
+ for name in to_create:
+ self.client.create_table(name, schema)
+
+ result = self.client.list_tables()
+ expected = [self.ex_table] + to_create
+ assert sorted(result) == expected
+
+ result = self.client.list_tables('foo')
+ assert sorted(result) == to_create
+
+ for name in to_create:
+ self.client.delete_table(name)
+
+ def test_is_multimaster(self):
+ assert not self.client.is_multimaster
+
+ def test_delete_table(self):
+ name = "peekaboo"
+ self.client.create_table(name, self.schema)
+ self.client.delete_table(name)
+ assert not self.client.table_exists(name)
+
+ # Should raise a more meaningful exception at some point
+ with self.assertRaises(kudu.KuduNotFound):
+ self.client.delete_table(name)
+
+ def test_table_nonexistent(self):
+ self.assertRaises(kudu.KuduNotFound, self.client.table,
+ '__donotexist__')
+
+ def test_insert_nonexistent_field(self):
+ table = self.client.table(self.ex_table)
+ op = table.new_insert()
+ self.assertRaises(KeyError, op.__setitem__, 'doesntexist', 12)
+
+ def test_insert_rows_and_delete(self):
+ nrows = 100
+ table = self.client.table(self.ex_table)
+ session = self.client.new_session()
+ for i in range(nrows):
+ op = table.new_insert()
+ op['key'] = i
+ op['int_val'] = i * 2
+ op['string_val'] = 'hello_%d' % i
+ session.apply(op)
+
+ # Cannot apply the same insert twice, C++ client does not indicate an
+ # error
+ self.assertRaises(Exception, session.apply, op)
+
+ # synchronous
+ session.flush()
+
+ scanner = table.scanner().open()
+ assert len(scanner.read_all_tuples()) == nrows
+
+ # Delete the rows we just wrote
+ for i in range(nrows):
+ op = table.new_delete()
+ op['key'] = i
+ session.apply(op)
+ session.flush()
+
+ scanner = table.scanner().open()
+ assert len(scanner.read_all_tuples()) == 0
+
+ def test_session_auto_open(self):
+ table = self.client.table(self.ex_table)
+ scanner = table.scanner()
+ result = scanner.read_all_tuples()
+ assert len(result) == 0
+
+ def test_session_open_idempotent(self):
+ table = self.client.table(self.ex_table)
+ scanner = table.scanner().open().open()
+ result = scanner.read_all_tuples()
+ assert len(result) == 0
+
+ def test_session_flush_modes(self):
+ self.client.new_session(flush_mode=kudu.FLUSH_MANUAL)
+ self.client.new_session(flush_mode=kudu.FLUSH_AUTO_SYNC)
+
+ self.client.new_session(flush_mode='manual')
+ self.client.new_session(flush_mode='sync')
+
+ with self.assertRaises(kudu.KuduNotSupported):
+ self.client.new_session(flush_mode=kudu.FLUSH_AUTO_BACKGROUND)
+
+ with self.assertRaises(kudu.KuduNotSupported):
+ self.client.new_session(flush_mode='background')
+
+ with self.assertRaises(ValueError):
+ self.client.new_session(flush_mode='foo')
+
+ def test_connect_timeouts(self):
+ # it works! any other way to check
+ kudu.connect(self.master_host, self.master_port,
+ admin_timeout_ms=100,
+ rpc_timeout_ms=100)
+
+ def test_capture_kudu_error(self):
+ pass
+
+
+class TestMonoDelta(unittest.TestCase):
+
+ def test_empty_ctor(self):
+ delta = kudu.TimeDelta()
+ assert repr(delta) == 'kudu.TimeDelta()'
+
+ def test_static_ctors(self):
+ delta = kudu.timedelta(3.5)
+ assert delta.to_seconds() == 3.5
+
+ delta = kudu.timedelta(millis=3500)
+ assert delta.to_millis() == 3500
+
+ delta = kudu.timedelta(micros=3500)
+ assert delta.to_micros() == 3500
+
+ delta = kudu.timedelta(micros=1000)
+ assert delta.to_nanos() == long(1000000)
+
+ delta = kudu.timedelta(nanos=3500)
+ assert delta.to_nanos() == 3500
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/tests/test_kudu.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/test_kudu.py b/python/kudu/tests/test_kudu.py
deleted file mode 100644
index f74caee..0000000
--- a/python/kudu/tests/test_kudu.py
+++ /dev/null
@@ -1,325 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import division
-
-import json
-import fnmatch
-import nose
-import os
-import shutil
-import subprocess
-import tempfile
-import time
-import unittest
-import signal
-
-import kudu
-
-class KuduBasicsBase(object):
- """Base test class that will start a configurable number of master and tablet
- servers."""
-
- BASE_PORT = 37000
- NUM_TABLET_SERVERS = 3
-
- @classmethod
- def start_cluster(cls):
- local_path = tempfile.mkdtemp(dir=os.getenv("TEST_TMPDIR", None))
- bin_path="{0}/build/latest".format(os.getenv("KUDU_HOME"))
-
- os.makedirs("{0}/master/".format(local_path))
- os.makedirs("{0}/master/data".format(local_path))
- os.makedirs("{0}/master/logs".format(local_path))
-
- path = ["{0}/kudu-master".format(bin_path),
- "-rpc_server_allow_ephemeral_ports",
- "-rpc_bind_addresses=0.0.0.0:0",
- "-fs_wal_dir={0}/master/data".format(local_path),
- "-fs_data_dirs={0}/master/data".format(local_path),
- "-log_dir={0}/master/logs".format(local_path),
- "-logtostderr",
- "-webserver_port=0",
- "-server_dump_info_path={0}/master/config.json".format(local_path)
- ]
-
- p = subprocess.Popen(path, shell=False)
- fid = open("{0}/master/kudu-master.pid".format(local_path), "w+")
- fid.write("{0}".format(p.pid))
- fid.close()
-
- # We have to wait for the master to settle before the config file appears
- config_file = "{0}/master/config.json".format(local_path)
- for _ in range(30):
- if os.path.exists(config_file):
- break
- time.sleep(1)
- else:
- raise Exception("Could not find kudu-master config file")
-
- # If the server was started get the bind port from the config dump
- master_config = json.load(open("{0}/master/config.json".format(local_path), "r"))
- # One master bound on local host
- master_port = master_config["bound_rpc_addresses"][0]["port"]
-
- for m in range(cls.NUM_TABLET_SERVERS):
- os.makedirs("{0}/ts/{1}".format(local_path, m))
- os.makedirs("{0}/ts/{1}/logs".format(local_path, m))
-
- path = ["{0}/kudu-tserver".format(bin_path),
- "-rpc_server_allow_ephemeral_ports",
- "-rpc_bind_addresses=0.0.0.0:0",
- "-tserver_master_addrs=127.0.0.1:{0}".format(master_port),
- "-webserver_port=0",
- "-log_dir={0}/master/logs".format(local_path),
- "-logtostderr",
- "-fs_data_dirs={0}/ts/{1}/data".format(local_path, m),
- "-fs_wal_dir={0}/ts/{1}/data".format(local_path, m),
- ]
- p = subprocess.Popen(path, shell=False)
- fid = open("{0}/ts/{1}/kudu-tserver.pid".format(local_path, m), "w+")
- fid.write("{0}".format(p.pid))
- fid.close()
-
- return local_path, master_port
-
- @classmethod
- def stop_cluster(cls, path):
- for root, dirnames, filenames in os.walk('{0}/..'.format(path)):
- for filename in fnmatch.filter(filenames, '*.pid'):
- with open(os.path.join(root, filename)) as fid:
- a = fid.read()
- r = subprocess.Popen(["kill", "{0}".format(a)])
- r.wait()
- os.remove(os.path.join(root, filename))
- shutil.rmtree(path, True)
-
- @classmethod
- def setUpClass(cls):
- cls.cluster_path, master_port = cls.start_cluster()
- time.sleep(1)
- cls.client = kudu.Client('127.0.0.1:{0}'.format(master_port))
-
- cls.schema = cls.example_schema()
-
- cls.ex_table = 'example-table'
- if cls.client.table_exists(cls.ex_table):
- cls.client.delete_table(cls.ex_table)
- cls.client.create_table(cls.ex_table, cls.schema)
-
- @classmethod
- def tearDownClass(cls):
- cls.stop_cluster(cls.cluster_path)
-
- @classmethod
- def example_schema(cls):
- col1 = kudu.ColumnSchema.create('key', kudu.INT32)
- col2 = kudu.ColumnSchema.create('int_val', kudu.INT32)
- col3 = kudu.ColumnSchema.create('string_val', kudu.STRING)
-
- return kudu.schema_from_list([col1, col2, col3], 1)
-
-
-class TestSchema(unittest.TestCase):
-
- def test_column_schema(self):
- pass
-
- def test_create_schema(self):
- col1 = kudu.ColumnSchema.create('key', kudu.INT32)
- col2 = kudu.ColumnSchema.create('int_val', kudu.INT32)
- col3 = kudu.ColumnSchema.create('string_val', kudu.STRING)
-
- cols = [col1, col2, col3]
-
- # One key column
- schema = kudu.schema_from_list(cols, 1)
- self.assertEqual(len(schema), 3)
-
- # Question whether we want to go the overloading route
- self.assertTrue(schema.at(0).equals(col1))
- self.assertTrue(schema.at(1).equals(col2))
- self.assertTrue(schema.at(2).equals(col3))
-
- # This isn't yet very easy
- # self.assertEqual(schema['key'], col1)
- # self.assertEqual(schema['int_val'], col2)
- # self.assertEqual(schema['string_val'], col3)
-
- def test_column_schema_repr(self):
- col1 = kudu.ColumnSchema.create('key', kudu.INT32)
-
- result = repr(col1)
- expected = 'ColumnSchema(name=key, type=int32, nullable=False)'
- self.assertEqual(result, expected)
-
- def test_column_schema_default_value(self):
- pass
-
-
-class TestTable(KuduBasicsBase, unittest.TestCase):
-
- def setUp(self):
- pass
-
- def test_table_basics(self):
- table = self.client.open_table(self.ex_table)
-
- self.assertEqual(table.name, self.ex_table)
- self.assertEqual(table.num_columns, len(self.schema))
-
- def test_table_exists(self):
- self.assertFalse(self.client.table_exists('nonexistent-table'))
- self.assertTrue(self.client.table_exists(self.ex_table))
-
- def test_delete_table(self):
- name = "peekaboo"
- self.client.create_table(name, self.schema)
- self.assertTrue(self.client.delete_table(name))
- self.assertFalse(self.client.table_exists(name))
-
- # Should raise a more meaningful exception at some point
- val = self.client.delete_table(name)
- self.assertFalse(val)
-
- def test_open_table_nonexistent(self):
- self.assertRaises(kudu.KuduException, self.client.open_table,
- '__donotexist__')
-
- def test_insert_nonexistent_field(self):
- table = self.client.open_table(self.ex_table)
- op = table.insert()
- self.assertRaises(KeyError, op.__setitem__, 'doesntexist', 12)
-
- def test_insert_rows_and_delete(self):
- nrows = 100
- table = self.client.open_table(self.ex_table)
- session = self.client.new_session()
- for i in range(nrows):
- op = table.insert()
- op['key'] = i
- op['int_val'] = i * 2
- op['string_val'] = 'hello_%d' % i
- session.apply(op)
-
- # Cannot apply the same insert twice, does not blow up in C++
- self.assertRaises(Exception, session.apply, op)
-
- # synchronous
- self.assertTrue(session.flush())
-
- # Delete the rows we just wrote
- for i in range(nrows):
- op = table.delete()
- op['key'] = i
- session.apply(op)
- session.flush()
- # TODO: verify the table is now empty
-
- def test_capture_kudu_error(self):
- pass
-
-
-class TestScanner(KuduBasicsBase, unittest.TestCase):
-
- @classmethod
- def setUpClass(cls):
- super(TestScanner, cls).setUpClass()
-
- cls.nrows = 100
- table = cls.client.open_table(cls.ex_table)
- session = cls.client.new_session()
-
- tuples = []
- for i in range(cls.nrows):
- op = table.insert()
- tup = i, i * 2, 'hello_%d' % i
- op['key'] = tup[0]
- op['int_val'] = tup[1]
- op['string_val'] = tup[2]
- session.apply(op)
- tuples.append(tup)
- session.flush()
-
- cls.table = table
- cls.tuples = tuples
-
- @classmethod
- def tearDownClass(cls):
- pass
-
- def setUp(self):
- pass
-
- def test_scan_rows_basic(self):
- # Let's scan with no predicates
- scanner = self.table.scanner().open()
-
- batch = scanner.read_all()
- self.assertEqual(len(batch), self.nrows)
-
- result_tuples = batch.as_tuples()
- self.assertEqual(result_tuples, self.tuples)
-
- def test_scan_rows_simple_predicate(self):
- scanner = self.table.scanner()
- scanner.add_comparison_predicate("key", kudu.GREATER_EQUAL, 20)
- scanner.add_comparison_predicate("key", kudu.LESS_EQUAL, 49)
- scanner.open()
-
- batch = scanner.read_all()
- tuples = batch.as_tuples()
-
- self.assertEqual(tuples, self.tuples[20:50])
-
- def test_scan_rows_string_predicate(self):
- scanner = self.table.scanner()
-
- scanner.add_comparison_predicate("string_val", kudu.GREATER_EQUAL, "hello_20")
- scanner.add_comparison_predicate("string_val", kudu.LESS_EQUAL, "hello_25")
- scanner.open()
-
- batch = scanner.read_all()
- tuples = batch.as_tuples()
-
- self.assertEqual(tuples, self.tuples[20:26])
-
- def test_scan_invalid_predicates(self):
- scanner = self.table.scanner()
- try:
- scanner.add_comparison_predicate("foo", kudu.GREATER_EQUAL, "x")
- except Exception, e:
- self.assertEqual("Not found: column not found: foo", str(e))
-
- try:
- scanner.add_comparison_predicate("string_val", kudu.GREATER_EQUAL, 1)
- except Exception, e:
- self.assertEqual("Invalid argument: non-string value " +
- "for string column string_val", str(e))
-
- try:
- scanner.add_comparison_predicate("string_val", kudu.GREATER_EQUAL, None)
- except Exception, e:
- self.assertEqual("unable to convert python type <type 'NoneType'>", str(e))
-
-
-if __name__ == '__main__':
- nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb',
- '--pdb-failure', '-s'], exit=False)
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/tests/test_scanner.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py
new file mode 100644
index 0000000..86decb1
--- /dev/null
+++ b/python/kudu/tests/test_scanner.py
@@ -0,0 +1,102 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import division
+
+from kudu.compat import unittest
+from kudu.tests.common import KuduTestBase
+import kudu
+
+
+class TestScanner(KuduTestBase, unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ super(TestScanner, cls).setUpClass()
+
+ cls.nrows = 100
+ table = cls.client.table(cls.ex_table)
+ session = cls.client.new_session()
+
+ tuples = []
+ for i in range(cls.nrows):
+ op = table.new_insert()
+ tup = i, i * 2, 'hello_%d' % i
+ op['key'] = tup[0]
+ op['int_val'] = tup[1]
+ op['string_val'] = tup[2]
+ session.apply(op)
+ tuples.append(tup)
+ session.flush()
+
+ cls.table = table
+ cls.tuples = tuples
+
+ @classmethod
+ def tearDownClass(cls):
+ pass
+
+ def setUp(self):
+ pass
+
+ def test_scan_rows_basic(self):
+ # Let's scan with no predicates
+ scanner = self.table.scanner().open()
+
+ tuples = scanner.read_all_tuples()
+ self.assertEqual(sorted(tuples), self.tuples)
+
+ def test_scan_rows_simple_predicate(self):
+ key = self.table['key']
+ preds = [key >= 20, key <= 49]
+
+ def _read_predicates(preds):
+ scanner = self.table.scanner()
+ scanner.add_predicates(preds)
+ scanner.open()
+ return scanner.read_all_tuples()
+
+ tuples = _read_predicates(preds)
+ self.assertEqual(sorted(tuples), self.tuples[20:50])
+
+ # verify predicates reusable
+ tuples = _read_predicates(preds)
+ self.assertEqual(sorted(tuples), self.tuples[20:50])
+
+ def test_scan_rows_string_predicate(self):
+ scanner = self.table.scanner()
+
+ sv = self.table['string_val']
+
+ scanner.add_predicates([sv >= 'hello_20',
+ sv <= 'hello_25'])
+ scanner.open()
+
+ tuples = scanner.read_all_tuples()
+
+ self.assertEqual(sorted(tuples), self.tuples[20:26])
+
+ def test_scan_invalid_predicates(self):
+ scanner = self.table.scanner()
+ sv = self.table['string_val']
+
+ with self.assertRaises(TypeError):
+ scanner.add_predicates([sv >= None])
+
+ with self.assertRaises(kudu.KuduInvalidArgument):
+ scanner.add_predicates([sv >= 1])
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/tests/test_schema.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/test_schema.py b/python/kudu/tests/test_schema.py
new file mode 100644
index 0000000..b75c61d
--- /dev/null
+++ b/python/kudu/tests/test_schema.py
@@ -0,0 +1,182 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import division
+
+from kudu.compat import unittest
+import kudu
+
+
+class TestSchema(unittest.TestCase):
+
+ def setUp(self):
+ self.columns = [('one', 'int32', False),
+ ('two', 'int8', False),
+ ('three', 'double', True),
+ ('four', 'string', False)]
+
+ self.primary_keys = ['one', 'two']
+
+ self.builder = kudu.schema_builder()
+ for name, typename, nullable in self.columns:
+ self.builder.add_column(name, typename, nullable=nullable)
+
+ self.builder.set_primary_keys(self.primary_keys)
+ self.schema = self.builder.build()
+
+ def test_repr(self):
+ result = repr(self.schema)
+ for name, _, _ in self.columns:
+ assert name in result
+
+ assert 'PRIMARY KEY (one, two)' in result
+
+ def test_schema_length(self):
+ assert len(self.schema) == 4
+
+ def test_names(self):
+ assert self.schema.names == ['one', 'two', 'three', 'four']
+
+ def test_primary_keys(self):
+ assert self.schema.primary_key_indices() == [0, 1]
+ assert self.schema.primary_keys() == ['one', 'two']
+
+ def test_getitem_boundschecking(self):
+ with self.assertRaises(IndexError):
+ self.schema[4]
+
+ def test_getitem_wraparound(self):
+ # wraparound
+ result = self.schema[-1]
+ expected = self.schema[3]
+
+ assert result.equals(expected)
+
+ def test_getitem_string(self):
+ result = self.schema['three']
+ expected = self.schema[2]
+
+ assert result.equals(expected)
+
+ with self.assertRaises(KeyError):
+ self.schema['not_found']
+
+ def test_schema_equals(self):
+ assert self.schema.equals(self.schema)
+
+ builder = kudu.schema_builder()
+ builder.add_column('key', 'int64', nullable=False, primary_key=True)
+ schema = builder.build()
+
+ assert not self.schema.equals(schema)
+
+ def test_column_equals(self):
+ assert not self.schema[0].equals(self.schema[1])
+
+ def test_type(self):
+ builder = kudu.schema_builder()
+ (builder.add_column('key')
+ .type('int32')
+ .primary_key()
+ .nullable(False))
+ schema = builder.build()
+
+ tp = schema[0].type
+ assert tp.name == 'int32'
+ assert tp.type == kudu.schema.INT32
+
+ def test_compression(self):
+ builder = kudu.schema_builder()
+ builder.add_column('key', 'int64', nullable=False)
+
+ foo = builder.add_column('foo', 'string').compression('lz4')
+ assert foo is not None
+
+ bar = builder.add_column('bar', 'string')
+ bar.compression(kudu.COMPRESSION_ZLIB)
+
+ with self.assertRaises(ValueError):
+ bar = builder.add_column('qux', 'string', compression='unknown')
+
+ builder.set_primary_keys(['key'])
+ builder.build()
+
+ # TODO; The C++ client does not give us an API to see the storage
+ # attributes of a column
+
+ def test_encoding(self):
+ builder = kudu.schema_builder()
+ builder.add_column('key', 'int64', nullable=False)
+
+ foo = builder.add_column('foo', 'string').encoding('rle')
+ assert foo is not None
+
+ bar = builder.add_column('bar', 'string')
+ bar.encoding(kudu.ENCODING_PLAIN)
+
+ with self.assertRaises(ValueError):
+ builder.add_column('qux', 'string', encoding='unknown')
+
+ builder.set_primary_keys(['key'])
+ builder.build()
+ # TODO(wesm): The C++ client does not give us an API to see the storage
+ # attributes of a column
+
+ def test_set_column_spec_pk(self):
+ builder = kudu.schema_builder()
+ key = (builder.add_column('key', 'int64', nullable=False)
+ .primary_key())
+ assert key is not None
+ schema = builder.build()
+ assert 'key' in schema.primary_keys()
+
+ builder = kudu.schema_builder()
+ key = (builder.add_column('key', 'int64', nullable=False,
+ primary_key=True))
+ schema = builder.build()
+ assert 'key' in schema.primary_keys()
+
+ def test_partition_schema(self):
+ pass
+
+ def test_nullable_not_null(self):
+ builder = kudu.schema_builder()
+ (builder.add_column('key', 'int64', nullable=False)
+ .primary_key())
+
+ builder.add_column('data1', 'double').nullable(True)
+ builder.add_column('data2', 'double').nullable(False)
+ builder.add_column('data3', 'double', nullable=True)
+ builder.add_column('data4', 'double', nullable=False)
+
+ schema = builder.build()
+
+ assert not schema[0].nullable
+ assert schema[1].nullable
+ assert not schema[2].nullable
+
+ assert schema[3].nullable
+ assert not schema[4].nullable
+
+ def test_default_value(self):
+ pass
+
+ def test_column_schema_repr(self):
+ result = repr(self.schema[0])
+ expected = 'ColumnSchema(name=one, type=int32, nullable=False)'
+ self.assertEqual(result, expected)
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/kudu/util.py
----------------------------------------------------------------------
diff --git a/python/kudu/util.py b/python/kudu/util.py
new file mode 100644
index 0000000..a2b65cf
--- /dev/null
+++ b/python/kudu/util.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+def indent(text, spaces):
+ block = ' ' * spaces
+ return '\n'.join(block + x for x in text.split('\n'))
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/requirements.txt
----------------------------------------------------------------------
diff --git a/python/requirements.txt b/python/requirements.txt
index 55a8b18..ef646fa 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,3 +1,6 @@
+pytest
+numpy>=1.7.0
cython >= 0.21
-nose >= 1.0
setuptools >= 0.8
+six
+unittest2
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/setup.cfg
----------------------------------------------------------------------
diff --git a/python/setup.cfg b/python/setup.cfg
new file mode 100644
index 0000000..9af7e6f
--- /dev/null
+++ b/python/setup.cfg
@@ -0,0 +1,2 @@
+[aliases]
+test=pytest
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53f976f0/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 9c7d7e9..4a9828f 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -22,24 +22,45 @@ from Cython.Distutils import build_ext
from Cython.Build import cythonize
import Cython
-if Cython.__version__ < '0.19.1':
- raise Exception('Please upgrade to Cython 0.19.1 or newer')
import sys
from setuptools import setup
+from distutils.command.clean import clean as _clean
from distutils.extension import Extension
import os
+if Cython.__version__ < '0.19.1':
+ raise Exception('Please upgrade to Cython 0.19.1 or newer')
+
MAJOR = 0
-MINOR = 0
-MICRO = 1
+MINOR = 1
+MICRO = 0
VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)
+ISRELEASED = True
+
+setup_dir = os.path.abspath(os.path.dirname(__file__))
+
+
+def write_version_py(filename=os.path.join(setup_dir, 'kudu/version.py')):
+ version = VERSION
+ if not ISRELEASED:
+ version += '.dev'
+
+ a = open(filename, 'w')
+ file_content = "\n".join(["",
+ "# THIS FILE IS GENERATED FROM SETUP.PY",
+ "version = '%(version)s'",
+ "isrelease = '%(isrelease)s'"])
+
+ a.write(file_content % {'version': VERSION,
+ 'isrelease': str(ISRELEASED)})
+ a.close()
-from distutils.command.clean import clean as _clean
class clean(_clean):
def run(self):
_clean.run(self)
- for x in ['kudu/client.cpp']:
+ for x in ['kudu/client.cpp', 'kudu/schema.cpp',
+ 'kudu/errors.cpp']:
try:
os.remove(x)
except OSError:
@@ -48,20 +69,22 @@ class clean(_clean):
# If we're in the context of the Kudu git repository, build against the
# latest in-tree build artifacts
-if 'KUDU_HOME' in os.environ and \
- os.path.exists(os.path.join(os.environ['KUDU_HOME'], "build/latest")):
- print >>sys.stderr, "Building from in-tree build artifacts"
+if ('KUDU_HOME' in os.environ and
+ os.path.exists(os.path.join(os.environ['KUDU_HOME'],
+ "build/latest"))):
+ sys.stderr.write("Building from in-tree build artifacts\n")
kudu_include_dir = os.path.join(os.environ['KUDU_HOME'], 'src')
- kudu_lib_dir = os.path.join(os.environ['KUDU_HOME'], 'build/latest/exported')
+ kudu_lib_dir = os.path.join(os.environ['KUDU_HOME'],
+ 'build/latest/exported')
else:
if os.path.exists("/usr/local/include/kudu"):
- prefix="/usr/local"
+ prefix = "/usr/local"
elif os.path.exists("/usr/include/kudu"):
- prefix="/usr"
+ prefix = "/usr"
else:
- print >>sys.stderr, "Cannot find installed kudu client."
+ sys.stderr.write("Cannot find installed kudu client.\n")
sys.exit(1)
- print >>sys.stderr, "Building from system prefix ", prefix
+ sys.stderr.write("Building from system prefix {0}\n".format(prefix))
kudu_include_dir = prefix + "/include"
kudu_lib_dir = prefix + "/lib"
@@ -69,32 +92,59 @@ INCLUDE_PATHS = [kudu_include_dir]
LIBRARY_DIRS = [kudu_lib_dir]
RT_LIBRARY_DIRS = LIBRARY_DIRS
-client_ext = Extension('kudu.client', ['kudu/client.pyx'],
- libraries=['kudu_client'],
- # Disable the 'new' gcc5 ABI; see the top-level
- # CMakeLists.txt for details.
- define_macros=[('_GLIBCXX_USE_CXX11_ABI', '0')],
- include_dirs=INCLUDE_PATHS,
- library_dirs=LIBRARY_DIRS,
- runtime_library_dirs=RT_LIBRARY_DIRS)
+ext_submodules = ['client', 'errors', 'schema']
+
+extensions = []
+
+for submodule_name in ext_submodules:
+ ext = Extension('kudu.{0}'.format(submodule_name),
+ ['kudu/{0}.pyx'.format(submodule_name)],
+ libraries=['kudu_client'],
+ # Disable the 'new' gcc5 ABI; see the top-level
+ # CMakeLists.txt for details.
+ define_macros=[('_GLIBCXX_USE_CXX11_ABI', '0')],
+ include_dirs=INCLUDE_PATHS,
+ library_dirs=LIBRARY_DIRS,
+ runtime_library_dirs=RT_LIBRARY_DIRS)
+ extensions.append(ext)
+
+extensions = cythonize(extensions)
+
+write_version_py()
+
+LONG_DESCRIPTION = open(os.path.join(setup_dir, "README.md")).read()
+DESCRIPTION = "Python interface to the Apache Kudu (incubating) C++ Client API"
-extensions = cythonize([client_ext])
+CLASSIFIERS = [
+ 'Development Status :: 3 - Alpha',
+ 'Environment :: Console',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Cython'
+]
setup(
- name="python-kudu",
- packages=["kudu"],
+ name="kudu-python",
+ packages=['kudu', 'kudu.tests'],
version=VERSION,
- setup_requires=['nose>=1.0'],
package_data={'kudu': ['*.pxd', '*.pyx']},
ext_modules=extensions,
- cmdclass = {
+ cmdclass={
'clean': clean,
'build_ext': build_ext
},
+ setup_requires=['pytest-runner'],
+ tests_require=['pytest'],
install_requires=['cython >= 0.21'],
- description="Cython wrapper for the Kudu C++ API",
- license='Proprietary',
- author="Wes McKinney",
- maintainer_email="wes@cloudera.com",
+ description=DESCRIPTION,
+ long_description=LONG_DESCRIPTION,
+ license='Apache License, Version 2.0',
+ classifiers=CLASSIFIERS,
+ author="Apache Kudu (incubating) team",
+ maintainer_email="dev@kudu.incubator.apache.org",
test_suite="kudu.tests"
)