You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/07/23 18:39:08 UTC
[01/14] arrow git commit: [C++] Remove Plasma source tree for 0.5.0
release pending IP Clearance
Repository: arrow
Updated Branches:
refs/heads/master db181d19b -> 2c8101515
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/xxhash.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/xxhash.cc b/cpp/src/plasma/thirdparty/xxhash.cc
deleted file mode 100644
index f74880b..0000000
--- a/cpp/src/plasma/thirdparty/xxhash.cc
+++ /dev/null
@@ -1,889 +0,0 @@
-/*
-* xxHash - Fast Hash algorithm
-* Copyright (C) 2012-2016, Yann Collet
-*
-* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions are
-* met:
-*
-* * Redistributions of source code must retain the above copyright
-* notice, this list of conditions and the following disclaimer.
-* * Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following disclaimer
-* in the documentation and/or other materials provided with the
-* distribution.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*
-* You can contact the author at :
-* - xxHash homepage: http://www.xxhash.com
-* - xxHash source repository : https://github.com/Cyan4973/xxHash
-*/
-
-
-/* *************************************
-* Tuning parameters
-***************************************/
-/*!XXH_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
- * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
- * It can generate buggy code on targets which do not support unaligned memory accesses.
- * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://stackoverflow.com/a/32095106/646947 for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
-# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
-# define XXH_FORCE_MEMORY_ACCESS 2
-# elif defined(__INTEL_COMPILER) || \
- (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
-# define XXH_FORCE_MEMORY_ACCESS 1
-# endif
-#endif
-
-/*!XXH_ACCEPT_NULL_INPUT_POINTER :
- * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
- * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
- * By default, this option is disabled. To enable it, uncomment below define :
- */
-/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
-
-/*!XXH_FORCE_NATIVE_FORMAT :
- * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
- * Results are therefore identical for little-endian and big-endian CPU.
- * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
- * Should endian-independence be of no importance for your application, you may set the #define below to 1,
- * to improve speed for Big-endian CPU.
- * This option has no impact on Little_Endian CPU.
- */
-#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */
-# define XXH_FORCE_NATIVE_FORMAT 0
-#endif
-
-/*!XXH_FORCE_ALIGN_CHECK :
- * This is a minor performance trick, only useful with lots of very small keys.
- * It means : check for aligned/unaligned input.
- * The check costs one initial branch per hash; set to 0 when the input data
- * is guaranteed to be aligned.
- */
-#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
-# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
-# define XXH_FORCE_ALIGN_CHECK 0
-# else
-# define XXH_FORCE_ALIGN_CHECK 1
-# endif
-#endif
-
-
-/* *************************************
-* Includes & Memory related functions
-***************************************/
-/*! Modify the local functions below should you wish to use some other memory routines
-* for malloc(), free() */
-#include <stdlib.h>
-static void* XXH_malloc(size_t s) { return malloc(s); }
-static void XXH_free (void* p) { free(p); }
-/*! and for memcpy() */
-#include <string.h>
-static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
-
-#define XXH_STATIC_LINKING_ONLY
-#include "xxhash.h"
-
-
-/* *************************************
-* Compiler Specific Options
-***************************************/
-#ifdef _MSC_VER /* Visual Studio */
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-# define FORCE_INLINE static __forceinline
-#else
-# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
-# else
-# define FORCE_INLINE static
-# endif /* __STDC_VERSION__ */
-#endif
-
-
-/* *************************************
-* Basic Types
-***************************************/
-#ifndef MEM_MODULE
-# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
-# else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
-# endif
-#endif
-
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
-
-/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
-static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
-
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U32 u32; } __attribute__((packed)) unalign;
-static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-
-#else
-
-/* portable and safe solution. Generally efficient.
- * see : http://stackoverflow.com/a/32095106/646947
- */
-static U32 XXH_read32(const void* memPtr)
-{
- U32 val;
- memcpy(&val, memPtr, sizeof(val));
- return val;
-}
-
-#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
-
-
-/* ****************************************
-* Compiler-specific Functions and Macros
-******************************************/
-#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
-/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
-#if defined(_MSC_VER)
-# define XXH_rotl32(x,r) _rotl(x,r)
-# define XXH_rotl64(x,r) _rotl64(x,r)
-#else
-# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
-#endif
-
-#if defined(_MSC_VER) /* Visual Studio */
-# define XXH_swap32 _byteswap_ulong
-#elif XXH_GCC_VERSION >= 403
-# define XXH_swap32 __builtin_bswap32
-#else
-static U32 XXH_swap32 (U32 x)
-{
- return ((x << 24) & 0xff000000 ) |
- ((x << 8) & 0x00ff0000 ) |
- ((x >> 8) & 0x0000ff00 ) |
- ((x >> 24) & 0x000000ff );
-}
-#endif
-
-
-/* *************************************
-* Architecture Macros
-***************************************/
-typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
-
-/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
-#ifndef XXH_CPU_LITTLE_ENDIAN
- static const int g_one = 1;
-# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))
-#endif
-
-
-/* ***************************
-* Memory reads
-*****************************/
-typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
-
-FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
-{
- if (align==XXH_unaligned)
- return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
- else
- return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
-}
-
-FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
-{
- return XXH_readLE32_align(ptr, endian, XXH_unaligned);
-}
-
-static U32 XXH_readBE32(const void* ptr)
-{
- return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
-}
-
-
-/* *************************************
-* Macros
-***************************************/
-#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
-XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
-
-
-/* *******************************************************************
-* 32-bits hash functions
-*********************************************************************/
-static const U32 PRIME32_1 = 2654435761U;
-static const U32 PRIME32_2 = 2246822519U;
-static const U32 PRIME32_3 = 3266489917U;
-static const U32 PRIME32_4 = 668265263U;
-static const U32 PRIME32_5 = 374761393U;
-
-static U32 XXH32_round(U32 seed, U32 input)
-{
- seed += input * PRIME32_2;
- seed = XXH_rotl32(seed, 13);
- seed *= PRIME32_1;
- return seed;
-}
-
-FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
-{
- const BYTE* p = (const BYTE*)input;
- const BYTE* bEnd = p + len;
- U32 h32;
-#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (p==NULL) {
- len=0;
- bEnd=p=(const BYTE*)(size_t)16;
- }
-#endif
-
- if (len>=16) {
- const BYTE* const limit = bEnd - 16;
- U32 v1 = seed + PRIME32_1 + PRIME32_2;
- U32 v2 = seed + PRIME32_2;
- U32 v3 = seed + 0;
- U32 v4 = seed - PRIME32_1;
-
- do {
- v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
- v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
- v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
- v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
- } while (p<=limit);
-
- h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
- } else {
- h32 = seed + PRIME32_5;
- }
-
- h32 += (U32) len;
-
- while (p+4<=bEnd) {
- h32 += XXH_get32bits(p) * PRIME32_3;
- h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
- p+=4;
- }
-
- while (p<bEnd) {
- h32 += (*p) * PRIME32_5;
- h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
- p++;
- }
-
- h32 ^= h32 >> 15;
- h32 *= PRIME32_2;
- h32 ^= h32 >> 13;
- h32 *= PRIME32_3;
- h32 ^= h32 >> 16;
-
- return h32;
-}
-
-
-XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
-{
-#if 0
- /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
- XXH32_state_t state;
- XXH32_reset(&state, seed);
- XXH32_update(&state, input, len);
- return XXH32_digest(&state);
-#else
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
- if (XXH_FORCE_ALIGN_CHECK) {
- if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
- else
- return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
- } }
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
- else
- return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
-#endif
-}
-
-
-
-/*====== Hash streaming ======*/
-
-XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
-{
- return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
-}
-XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
-{
- XXH_free(statePtr);
- return XXH_OK;
-}
-
-XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
-{
- memcpy(dstState, srcState, sizeof(*dstState));
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
-{
- XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
- memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */
- state.v1 = seed + PRIME32_1 + PRIME32_2;
- state.v2 = seed + PRIME32_2;
- state.v3 = seed + 0;
- state.v4 = seed - PRIME32_1;
- memcpy(statePtr, &state, sizeof(state));
- return XXH_OK;
-}
-
-
-FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
-{
- const BYTE* p = (const BYTE*)input;
- const BYTE* const bEnd = p + len;
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (input==NULL) return XXH_ERROR;
-#endif
-
- state->total_len_32 += (unsigned)len;
- state->large_len |= (len>=16) | (state->total_len_32>=16);
-
- if (state->memsize + len < 16) { /* fill in tmp buffer */
- XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
- state->memsize += (unsigned)len;
- return XXH_OK;
- }
-
- if (state->memsize) { /* some data left from previous update */
- XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
- { const U32* p32 = state->mem32;
- state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
- state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
- state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
- state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
- }
- p += 16-state->memsize;
- state->memsize = 0;
- }
-
- if (p <= bEnd-16) {
- const BYTE* const limit = bEnd - 16;
- U32 v1 = state->v1;
- U32 v2 = state->v2;
- U32 v3 = state->v3;
- U32 v4 = state->v4;
-
- do {
- v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
- v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
- v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
- v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
- } while (p<=limit);
-
- state->v1 = v1;
- state->v2 = v2;
- state->v3 = v3;
- state->v4 = v4;
- }
-
- if (p < bEnd) {
- XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
- state->memsize = (unsigned)(bEnd-p);
- }
-
- return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
-{
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
- else
- return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
-}
-
-
-
-FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
-{
- const BYTE * p = (const BYTE*)state->mem32;
- const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
- U32 h32;
-
- if (state->large_len) {
- h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
- } else {
- h32 = state->v3 /* == seed */ + PRIME32_5;
- }
-
- h32 += state->total_len_32;
-
- while (p+4<=bEnd) {
- h32 += XXH_readLE32(p, endian) * PRIME32_3;
- h32 = XXH_rotl32(h32, 17) * PRIME32_4;
- p+=4;
- }
-
- while (p<bEnd) {
- h32 += (*p) * PRIME32_5;
- h32 = XXH_rotl32(h32, 11) * PRIME32_1;
- p++;
- }
-
- h32 ^= h32 >> 15;
- h32 *= PRIME32_2;
- h32 ^= h32 >> 13;
- h32 *= PRIME32_3;
- h32 ^= h32 >> 16;
-
- return h32;
-}
-
-
-XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
-{
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_digest_endian(state_in, XXH_littleEndian);
- else
- return XXH32_digest_endian(state_in, XXH_bigEndian);
-}
-
-
-/*====== Canonical representation ======*/
-
-/*! Default XXH result types are basic unsigned 32 and 64 bits.
-* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
-* These functions allow transformation of hash result into and from its canonical format.
-* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
-*/
-
-XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
-{
- XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
- if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
- memcpy(dst, &hash, sizeof(*dst));
-}
-
-XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
-{
- return XXH_readBE32(src);
-}
-
-
-#ifndef XXH_NO_LONG_LONG
-
-/* *******************************************************************
-* 64-bits hash functions
-*********************************************************************/
-
-/*====== Memory access ======*/
-
-#ifndef MEM_MODULE
-# define MEM_MODULE
-# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include <stdint.h>
- typedef uint64_t U64;
-# else
- typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
-# endif
-#endif
-
-
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
-
-/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
-static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
-
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
-static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
-
-#else
-
-/* portable and safe solution. Generally efficient.
- * see : http://stackoverflow.com/a/32095106/646947
- */
-
-static U64 XXH_read64(const void* memPtr)
-{
- U64 val;
- memcpy(&val, memPtr, sizeof(val));
- return val;
-}
-
-#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
-
-#if defined(_MSC_VER) /* Visual Studio */
-# define XXH_swap64 _byteswap_uint64
-#elif XXH_GCC_VERSION >= 403
-# define XXH_swap64 __builtin_bswap64
-#else
-static U64 XXH_swap64 (U64 x)
-{
- return ((x << 56) & 0xff00000000000000ULL) |
- ((x << 40) & 0x00ff000000000000ULL) |
- ((x << 24) & 0x0000ff0000000000ULL) |
- ((x << 8) & 0x000000ff00000000ULL) |
- ((x >> 8) & 0x00000000ff000000ULL) |
- ((x >> 24) & 0x0000000000ff0000ULL) |
- ((x >> 40) & 0x000000000000ff00ULL) |
- ((x >> 56) & 0x00000000000000ffULL);
-}
-#endif
-
-FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
-{
- if (align==XXH_unaligned)
- return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
- else
- return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
-}
-
-FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
-{
- return XXH_readLE64_align(ptr, endian, XXH_unaligned);
-}
-
-static U64 XXH_readBE64(const void* ptr)
-{
- return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
-}
-
-
-/*====== xxh64 ======*/
-
-static const U64 PRIME64_1 = 11400714785074694791ULL;
-static const U64 PRIME64_2 = 14029467366897019727ULL;
-static const U64 PRIME64_3 = 1609587929392839161ULL;
-static const U64 PRIME64_4 = 9650029242287828579ULL;
-static const U64 PRIME64_5 = 2870177450012600261ULL;
-
-static U64 XXH64_round(U64 acc, U64 input)
-{
- acc += input * PRIME64_2;
- acc = XXH_rotl64(acc, 31);
- acc *= PRIME64_1;
- return acc;
-}
-
-static U64 XXH64_mergeRound(U64 acc, U64 val)
-{
- val = XXH64_round(0, val);
- acc ^= val;
- acc = acc * PRIME64_1 + PRIME64_4;
- return acc;
-}
-
-FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
-{
- const BYTE* p = (const BYTE*)input;
- const BYTE* bEnd = p + len;
- U64 h64;
-#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (p==NULL) {
- len=0;
- bEnd=p=(const BYTE*)(size_t)32;
- }
-#endif
-
- if (len>=32) {
- const BYTE* const limit = bEnd - 32;
- U64 v1 = seed + PRIME64_1 + PRIME64_2;
- U64 v2 = seed + PRIME64_2;
- U64 v3 = seed + 0;
- U64 v4 = seed - PRIME64_1;
-
- do {
- v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
- v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
- v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
- v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
- } while (p<=limit);
-
- h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
- h64 = XXH64_mergeRound(h64, v1);
- h64 = XXH64_mergeRound(h64, v2);
- h64 = XXH64_mergeRound(h64, v3);
- h64 = XXH64_mergeRound(h64, v4);
-
- } else {
- h64 = seed + PRIME64_5;
- }
-
- h64 += (U64) len;
-
- while (p+8<=bEnd) {
- U64 const k1 = XXH64_round(0, XXH_get64bits(p));
- h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
- p+=8;
- }
-
- if (p+4<=bEnd) {
- h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
- h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
- p+=4;
- }
-
- while (p<bEnd) {
- h64 ^= (*p) * PRIME64_5;
- h64 = XXH_rotl64(h64, 11) * PRIME64_1;
- p++;
- }
-
- h64 ^= h64 >> 33;
- h64 *= PRIME64_2;
- h64 ^= h64 >> 29;
- h64 *= PRIME64_3;
- h64 ^= h64 >> 32;
-
- return h64;
-}
-
-
-XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
-{
-#if 0
- /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
- XXH64_state_t state;
- XXH64_reset(&state, seed);
- XXH64_update(&state, input, len);
- return XXH64_digest(&state);
-#else
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
- if (XXH_FORCE_ALIGN_CHECK) {
- if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
- else
- return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
- } }
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
- else
- return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
-#endif
-}
-
-/*====== Hash Streaming ======*/
-
-XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
-{
- return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
-}
-XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
-{
- XXH_free(statePtr);
- return XXH_OK;
-}
-
-XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
-{
- memcpy(dstState, srcState, sizeof(*dstState));
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
-{
- XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
- memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */
- state.v1 = seed + PRIME64_1 + PRIME64_2;
- state.v2 = seed + PRIME64_2;
- state.v3 = seed + 0;
- state.v4 = seed - PRIME64_1;
- memcpy(statePtr, &state, sizeof(state));
- return XXH_OK;
-}
-
-FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
-{
- const BYTE* p = (const BYTE*)input;
- const BYTE* const bEnd = p + len;
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (input==NULL) return XXH_ERROR;
-#endif
-
- state->total_len += len;
-
- if (state->memsize + len < 32) { /* fill in tmp buffer */
- XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
- state->memsize += (U32)len;
- return XXH_OK;
- }
-
- if (state->memsize) { /* tmp buffer is full */
- XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
- state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
- state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
- state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
- state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
- p += 32-state->memsize;
- state->memsize = 0;
- }
-
- if (p+32 <= bEnd) {
- const BYTE* const limit = bEnd - 32;
- U64 v1 = state->v1;
- U64 v2 = state->v2;
- U64 v3 = state->v3;
- U64 v4 = state->v4;
-
- do {
- v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
- v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
- v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
- v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
- } while (p<=limit);
-
- state->v1 = v1;
- state->v2 = v2;
- state->v3 = v3;
- state->v4 = v4;
- }
-
- if (p < bEnd) {
- XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
- state->memsize = (unsigned)(bEnd-p);
- }
-
- return XXH_OK;
-}
-
-XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
-{
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
- else
- return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
-}
-
-FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
-{
- const BYTE * p = (const BYTE*)state->mem64;
- const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
- U64 h64;
-
- if (state->total_len >= 32) {
- U64 const v1 = state->v1;
- U64 const v2 = state->v2;
- U64 const v3 = state->v3;
- U64 const v4 = state->v4;
-
- h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
- h64 = XXH64_mergeRound(h64, v1);
- h64 = XXH64_mergeRound(h64, v2);
- h64 = XXH64_mergeRound(h64, v3);
- h64 = XXH64_mergeRound(h64, v4);
- } else {
- h64 = state->v3 + PRIME64_5;
- }
-
- h64 += (U64) state->total_len;
-
- while (p+8<=bEnd) {
- U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
- h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
- p+=8;
- }
-
- if (p+4<=bEnd) {
- h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
- h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
- p+=4;
- }
-
- while (p<bEnd) {
- h64 ^= (*p) * PRIME64_5;
- h64 = XXH_rotl64(h64, 11) * PRIME64_1;
- p++;
- }
-
- h64 ^= h64 >> 33;
- h64 *= PRIME64_2;
- h64 ^= h64 >> 29;
- h64 *= PRIME64_3;
- h64 ^= h64 >> 32;
-
- return h64;
-}
-
-XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
-{
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH64_digest_endian(state_in, XXH_littleEndian);
- else
- return XXH64_digest_endian(state_in, XXH_bigEndian);
-}
-
-
-/*====== Canonical representation ======*/
-
-XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
-{
- XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
- if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
- memcpy(dst, &hash, sizeof(*dst));
-}
-
-XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
-{
- return XXH_readBE64(src);
-}
-
-#endif /* XXH_NO_LONG_LONG */
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/xxhash.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/xxhash.h b/cpp/src/plasma/thirdparty/xxhash.h
deleted file mode 100644
index 9d831e0..0000000
--- a/cpp/src/plasma/thirdparty/xxhash.h
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- xxHash - Extremely Fast Hash algorithm
- Header File
- Copyright (C) 2012-2016, Yann Collet.
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - xxHash source repository : https://github.com/Cyan4973/xxHash
-*/
-
-/* Notice extracted from xxHash homepage :
-
-xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
-It also successfully passes all tests from the SMHasher suite.
-
-Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
-
-Name Speed Q.Score Author
-xxHash 5.4 GB/s 10
-CrapWow 3.2 GB/s 2 Andrew
-MumurHash 3a 2.7 GB/s 10 Austin Appleby
-SpookyHash 2.0 GB/s 10 Bob Jenkins
-SBox 1.4 GB/s 9 Bret Mulvey
-Lookup3 1.2 GB/s 9 Bob Jenkins
-SuperFastHash 1.2 GB/s 1 Paul Hsieh
-CityHash64 1.05 GB/s 10 Pike & Alakuijala
-FNV 0.55 GB/s 5 Fowler, Noll, Vo
-CRC32 0.43 GB/s 9
-MD5-32 0.33 GB/s 10 Ronald L. Rivest
-SHA1-32 0.28 GB/s 10
-
-Q.Score is a measure of quality of the hash function.
-It depends on successfully passing SMHasher test set.
-10 is a perfect score.
-
-A 64-bits version, named XXH64, is available since r35.
-It offers much better speed, but for 64-bits applications only.
-Name Speed on 64 bits Speed on 32 bits
-XXH64 13.8 GB/s 1.9 GB/s
-XXH32 6.8 GB/s 6.0 GB/s
-*/
-
-#ifndef XXHASH_H_5627135585666179
-#define XXHASH_H_5627135585666179 1
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************
-* Definitions
-******************************/
-#include <stddef.h> /* size_t */
-typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
-
-
-/* ****************************
-* API modifier
-******************************/
-/** XXH_PRIVATE_API
-* This is useful to include xxhash functions in `static` mode
-* in order to inline them, and remove their symbol from the public list.
-* Methodology :
-* #define XXH_PRIVATE_API
-* #include "xxhash.h"
-* `xxhash.c` is automatically included.
-* It's not useful to compile and link it as a separate module.
-*/
-#ifdef XXH_PRIVATE_API
-# ifndef XXH_STATIC_LINKING_ONLY
-# define XXH_STATIC_LINKING_ONLY
-# endif
-# if defined(__GNUC__)
-# define XXH_PUBLIC_API static __inline __attribute__((unused))
-# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define XXH_PUBLIC_API static inline
-# elif defined(_MSC_VER)
-# define XXH_PUBLIC_API static __inline
-# else
-# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */
-# endif
-#else
-# define XXH_PUBLIC_API /* do nothing */
-#endif /* XXH_PRIVATE_API */
-
-/*!XXH_NAMESPACE, aka Namespace Emulation :
-
-If you want to include _and expose_ xxHash functions from within your own library,
-but also want to avoid symbol collisions with other libraries which may also include xxHash,
-
-you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
-with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
-
-Note that no change is required within the calling program as long as it includes `xxhash.h` :
-regular symbol name will be automatically translated by this header.
-*/
-#ifdef XXH_NAMESPACE
-# define XXH_CAT(A,B) A##B
-# define XXH_NAME2(A,B) XXH_CAT(A,B)
-# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
-# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
-# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
-# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
-# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
-# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
-# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
-# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
-# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
-# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
-# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
-# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
-# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
-# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
-# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
-# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
-# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
-# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
-# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
-#endif
-
-
-/* *************************************
-* Version
-***************************************/
-#define XXH_VERSION_MAJOR 0
-#define XXH_VERSION_MINOR 6
-#define XXH_VERSION_RELEASE 2
-#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
-XXH_PUBLIC_API unsigned XXH_versionNumber (void);
-
-
-/*-**********************************************************************
-* 32-bits hash
-************************************************************************/
-typedef unsigned int XXH32_hash_t;
-
-/*! XXH32() :
- Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
- The memory between input & input+length must be valid (allocated and read-accessible).
- "seed" can be used to alter the result predictably.
- Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
-XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
-
-/*====== Streaming ======*/
-typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */
-XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
-XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
-XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
-
-XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);
-XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
-
-/*
-These functions generate the xxHash of an input provided in multiple segments.
-Note that, for small input, they are slower than single-call functions, due to state management.
-For small input, prefer `XXH32()` and `XXH64()` .
-
-XXH state must first be allocated, using XXH*_createState() .
-
-Start a new hash by initializing state with a seed, using XXH*_reset().
-
-Then, feed the hash state by calling XXH*_update() as many times as necessary.
-Obviously, input must be allocated and read accessible.
-The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
-
-Finally, a hash value can be produced anytime, by using XXH*_digest().
-This function returns the nn-bits hash as an int or long long.
-
-It's still possible to continue inserting input into the hash state after a digest,
-and generate some new hashes later on, by calling again XXH*_digest().
-
-When done, free XXH state space if it was allocated dynamically.
-*/
-
-/*====== Canonical representation ======*/
-
-typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
-XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
-XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
-
-/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
-* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
-* These functions allow transformation of hash result into and from its canonical format.
-* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
-*/
-
-
-#ifndef XXH_NO_LONG_LONG
-/*-**********************************************************************
-* 64-bits hash
-************************************************************************/
-typedef unsigned long long XXH64_hash_t;
-
-/*! XXH64() :
- Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
- "seed" can be used to alter the result predictably.
- This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
-*/
-XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
-
-/*====== Streaming ======*/
-typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
-XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
-XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
-XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
-
-XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
-XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
-
-/*====== Canonical representation ======*/
-typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
-XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
-XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
-#endif /* XXH_NO_LONG_LONG */
-
-
-#ifdef XXH_STATIC_LINKING_ONLY
-
-/* ================================================================================================
- This section contains definitions which are not guaranteed to remain stable.
- They may change in future versions, becoming incompatible with a different version of the library.
- They shall only be used with static linking.
- Never use these definitions in association with dynamic linking !
-=================================================================================================== */
-
-/* These definitions are only meant to make possible
- static allocation of XXH state, on stack or in a struct for example.
- Never use members directly. */
-
-struct XXH32_state_s {
- unsigned total_len_32;
- unsigned large_len;
- unsigned v1;
- unsigned v2;
- unsigned v3;
- unsigned v4;
- unsigned mem32[4]; /* buffer defined as U32 for alignment */
- unsigned memsize;
- unsigned reserved; /* never read nor write, will be removed in a future version */
-}; /* typedef'd to XXH32_state_t */
-
-#ifndef XXH_NO_LONG_LONG /* remove 64-bits support */
-struct XXH64_state_s {
- unsigned long long total_len;
- unsigned long long v1;
- unsigned long long v2;
- unsigned long long v3;
- unsigned long long v4;
- unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
- unsigned memsize;
- unsigned reserved[2]; /* never read nor write, will be removed in a future version */
-}; /* typedef'd to XXH64_state_t */
-#endif
-
-#ifdef XXH_PRIVATE_API
-# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */
-#endif
-
-#endif /* XXH_STATIC_LINKING_ONLY */
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* XXHASH_H_5627135585666179 */
[13/14] arrow git commit: [C++] Restore Plasma source tree after
0.5.0 release
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/fling.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
new file mode 100644
index 0000000..79da4f4
--- /dev/null
+++ b/cpp/src/plasma/fling.cc
@@ -0,0 +1,90 @@
+// Copyright 2013 Sharvil Nanavati
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "plasma/fling.h"
+
+#include <string.h>
+
+void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len) {
+ iov->iov_base = buf;
+ iov->iov_len = 1;
+
+ msg->msg_iov = iov;
+ msg->msg_iovlen = 1;
+ msg->msg_control = buf;
+ msg->msg_controllen = buf_len;
+ msg->msg_name = NULL;
+ msg->msg_namelen = 0;
+}
+
+int send_fd(int conn, int fd) {
+ struct msghdr msg;
+ struct iovec iov;
+ char buf[CMSG_SPACE(sizeof(int))];
+ memset(&buf, 0, CMSG_SPACE(sizeof(int)));
+
+ init_msg(&msg, &iov, buf, sizeof(buf));
+
+ struct cmsghdr* header = CMSG_FIRSTHDR(&msg);
+ header->cmsg_level = SOL_SOCKET;
+ header->cmsg_type = SCM_RIGHTS;
+ header->cmsg_len = CMSG_LEN(sizeof(int));
+ *reinterpret_cast<int*>(CMSG_DATA(header)) = fd;
+
+ // Send file descriptor.
+ ssize_t r = sendmsg(conn, &msg, 0);
+ if (r >= 0) {
+ return 0;
+ } else {
+ return static_cast<int>(r);
+ }
+}
+
+int recv_fd(int conn) {
+ struct msghdr msg;
+ struct iovec iov;
+ char buf[CMSG_SPACE(sizeof(int))];
+ init_msg(&msg, &iov, buf, sizeof(buf));
+
+ if (recvmsg(conn, &msg, 0) == -1) return -1;
+
+ int found_fd = -1;
+ int oh_noes = 0;
+ for (struct cmsghdr* header = CMSG_FIRSTHDR(&msg); header != NULL;
+ header = CMSG_NXTHDR(&msg, header))
+ if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) {
+ ssize_t count =
+ (header->cmsg_len - (CMSG_DATA(header) - (unsigned char*)header)) / sizeof(int);
+ for (int i = 0; i < count; ++i) {
+ int fd = (reinterpret_cast<int*>(CMSG_DATA(header)))[i];
+ if (found_fd == -1) {
+ found_fd = fd;
+ } else {
+ close(fd);
+ oh_noes = 1;
+ }
+ }
+ }
+
+ // The sender sent us more than one file descriptor. We've closed
+ // them all to prevent fd leaks but notify the caller that we got
+ // a bad message.
+ if (oh_noes) {
+ close(found_fd);
+ errno = EBADMSG;
+ return -1;
+ }
+
+ return found_fd;
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/fling.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/fling.h b/cpp/src/plasma/fling.h
new file mode 100644
index 0000000..78ac9d1
--- /dev/null
+++ b/cpp/src/plasma/fling.h
@@ -0,0 +1,52 @@
+// Copyright 2013 Sharvil Nanavati
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// FLING: Exchanging file descriptors over sockets
+//
+// This is a little library for sending file descriptors over a socket
+// between processes. The reason for doing that (as opposed to using
+// filenames to share the files) is so (a) no files remain in the
+// filesystem after all the processes terminate, (b) to make sure that
+// there are no name collisions and (c) to be able to control who has
+// access to the data.
+//
+// Most of the code is from https://github.com/sharvil/flingfd
+
+#include <errno.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+// This is neccessary for Mac OS X, see http://www.apuebook.com/faqs2e.html
+// (10).
+#if !defined(CMSG_SPACE) && !defined(CMSG_LEN)
+#define CMSG_SPACE(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + __DARWIN_ALIGN32(len))
+#define CMSG_LEN(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + (len))
+#endif
+
+void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len);
+
+// Send a file descriptor over a unix domain socket.
+//
+// @param conn Unix domain socket to send the file descriptor over.
+// @param fd File descriptor to send over.
+// @return Status code which is < 0 on failure.
+int send_fd(int conn, int fd);
+
+// Receive a file descriptor over a unix domain socket.
+//
+// @param conn Unix domain socket to receive the file descriptor from.
+// @return File descriptor or a value < 0 on failure.
+int recv_fd(int conn);
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/format/.gitignore
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/format/.gitignore b/cpp/src/plasma/format/.gitignore
new file mode 100644
index 0000000..b2ddb05
--- /dev/null
+++ b/cpp/src/plasma/format/.gitignore
@@ -0,0 +1 @@
+*_generated.h
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/format/common.fbs
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/format/common.fbs b/cpp/src/plasma/format/common.fbs
new file mode 100644
index 0000000..4d7d285
--- /dev/null
+++ b/cpp/src/plasma/format/common.fbs
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Object information data structure.
+table ObjectInfo {
+ // Object ID of this object.
+ object_id: string;
+ // Number of bytes the content of this object occupies in memory.
+ data_size: long;
+ // Number of bytes the metadata of this object occupies in memory.
+ metadata_size: long;
+ // Unix epoch of when this object was created.
+ create_time: long;
+ // How long creation of this object took.
+ construct_duration: long;
+ // Hash of the object content.
+ digest: string;
+ // Specifies if this object was deleted or added.
+ is_deletion: bool;
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/format/plasma.fbs
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
new file mode 100644
index 0000000..23782ad
--- /dev/null
+++ b/cpp/src/plasma/format/plasma.fbs
@@ -0,0 +1,291 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Plasma protocol specification
+
+enum MessageType:int {
+ // Create a new object.
+ PlasmaCreateRequest = 1,
+ PlasmaCreateReply,
+ // Seal an object.
+ PlasmaSealRequest,
+ PlasmaSealReply,
+ // Get an object that is stored on the local Plasma store.
+ PlasmaGetRequest,
+ PlasmaGetReply,
+ // Release an object.
+ PlasmaReleaseRequest,
+ PlasmaReleaseReply,
+ // Delete an object.
+ PlasmaDeleteRequest,
+ PlasmaDeleteReply,
+ // Get status of an object.
+ PlasmaStatusRequest,
+ PlasmaStatusReply,
+ // See if the store contains an object (will be deprecated).
+ PlasmaContainsRequest,
+ PlasmaContainsReply,
+ // Get information for a newly connecting client.
+ PlasmaConnectRequest,
+ PlasmaConnectReply,
+ // Make room for new objects in the plasma store.
+ PlasmaEvictRequest,
+ PlasmaEvictReply,
+ // Fetch objects from remote Plasma stores.
+ PlasmaFetchRequest,
+ // Wait for objects to be ready either from local or remote Plasma stores.
+ PlasmaWaitRequest,
+ PlasmaWaitReply,
+ // Subscribe to a list of objects or to all objects.
+ PlasmaSubscribeRequest,
+ // Unsubscribe.
+ PlasmaUnsubscribeRequest,
+ // Sending and receiving data.
+ // PlasmaDataRequest initiates sending the data, there will be one
+ // such message per data transfer.
+ PlasmaDataRequest,
+ // PlasmaDataReply contains the actual data and is sent back to the
+ // object store that requested the data. For each transfer, multiple
+ // reply messages get sent. Each one contains a fixed number of bytes.
+ PlasmaDataReply,
+ // Object notifications.
+ PlasmaNotification
+}
+
+enum PlasmaError:int {
+ // Operation was successful.
+ OK,
+ // Trying to create an object that already exists.
+ ObjectExists,
+ // Trying to access an object that doesn't exist.
+ ObjectNonexistent,
+ // Trying to create an object but there isn't enough space in the store.
+ OutOfMemory
+}
+
+// Plasma store messages
+
+struct PlasmaObjectSpec {
+ // Index of the memory segment (= memory mapped file) that
+ // this object is allocated in.
+ segment_index: int;
+ // Size in bytes of this segment (needed to call mmap).
+ mmap_size: ulong;
+ // The offset in bytes in the memory mapped file of the data.
+ data_offset: ulong;
+ // The size in bytes of the data.
+ data_size: ulong;
+ // The offset in bytes in the memory mapped file of the metadata.
+ metadata_offset: ulong;
+ // The size in bytes of the metadata.
+ metadata_size: ulong;
+}
+
+table PlasmaCreateRequest {
+ // ID of the object to be created.
+ object_id: string;
+ // The size of the object's data in bytes.
+ data_size: ulong;
+ // The size of the object's metadata in bytes.
+ metadata_size: ulong;
+}
+
+table PlasmaCreateReply {
+ // ID of the object that was created.
+ object_id: string;
+ // The object that is returned with this reply.
+ plasma_object: PlasmaObjectSpec;
+ // Error that occurred for this call.
+ error: PlasmaError;
+}
+
+table PlasmaSealRequest {
+ // ID of the object to be sealed.
+ object_id: string;
+ // Hash of the object data.
+ digest: string;
+}
+
+table PlasmaSealReply {
+ // ID of the object that was sealed.
+ object_id: string;
+ // Error code.
+ error: PlasmaError;
+}
+
+table PlasmaGetRequest {
+ // IDs of the objects stored at local Plasma store we are getting.
+ object_ids: [string];
+ // The number of milliseconds before the request should timeout.
+ timeout_ms: long;
+}
+
+table PlasmaGetReply {
+ // IDs of the objects being returned.
+ // This number can be smaller than the number of requested
+ // objects if not all requested objects are stored and sealed
+ // in the local Plasma store.
+ object_ids: [string];
+ // Plasma object information, in the same order as their IDs.
+ plasma_objects: [PlasmaObjectSpec];
+ // The number of elements in both object_ids and plasma_objects arrays must agree.
+}
+
+table PlasmaReleaseRequest {
+ // ID of the object to be released.
+ object_id: string;
+}
+
+table PlasmaReleaseReply {
+ // ID of the object that was released.
+ object_id: string;
+ // Error code.
+ error: PlasmaError;
+}
+
+table PlasmaDeleteRequest {
+ // ID of the object to be deleted.
+ object_id: string;
+}
+
+table PlasmaDeleteReply {
+ // ID of the object that was deleted.
+ object_id: string;
+ // Error code.
+ error: PlasmaError;
+}
+
+table PlasmaStatusRequest {
+ // IDs of the objects stored at local Plasma store we request the status of.
+ object_ids: [string];
+}
+
+enum ObjectStatus:int {
+ // Object is stored in the local Plasma Store.
+ Local = 1,
+ // Object is stored on a remote Plasma store, and it is not stored on the
+ // local Plasma Store.
+ Remote,
+ // Object is not stored in the system.
+ Nonexistent,
+ // Object is currently transferred from a remote Plasma store the the local
+ // Plasma Store.
+ Transfer
+}
+
+table PlasmaStatusReply {
+ // IDs of the objects being returned.
+ object_ids: [string];
+ // Status of the object.
+ status: [ObjectStatus];
+}
+
+// PlasmaContains is a subset of PlasmaStatus which does not
+// involve the plasma manager, only the store. We should consider
+// unifying them in the future and deprecating PlasmaContains.
+
+table PlasmaContainsRequest {
+ // ID of the object we are querying.
+ object_id: string;
+}
+
+table PlasmaContainsReply {
+ // ID of the object we are querying.
+ object_id: string;
+ // 1 if the object is in the store and 0 otherwise.
+ has_object: int;
+}
+
+// PlasmaConnect is used by a plasma client the first time it connects with the
+// store. This is not really necessary, but is used to get some information
+// about the store such as its memory capacity.
+
+table PlasmaConnectRequest {
+}
+
+table PlasmaConnectReply {
+ // The memory capacity of the store.
+ memory_capacity: long;
+}
+
+table PlasmaEvictRequest {
+ // Number of bytes that shall be freed.
+ num_bytes: ulong;
+}
+
+table PlasmaEvictReply {
+ // Number of bytes that have been freed.
+ num_bytes: ulong;
+}
+
+table PlasmaFetchRequest {
+ // IDs of objects to be gotten.
+ object_ids: [string];
+}
+
+table ObjectRequestSpec {
+ // ID of the object.
+ object_id: string;
+ // The type of the object. This specifies whether we
+ // will be waiting for an object store in the local or
+ // global Plasma store.
+ type: int;
+}
+
+table PlasmaWaitRequest {
+ // Array of object requests whose status we are asking for.
+ object_requests: [ObjectRequestSpec];
+ // Number of objects expected to be returned, if available.
+ num_ready_objects: int;
+ // timeout
+ timeout: long;
+}
+
+table ObjectReply {
+ // ID of the object.
+ object_id: string;
+ // The object status. This specifies where the object is stored.
+ status: int;
+}
+
+table PlasmaWaitReply {
+ // Array of object requests being returned.
+ object_requests: [ObjectReply];
+ // Number of objects expected to be returned, if available.
+ num_ready_objects: int;
+}
+
+table PlasmaSubscribeRequest {
+}
+
+table PlasmaDataRequest {
+ // ID of the object that is requested.
+ object_id: string;
+ // The host address where the data shall be sent to.
+ address: string;
+ // The port of the manager the data shall be sent to.
+ port: int;
+}
+
+table PlasmaDataReply {
+ // ID of the object that will be sent.
+ object_id: string;
+ // Size of the object data in bytes.
+ object_size: ulong;
+ // Size of the metadata in bytes.
+ metadata_size: ulong;
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/io.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
new file mode 100644
index 0000000..5875ebb
--- /dev/null
+++ b/cpp/src/plasma/io.cc
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/io.h"
+
+#include "plasma/common.h"
+
+using arrow::Status;
+
+/* Number of times we try binding to a socket. */
+#define NUM_BIND_ATTEMPTS 5
+#define BIND_TIMEOUT_MS 100
+
+/* Number of times we try connecting to a socket. */
+#define NUM_CONNECT_ATTEMPTS 50
+#define CONNECT_TIMEOUT_MS 100
+
+Status WriteBytes(int fd, uint8_t* cursor, size_t length) {
+ ssize_t nbytes = 0;
+ size_t bytesleft = length;
+ size_t offset = 0;
+ while (bytesleft > 0) {
+ /* While we haven't written the whole message, write to the file descriptor,
+ * advance the cursor, and decrease the amount left to write. */
+ nbytes = write(fd, cursor + offset, bytesleft);
+ if (nbytes < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
+ return Status::IOError(std::string(strerror(errno)));
+ } else if (nbytes == 0) {
+ return Status::IOError("Encountered unexpected EOF");
+ }
+ ARROW_CHECK(nbytes > 0);
+ bytesleft -= nbytes;
+ offset += nbytes;
+ }
+
+ return Status::OK();
+}
+
+Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes) {
+ int64_t version = PLASMA_PROTOCOL_VERSION;
+ RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)));
+ RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&type), sizeof(type)));
+ RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)));
+ return WriteBytes(fd, bytes, length * sizeof(char));
+}
+
+Status ReadBytes(int fd, uint8_t* cursor, size_t length) {
+ ssize_t nbytes = 0;
+ /* Termination condition: EOF or read 'length' bytes total. */
+ size_t bytesleft = length;
+ size_t offset = 0;
+ while (bytesleft > 0) {
+ nbytes = read(fd, cursor + offset, bytesleft);
+ if (nbytes < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
+ return Status::IOError(std::string(strerror(errno)));
+ } else if (0 == nbytes) {
+ return Status::IOError("Encountered unexpected EOF");
+ }
+ ARROW_CHECK(nbytes > 0);
+ bytesleft -= nbytes;
+ offset += nbytes;
+ }
+
+ return Status::OK();
+}
+
+Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
+ int64_t version;
+ RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)),
+ *type = DISCONNECT_CLIENT);
+ ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version;
+ size_t length;
+ RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(type), sizeof(*type)),
+ *type = DISCONNECT_CLIENT);
+ RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)),
+ *type = DISCONNECT_CLIENT);
+ if (length > buffer->size()) { buffer->resize(length); }
+ RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer->data(), length), *type = DISCONNECT_CLIENT);
+ return Status::OK();
+}
+
+int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
+ struct sockaddr_un socket_address;
+ int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+ if (socket_fd < 0) {
+ ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
+ return -1;
+ }
+ /* Tell the system to allow the port to be reused. */
+ int on = 1;
+ if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&on),
+ sizeof(on)) < 0) {
+ ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname;
+ close(socket_fd);
+ return -1;
+ }
+
+ unlink(pathname.c_str());
+ memset(&socket_address, 0, sizeof(socket_address));
+ socket_address.sun_family = AF_UNIX;
+ if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
+ ARROW_LOG(ERROR) << "Socket pathname is too long.";
+ close(socket_fd);
+ return -1;
+ }
+ strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
+
+ if (bind(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) != 0) {
+ ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname;
+ close(socket_fd);
+ return -1;
+ }
+ if (shall_listen && listen(socket_fd, 128) == -1) {
+ ARROW_LOG(ERROR) << "Could not listen to socket " << pathname;
+ close(socket_fd);
+ return -1;
+ }
+ return socket_fd;
+}
+
+int connect_ipc_sock_retry(
+ const std::string& pathname, int num_retries, int64_t timeout) {
+ /* Pick the default values if the user did not specify. */
+ if (num_retries < 0) { num_retries = NUM_CONNECT_ATTEMPTS; }
+ if (timeout < 0) { timeout = CONNECT_TIMEOUT_MS; }
+
+ int fd = -1;
+ for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
+ fd = connect_ipc_sock(pathname);
+ if (fd >= 0) { break; }
+ if (num_attempts == 0) {
+ ARROW_LOG(ERROR) << "Connection to socket failed for pathname " << pathname;
+ }
+ /* Sleep for timeout milliseconds. */
+ usleep(static_cast<int>(timeout * 1000));
+ }
+ /* If we could not connect to the socket, exit. */
+ if (fd == -1) { ARROW_LOG(FATAL) << "Could not connect to socket " << pathname; }
+ return fd;
+}
+
+int connect_ipc_sock(const std::string& pathname) {
+ struct sockaddr_un socket_address;
+ int socket_fd;
+
+ socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+ if (socket_fd < 0) {
+ ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
+ return -1;
+ }
+
+ memset(&socket_address, 0, sizeof(socket_address));
+ socket_address.sun_family = AF_UNIX;
+ if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
+ ARROW_LOG(ERROR) << "Socket pathname is too long.";
+ return -1;
+ }
+ strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
+
+ if (connect(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) !=
+ 0) {
+ close(socket_fd);
+ return -1;
+ }
+
+ return socket_fd;
+}
+
+int AcceptClient(int socket_fd) {
+ int client_fd = accept(socket_fd, NULL, NULL);
+ if (client_fd < 0) {
+ ARROW_LOG(ERROR) << "Error reading from socket.";
+ return -1;
+ }
+ return client_fd;
+}
+
+uint8_t* read_message_async(int sock) {
+ int64_t size;
+ Status s = ReadBytes(sock, reinterpret_cast<uint8_t*>(&size), sizeof(int64_t));
+ if (!s.ok()) {
+ /* The other side has closed the socket. */
+ ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
+ close(sock);
+ return NULL;
+ }
+ uint8_t* message = reinterpret_cast<uint8_t*>(malloc(size));
+ s = ReadBytes(sock, message, size);
+ if (!s.ok()) {
+ /* The other side has closed the socket. */
+ ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
+ close(sock);
+ return NULL;
+ }
+ return message;
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/io.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/io.h b/cpp/src/plasma/io.h
new file mode 100644
index 0000000..43c3fb5
--- /dev/null
+++ b/cpp/src/plasma/io.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_IO_H
+#define PLASMA_IO_H
+
+#include <inttypes.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <string>
+#include <vector>
+
+#include "arrow/status.h"
+
+// TODO(pcm): Replace our own custom message header (message type,
+// message length, plasma protocol verion) with one that is serialized
+// using flatbuffers.
+#define PLASMA_PROTOCOL_VERSION 0x0000000000000000
+#define DISCONNECT_CLIENT 0
+
+arrow::Status WriteBytes(int fd, uint8_t* cursor, size_t length);
+
+arrow::Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes);
+
+arrow::Status ReadBytes(int fd, uint8_t* cursor, size_t length);
+
+arrow::Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer);
+
+int bind_ipc_sock(const std::string& pathname, bool shall_listen);
+
+int connect_ipc_sock(const std::string& pathname);
+
+int connect_ipc_sock_retry(const std::string& pathname, int num_retries, int64_t timeout);
+
+int AcceptClient(int socket_fd);
+
+uint8_t* read_message_async(int sock);
+
+#endif // PLASMA_IO_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/malloc.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
new file mode 100644
index 0000000..97c9a16
--- /dev/null
+++ b/cpp/src/plasma/malloc.cc
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/malloc.h"
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <unordered_map>
+
+#include "plasma/common.h"
+
+extern "C" {
+void* fake_mmap(size_t);
+int fake_munmap(void*, int64_t);
+
+#define MMAP(s) fake_mmap(s)
+#define MUNMAP(a, s) fake_munmap(a, s)
+#define DIRECT_MMAP(s) fake_mmap(s)
+#define DIRECT_MUNMAP(a, s) fake_munmap(a, s)
+#define USE_DL_PREFIX
+#define HAVE_MORECORE 0
+#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
+#define DEFAULT_GRANULARITY ((size_t)128U * 1024U)
+
+#include "thirdparty/dlmalloc.c" // NOLINT
+
+#undef MMAP
+#undef MUNMAP
+#undef DIRECT_MMAP
+#undef DIRECT_MUNMAP
+#undef USE_DL_PREFIX
+#undef HAVE_MORECORE
+#undef DEFAULT_GRANULARITY
+}
+
+struct mmap_record {
+ int fd;
+ int64_t size;
+};
+
+namespace {
+
+/** Hashtable that contains one entry per segment that we got from the OS
+ * via mmap. Associates the address of that segment with its file descriptor
+ * and size. */
+std::unordered_map<void*, mmap_record> mmap_records;
+
+} /* namespace */
+
+constexpr int GRANULARITY_MULTIPLIER = 2;
+
+static void* pointer_advance(void* p, ptrdiff_t n) {
+ return (unsigned char*)p + n;
+}
+
+static void* pointer_retreat(void* p, ptrdiff_t n) {
+ return (unsigned char*)p - n;
+}
+
+static ptrdiff_t pointer_distance(void const* pfrom, void const* pto) {
+ return (unsigned char const*)pto - (unsigned char const*)pfrom;
+}
+
+/* Create a buffer. This is creating a temporary file and then
+ * immediately unlinking it so we do not leave traces in the system. */
+int create_buffer(int64_t size) {
+ int fd;
+#ifdef _WIN32
+ if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
+ (DWORD)((uint64_t)size >> (CHAR_BIT * sizeof(DWORD))), (DWORD)(uint64_t)size,
+ NULL)) {
+ fd = -1;
+ }
+#else
+#ifdef __linux__
+ constexpr char file_template[] = "/dev/shm/plasmaXXXXXX";
+#else
+ constexpr char file_template[] = "/tmp/plasmaXXXXXX";
+#endif
+ char file_name[32];
+ strncpy(file_name, file_template, 32);
+ fd = mkstemp(file_name);
+ if (fd < 0) return -1;
+ FILE* file = fdopen(fd, "a+");
+ if (!file) {
+ close(fd);
+ return -1;
+ }
+ if (unlink(file_name) != 0) {
+ ARROW_LOG(FATAL) << "unlink error";
+ return -1;
+ }
+ if (ftruncate(fd, (off_t)size) != 0) {
+ ARROW_LOG(FATAL) << "ftruncate error";
+ return -1;
+ }
+#endif
+ return fd;
+}
+
+void* fake_mmap(size_t size) {
+ /* Add sizeof(size_t) so that the returned pointer is deliberately not
+ * page-aligned. This ensures that the segments of memory returned by
+ * fake_mmap are never contiguous. */
+ size += sizeof(size_t);
+
+ int fd = create_buffer(size);
+ ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
+ void* pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (pointer == MAP_FAILED) { return pointer; }
+
+ /* Increase dlmalloc's allocation granularity directly. */
+ mparams.granularity *= GRANULARITY_MULTIPLIER;
+
+ mmap_record& record = mmap_records[pointer];
+ record.fd = fd;
+ record.size = size;
+
+ /* We lie to dlmalloc about where mapped memory actually lives. */
+ pointer = pointer_advance(pointer, sizeof(size_t));
+ ARROW_LOG(DEBUG) << pointer << " = fake_mmap(" << size << ")";
+ return pointer;
+}
+
+int fake_munmap(void* addr, int64_t size) {
+ ARROW_LOG(DEBUG) << "fake_munmap(" << addr << ", " << size << ")";
+ addr = pointer_retreat(addr, sizeof(size_t));
+ size += sizeof(size_t);
+
+ auto entry = mmap_records.find(addr);
+
+ if (entry == mmap_records.end() || entry->second.size != size) {
+ /* Reject requests to munmap that don't directly match previous
+ * calls to mmap, to prevent dlmalloc from trimming. */
+ return -1;
+ }
+
+ int r = munmap(addr, size);
+ if (r == 0) { close(entry->second.fd); }
+
+ mmap_records.erase(entry);
+ return r;
+}
+
+void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_size, ptrdiff_t* offset) {
+ /* TODO(rshin): Implement a more efficient search through mmap_records. */
+ for (const auto& entry : mmap_records) {
+ if (addr >= entry.first && addr < pointer_advance(entry.first, entry.second.size)) {
+ *fd = entry.second.fd;
+ *map_size = entry.second.size;
+ *offset = pointer_distance(entry.first, addr);
+ return;
+ }
+ }
+ *fd = -1;
+ *map_size = 0;
+ *offset = 0;
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/malloc.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/malloc.h b/cpp/src/plasma/malloc.h
new file mode 100644
index 0000000..b4af2c8
--- /dev/null
+++ b/cpp/src/plasma/malloc.h
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_MALLOC_H
+#define PLASMA_MALLOC_H
+
+#include <inttypes.h>
+#include <stddef.h>
+
+void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_length, ptrdiff_t* offset);
+
+#endif // MALLOC_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/plasma.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc
new file mode 100644
index 0000000..559d8e7
--- /dev/null
+++ b/cpp/src/plasma/plasma.cc
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/plasma.h"
+
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/common.h"
+#include "plasma/protocol.h"
+
+int warn_if_sigpipe(int status, int client_sock) {
+ if (status >= 0) { return 0; }
+ if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
+ ARROW_LOG(WARNING) << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when "
+ "sending a message to client on fd "
+ << client_sock << ". The client on the other end may "
+ "have hung up.";
+ return errno;
+ }
+ ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock << ".";
+ return -1; // This is never reached.
+}
+
+/**
+ * This will create a new ObjectInfo buffer. The first sizeof(int64_t) bytes
+ * of this buffer are the length of the remaining message and the
+ * remaining message is a serialized version of the object info.
+ *
+ * @param object_info The object info to be serialized
+ * @return The object info buffer. It is the caller's responsibility to free
+ * this buffer with "delete" after it has been used.
+ */
+uint8_t* create_object_info_buffer(ObjectInfoT* object_info) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreateObjectInfo(fbb, object_info);
+ fbb.Finish(message);
+ uint8_t* notification = new uint8_t[sizeof(int64_t) + fbb.GetSize()];
+ *(reinterpret_cast<int64_t*>(notification)) = fbb.GetSize();
+ memcpy(notification + sizeof(int64_t), fbb.GetBufferPointer(), fbb.GetSize());
+ return notification;
+}
+
+ObjectTableEntry* get_object_table_entry(
+ PlasmaStoreInfo* store_info, const ObjectID& object_id) {
+ auto it = store_info->objects.find(object_id);
+ if (it == store_info->objects.end()) { return NULL; }
+ return it->second.get();
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/plasma.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
new file mode 100644
index 0000000..275d0c7
--- /dev/null
+++ b/cpp/src/plasma/plasma.h
@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_PLASMA_H
+#define PLASMA_PLASMA_H
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h> // pid_t
+
+#include <unordered_map>
+#include <unordered_set>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+#include "format/common_generated.h"
+#include "plasma/common.h"
+
+#define HANDLE_SIGPIPE(s, fd_) \
+ do { \
+ Status _s = (s); \
+ if (!_s.ok()) { \
+ if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) { \
+ ARROW_LOG(WARNING) \
+ << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \
+ "sending a message to client on fd " \
+ << fd_ << ". " \
+ "The client on the other end may have hung up."; \
+ } else { \
+ return _s; \
+ } \
+ } \
+ } while (0);
+
+/// Allocation granularity used in plasma for object allocation.
+#define BLOCK_SIZE 64
+
+/// Size of object hash digests.
+constexpr int64_t kDigestSize = sizeof(uint64_t);
+
+struct Client;
+
+/// Object request data structure. Used in the plasma_wait_for_objects()
+/// argument.
+typedef struct {
+ /// The ID of the requested object. If ID_NIL request any object.
+ ObjectID object_id;
+ /// Request associated to the object. It can take one of the following values:
+ /// - PLASMA_QUERY_LOCAL: return if or when the object is available in the
+ /// local Plasma Store.
+ /// - PLASMA_QUERY_ANYWHERE: return if or when the object is available in
+ /// the system (i.e., either in the local or a remote Plasma Store).
+ int type;
+ /// Object status. Same as the status returned by plasma_status() function
+ /// call. This is filled in by plasma_wait_for_objects1():
+ /// - ObjectStatus_Local: object is ready at the local Plasma Store.
+ /// - ObjectStatus_Remote: object is ready at a remote Plasma Store.
+ /// - ObjectStatus_Nonexistent: object does not exist in the system.
+ /// - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
+ /// for being transferred or it is transferring.
+ int status;
+} ObjectRequest;
+
+/// Mapping from object IDs to type and status of the request.
+typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectRequestMap;
+
+/// Handle to access memory mapped file and map it into client address space.
+typedef struct {
+ /// The file descriptor of the memory mapped file in the store. It is used as
+ /// a unique identifier of the file in the client to look up the corresponding
+ /// file descriptor on the client's side.
+ int store_fd;
+ /// The size in bytes of the memory mapped file.
+ int64_t mmap_size;
+} object_handle;
+
+// TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
+typedef struct {
+ /// Handle for memory mapped file the object is stored in.
+ object_handle handle;
+ /// The offset in bytes in the memory mapped file of the data.
+ ptrdiff_t data_offset;
+ /// The offset in bytes in the memory mapped file of the metadata.
+ ptrdiff_t metadata_offset;
+ /// The size in bytes of the data.
+ int64_t data_size;
+ /// The size in bytes of the metadata.
+ int64_t metadata_size;
+} PlasmaObject;
+
+typedef enum {
+ /// Object was created but not sealed in the local Plasma Store.
+ PLASMA_CREATED = 1,
+ /// Object is sealed and stored in the local Plasma Store.
+ PLASMA_SEALED
+} object_state;
+
+typedef enum {
+ /// The object was not found.
+ OBJECT_NOT_FOUND = 0,
+ /// The object was found.
+ OBJECT_FOUND = 1
+} object_status;
+
+typedef enum {
+ /// Query for object in the local plasma store.
+ PLASMA_QUERY_LOCAL = 1,
+ /// Query for object in the local plasma store or in a remote plasma store.
+ PLASMA_QUERY_ANYWHERE
+} object_request_type;
+
+/// This type is used by the Plasma store. It is here because it is exposed to
+/// the eviction policy.
+struct ObjectTableEntry {
+ /// Object id of this object.
+ ObjectID object_id;
+ /// Object info like size, creation time and owner.
+ ObjectInfoT info;
+ /// Memory mapped file containing the object.
+ int fd;
+ /// Size of the underlying map.
+ int64_t map_size;
+ /// Offset from the base of the mmap.
+ ptrdiff_t offset;
+ /// Pointer to the object data. Needed to free the object.
+ uint8_t* pointer;
+ /// Set of clients currently using this object.
+ std::unordered_set<Client*> clients;
+ /// The state of the object, e.g., whether it is open or sealed.
+ object_state state;
+ /// The digest of the object. Used to see if two objects are the same.
+ unsigned char digest[kDigestSize];
+};
+
+/// The plasma store information that is exposed to the eviction policy.
+struct PlasmaStoreInfo {
+ /// Objects that are in the Plasma store.
+ std::unordered_map<ObjectID, std::unique_ptr<ObjectTableEntry>, UniqueIDHasher> objects;
+ /// The amount of memory (in bytes) that we allow to be allocated in the
+ /// store.
+ int64_t memory_capacity;
+};
+
+/// Get an entry from the object table and return NULL if the object_id
+/// is not present.
+///
+/// @param store_info The PlasmaStoreInfo that contains the object table.
+/// @param object_id The object_id of the entry we are looking for.
+/// @return The entry associated with the object_id or NULL if the object_id
+/// is not present.
+ObjectTableEntry* get_object_table_entry(
+ PlasmaStoreInfo* store_info, const ObjectID& object_id);
+
+/// Print a warning if the status is less than zero. This should be used to check
+/// the success of messages sent to plasma clients. We print a warning instead of
+/// failing because the plasma clients are allowed to die. This is used to handle
+/// situations where the store writes to a client file descriptor, and the client
+/// may already have disconnected. If we have processed the disconnection and
+/// closed the file descriptor, we should get a BAD FILE DESCRIPTOR error. If we
+/// have not, then we should get a SIGPIPE. If we write to a TCP socket that
+/// isn't connected yet, then we should get an ECONNRESET.
+///
+/// @param status The status to check. If it is less less than zero, we will
+/// print a warning.
+/// @param client_sock The client socket. This is just used to print some extra
+/// information.
+/// @return The errno set.
+int warn_if_sigpipe(int status, int client_sock);
+
+uint8_t* create_object_info_buffer(ObjectInfoT* object_info);
+
+#endif // PLASMA_PLASMA_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/protocol.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
new file mode 100644
index 0000000..246aa29
--- /dev/null
+++ b/cpp/src/plasma/protocol.cc
@@ -0,0 +1,502 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/protocol.h"
+
+#include "flatbuffers/flatbuffers.h"
+#include "format/plasma_generated.h"
+
+#include "plasma/common.h"
+#include "plasma/io.h"
+
+using flatbuffers::uoffset_t;
+
+flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
+to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
+ int64_t num_objects) {
+ std::vector<flatbuffers::Offset<flatbuffers::String>> results;
+ for (int64_t i = 0; i < num_objects; i++) {
+ results.push_back(fbb->CreateString(object_ids[i].binary()));
+ }
+ return fbb->CreateVector(results);
+}
+
+Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer) {
+ int64_t type;
+ RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
+ ARROW_CHECK(type == message_type) << "type = " << type
+ << ", message_type = " << message_type;
+ return Status::OK();
+}
+
+template <typename Message>
+Status PlasmaSend(int sock, int64_t message_type, flatbuffers::FlatBufferBuilder* fbb,
+ const Message& message) {
+ fbb->Finish(message);
+ return WriteMessage(sock, message_type, fbb->GetSize(), fbb->GetBufferPointer());
+}
+
+// Create messages.
+
+Status SendCreateRequest(
+ int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaCreateRequest(
+ fbb, fbb.CreateString(object_id.binary()), data_size, metadata_size);
+ return PlasmaSend(sock, MessageType_PlasmaCreateRequest, &fbb, message);
+}
+
+Status ReadCreateRequest(
+ uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
+ *data_size = message->data_size();
+ *metadata_size = message->metadata_size();
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ return Status::OK();
+}
+
+Status SendCreateReply(
+ int sock, ObjectID object_id, PlasmaObject* object, int error_code) {
+ flatbuffers::FlatBufferBuilder fbb;
+ PlasmaObjectSpec plasma_object(object->handle.store_fd, object->handle.mmap_size,
+ object->data_offset, object->data_size, object->metadata_offset,
+ object->metadata_size);
+ auto message = CreatePlasmaCreateReply(
+ fbb, fbb.CreateString(object_id.binary()), &plasma_object, (PlasmaError)error_code);
+ return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
+}
+
+Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ object->handle.store_fd = message->plasma_object()->segment_index();
+ object->handle.mmap_size = message->plasma_object()->mmap_size();
+ object->data_offset = message->plasma_object()->data_offset();
+ object->data_size = message->plasma_object()->data_size();
+ object->metadata_offset = message->plasma_object()->metadata_offset();
+ object->metadata_size = message->plasma_object()->metadata_size();
+ return plasma_error_status(message->error());
+}
+
+// Seal messages.
+
+Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto digest_string = fbb.CreateString(reinterpret_cast<char*>(digest), kDigestSize);
+ auto message =
+ CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()), digest_string);
+ return PlasmaSend(sock, MessageType_PlasmaSealRequest, &fbb, message);
+}
+
+Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaSealRequest>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ ARROW_CHECK(message->digest()->size() == kDigestSize);
+ memcpy(digest, message->digest()->data(), kDigestSize);
+ return Status::OK();
+}
+
+Status SendSealReply(int sock, ObjectID object_id, int error) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaSealReply(
+ fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+ return PlasmaSend(sock, MessageType_PlasmaSealReply, &fbb, message);
+}
+
+Status ReadSealReply(uint8_t* data, ObjectID* object_id) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaSealReply>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ return plasma_error_status(message->error());
+}
+
+// Release messages.
+
+Status SendReleaseRequest(int sock, ObjectID object_id) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()));
+ return PlasmaSend(sock, MessageType_PlasmaReleaseRequest, &fbb, message);
+}
+
+Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaReleaseRequest>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ return Status::OK();
+}
+
+Status SendReleaseReply(int sock, ObjectID object_id, int error) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaReleaseReply(
+ fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+ return PlasmaSend(sock, MessageType_PlasmaReleaseReply, &fbb, message);
+}
+
+Status ReadReleaseReply(uint8_t* data, ObjectID* object_id) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ return plasma_error_status(message->error());
+}
+
+// Delete messages.
+
+Status SendDeleteRequest(int sock, ObjectID object_id) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaDeleteRequest(fbb, fbb.CreateString(object_id.binary()));
+ return PlasmaSend(sock, MessageType_PlasmaDeleteRequest, &fbb, message);
+}
+
+Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ return Status::OK();
+}
+
+Status SendDeleteReply(int sock, ObjectID object_id, int error) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaDeleteReply(
+ fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
+ return PlasmaSend(sock, MessageType_PlasmaDeleteReply, &fbb, message);
+}
+
+Status ReadDeleteReply(uint8_t* data, ObjectID* object_id) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaDeleteReply>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ return plasma_error_status(message->error());
+}
+
+// Satus messages.
+
+Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message =
+ CreatePlasmaStatusRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
+ return PlasmaSend(sock, MessageType_PlasmaStatusRequest, &fbb, message);
+}
+
+Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaStatusRequest>(data);
+ for (uoffset_t i = 0; i < num_objects; ++i) {
+ object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+ }
+ return Status::OK();
+}
+
+Status SendStatusReply(
+ int sock, ObjectID object_ids[], int object_status[], int64_t num_objects) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message =
+ CreatePlasmaStatusReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+ fbb.CreateVector(object_status, num_objects));
+ return PlasmaSend(sock, MessageType_PlasmaStatusReply, &fbb, message);
+}
+
+int64_t ReadStatusReply_num_objects(uint8_t* data) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
+ return message->object_ids()->size();
+}
+
+Status ReadStatusReply(
+ uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
+ for (uoffset_t i = 0; i < num_objects; ++i) {
+ object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+ }
+ for (uoffset_t i = 0; i < num_objects; ++i) {
+ object_status[i] = message->status()->data()[i];
+ }
+ return Status::OK();
+}
+
+// Contains messages.
+
+Status SendContainsRequest(int sock, ObjectID object_id) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaContainsRequest(fbb, fbb.CreateString(object_id.binary()));
+ return PlasmaSend(sock, MessageType_PlasmaContainsRequest, &fbb, message);
+}
+
+Status ReadContainsRequest(uint8_t* data, ObjectID* object_id) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaContainsRequest>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ return Status::OK();
+}
+
+Status SendContainsReply(int sock, ObjectID object_id, bool has_object) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message =
+ CreatePlasmaContainsReply(fbb, fbb.CreateString(object_id.binary()), has_object);
+ return PlasmaSend(sock, MessageType_PlasmaContainsReply, &fbb, message);
+}
+
+Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaContainsReply>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ *has_object = message->has_object();
+ return Status::OK();
+}
+
+// Connect messages.
+
+Status SendConnectRequest(int sock) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaConnectRequest(fbb);
+ return PlasmaSend(sock, MessageType_PlasmaConnectRequest, &fbb, message);
+}
+
+Status ReadConnectRequest(uint8_t* data) {
+ return Status::OK();
+}
+
+Status SendConnectReply(int sock, int64_t memory_capacity) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaConnectReply(fbb, memory_capacity);
+ return PlasmaSend(sock, MessageType_PlasmaConnectReply, &fbb, message);
+}
+
+Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaConnectReply>(data);
+ *memory_capacity = message->memory_capacity();
+ return Status::OK();
+}
+
+// Evict messages.
+
+Status SendEvictRequest(int sock, int64_t num_bytes) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaEvictRequest(fbb, num_bytes);
+ return PlasmaSend(sock, MessageType_PlasmaEvictRequest, &fbb, message);
+}
+
+Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaEvictRequest>(data);
+ *num_bytes = message->num_bytes();
+ return Status::OK();
+}
+
+Status SendEvictReply(int sock, int64_t num_bytes) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaEvictReply(fbb, num_bytes);
+ return PlasmaSend(sock, MessageType_PlasmaEvictReply, &fbb, message);
+}
+
+Status ReadEvictReply(uint8_t* data, int64_t& num_bytes) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaEvictReply>(data);
+ num_bytes = message->num_bytes();
+ return Status::OK();
+}
+
+// Get messages.
+
+Status SendGetRequest(
+ int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaGetRequest(
+ fbb, to_flatbuffer(&fbb, object_ids, num_objects), timeout_ms);
+ return PlasmaSend(sock, MessageType_PlasmaGetRequest, &fbb, message);
+}
+
+Status ReadGetRequest(
+ uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
+ for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
+ auto object_id = message->object_ids()->Get(i)->str();
+ object_ids.push_back(ObjectID::from_binary(object_id));
+ }
+ *timeout_ms = message->timeout_ms();
+ return Status::OK();
+}
+
+Status SendGetReply(int sock, ObjectID object_ids[],
+ std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
+ int64_t num_objects) {
+ flatbuffers::FlatBufferBuilder fbb;
+ std::vector<PlasmaObjectSpec> objects;
+
+ for (int i = 0; i < num_objects; ++i) {
+ const PlasmaObject& object = plasma_objects[object_ids[i]];
+ objects.push_back(PlasmaObjectSpec(object.handle.store_fd, object.handle.mmap_size,
+ object.data_offset, object.data_size, object.metadata_offset,
+ object.metadata_size));
+ }
+ auto message = CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
+ fbb.CreateVectorOfStructs(objects.data(), num_objects));
+ return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
+}
+
+Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
+ int64_t num_objects) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
+ for (uoffset_t i = 0; i < num_objects; ++i) {
+ object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
+ }
+ for (uoffset_t i = 0; i < num_objects; ++i) {
+ const PlasmaObjectSpec* object = message->plasma_objects()->Get(i);
+ plasma_objects[i].handle.store_fd = object->segment_index();
+ plasma_objects[i].handle.mmap_size = object->mmap_size();
+ plasma_objects[i].data_offset = object->data_offset();
+ plasma_objects[i].data_size = object->data_size();
+ plasma_objects[i].metadata_offset = object->metadata_offset();
+ plasma_objects[i].metadata_size = object->metadata_size();
+ }
+ return Status::OK();
+}
+
+// Fetch messages.
+
+Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message =
+ CreatePlasmaFetchRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
+ return PlasmaSend(sock, MessageType_PlasmaFetchRequest, &fbb, message);
+}
+
+Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaFetchRequest>(data);
+ for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
+ object_ids.push_back(ObjectID::from_binary(message->object_ids()->Get(i)->str()));
+ }
+ return Status::OK();
+}
+
+// Wait messages.
+
+Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
+ int num_ready_objects, int64_t timeout_ms) {
+ flatbuffers::FlatBufferBuilder fbb;
+
+ std::vector<flatbuffers::Offset<ObjectRequestSpec>> object_request_specs;
+ for (int i = 0; i < num_requests; i++) {
+ object_request_specs.push_back(CreateObjectRequestSpec(fbb,
+ fbb.CreateString(object_requests[i].object_id.binary()),
+ object_requests[i].type));
+ }
+
+ auto message = CreatePlasmaWaitRequest(
+ fbb, fbb.CreateVector(object_request_specs), num_ready_objects, timeout_ms);
+ return PlasmaSend(sock, MessageType_PlasmaWaitRequest, &fbb, message);
+}
+
+Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
+ int64_t* timeout_ms, int* num_ready_objects) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaWaitRequest>(data);
+ *num_ready_objects = message->num_ready_objects();
+ *timeout_ms = message->timeout();
+
+ for (uoffset_t i = 0; i < message->object_requests()->size(); i++) {
+ ObjectID object_id =
+ ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
+ ObjectRequest object_request({object_id, message->object_requests()->Get(i)->type(),
+ ObjectStatus_Nonexistent});
+ object_requests[object_id] = object_request;
+ }
+ return Status::OK();
+}
+
+Status SendWaitReply(
+ int sock, const ObjectRequestMap& object_requests, int num_ready_objects) {
+ flatbuffers::FlatBufferBuilder fbb;
+
+ std::vector<flatbuffers::Offset<ObjectReply>> object_replies;
+ for (const auto& entry : object_requests) {
+ const auto& object_request = entry.second;
+ object_replies.push_back(CreateObjectReply(
+ fbb, fbb.CreateString(object_request.object_id.binary()), object_request.status));
+ }
+
+ auto message = CreatePlasmaWaitReply(
+ fbb, fbb.CreateVector(object_replies.data(), num_ready_objects), num_ready_objects);
+ return PlasmaSend(sock, MessageType_PlasmaWaitReply, &fbb, message);
+}
+
+Status ReadWaitReply(
+ uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects) {
+ DCHECK(data);
+
+ auto message = flatbuffers::GetRoot<PlasmaWaitReply>(data);
+ *num_ready_objects = message->num_ready_objects();
+ for (int i = 0; i < *num_ready_objects; i++) {
+ object_requests[i].object_id =
+ ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
+ object_requests[i].status = message->object_requests()->Get(i)->status();
+ }
+ return Status::OK();
+}
+
+// Subscribe messages.
+
+Status SendSubscribeRequest(int sock) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaSubscribeRequest(fbb);
+ return PlasmaSend(sock, MessageType_PlasmaSubscribeRequest, &fbb, message);
+}
+
+// Data messages.
+
+Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto addr = fbb.CreateString(address, strlen(address));
+ auto message =
+ CreatePlasmaDataRequest(fbb, fbb.CreateString(object_id.binary()), addr, port);
+ return PlasmaSend(sock, MessageType_PlasmaDataRequest, &fbb, message);
+}
+
+Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaDataRequest>(data);
+ DCHECK(message->object_id()->size() == sizeof(ObjectID));
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ *address = strdup(message->address()->c_str());
+ *port = message->port();
+ return Status::OK();
+}
+
+Status SendDataReply(
+ int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size) {
+ flatbuffers::FlatBufferBuilder fbb;
+ auto message = CreatePlasmaDataReply(
+ fbb, fbb.CreateString(object_id.binary()), object_size, metadata_size);
+ return PlasmaSend(sock, MessageType_PlasmaDataReply, &fbb, message);
+}
+
+Status ReadDataReply(
+ uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size) {
+ DCHECK(data);
+ auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
+ *object_id = ObjectID::from_binary(message->object_id()->str());
+ *object_size = (int64_t)message->object_size();
+ *metadata_size = (int64_t)message->metadata_size();
+ return Status::OK();
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/protocol.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
new file mode 100644
index 0000000..5d9d136
--- /dev/null
+++ b/cpp/src/plasma/protocol.h
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_PROTOCOL_H
+#define PLASMA_PROTOCOL_H
+
+#include <vector>
+
+#include "arrow/status.h"
+#include "format/plasma_generated.h"
+#include "plasma/plasma.h"
+
+using arrow::Status;
+
+/* Plasma receive message. */
+
+Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer);
+
+/* Plasma Create message functions. */
+
+Status SendCreateRequest(
+ int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size);
+
+Status ReadCreateRequest(
+ uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size);
+
+Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error);
+
+Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object);
+
+/* Plasma Seal message functions. */
+
+Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest);
+
+Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest);
+
+Status SendSealReply(int sock, ObjectID object_id, int error);
+
+Status ReadSealReply(uint8_t* data, ObjectID* object_id);
+
+/* Plasma Get message functions. */
+
+Status SendGetRequest(
+ int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms);
+
+Status ReadGetRequest(
+ uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms);
+
+Status SendGetReply(int sock, ObjectID object_ids[],
+ std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
+ int64_t num_objects);
+
+Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
+ int64_t num_objects);
+
+/* Plasma Release message functions. */
+
+Status SendReleaseRequest(int sock, ObjectID object_id);
+
+Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendReleaseReply(int sock, ObjectID object_id, int error);
+
+Status ReadReleaseReply(uint8_t* data, ObjectID* object_id);
+
+/* Plasma Delete message functions. */
+
+Status SendDeleteRequest(int sock, ObjectID object_id);
+
+Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendDeleteReply(int sock, ObjectID object_id, int error);
+
+Status ReadDeleteReply(uint8_t* data, ObjectID* object_id);
+
+/* Satus messages. */
+
+Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
+
+Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects);
+
+Status SendStatusReply(
+ int sock, ObjectID object_ids[], int object_status[], int64_t num_objects);
+
+int64_t ReadStatusReply_num_objects(uint8_t* data);
+
+Status ReadStatusReply(
+ uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects);
+
+/* Plasma Constains message functions. */
+
+Status SendContainsRequest(int sock, ObjectID object_id);
+
+Status ReadContainsRequest(uint8_t* data, ObjectID* object_id);
+
+Status SendContainsReply(int sock, ObjectID object_id, bool has_object);
+
+Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object);
+
+/* Plasma Connect message functions. */
+
+Status SendConnectRequest(int sock);
+
+Status ReadConnectRequest(uint8_t* data);
+
+Status SendConnectReply(int sock, int64_t memory_capacity);
+
+Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity);
+
+/* Plasma Evict message functions (no reply so far). */
+
+Status SendEvictRequest(int sock, int64_t num_bytes);
+
+Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes);
+
+Status SendEvictReply(int sock, int64_t num_bytes);
+
+Status ReadEvictReply(uint8_t* data, int64_t& num_bytes);
+
+/* Plasma Fetch Remote message functions. */
+
+Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
+
+Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids);
+
+/* Plasma Wait message functions. */
+
+Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
+ int num_ready_objects, int64_t timeout_ms);
+
+Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
+ int64_t* timeout_ms, int* num_ready_objects);
+
+Status SendWaitReply(
+ int sock, const ObjectRequestMap& object_requests, int num_ready_objects);
+
+Status ReadWaitReply(
+ uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects);
+
+/* Plasma Subscribe message functions. */
+
+Status SendSubscribeRequest(int sock);
+
+/* Data messages. */
+
+Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port);
+
+Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port);
+
+Status SendDataReply(
+ int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size);
+
+Status ReadDataReply(
+ uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size);
+
+#endif /* PLASMA_PROTOCOL */
[08/14] arrow git commit: [maven-release-plugin] prepare for next
development iteration
Posted by we...@apache.org.
[maven-release-plugin] prepare for next development iteration
Change-Id: I8fff2322e3b3feb6e80cfc2be1d5743e9ec9bbb7
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/9b26ed84
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/9b26ed84
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/9b26ed84
Branch: refs/heads/master
Commit: 9b26ed84e144a2c251a8e163d88302a5bdaf7dfd
Parents: e9f76e1
Author: Wes McKinney <we...@twosigma.com>
Authored: Thu Jul 20 13:02:51 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Thu Jul 20 13:02:51 2017 -0400
----------------------------------------------------------------------
java/format/pom.xml | 2 +-
java/memory/pom.xml | 2 +-
java/pom.xml | 4 ++--
java/tools/pom.xml | 2 +-
java/vector/pom.xml | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/9b26ed84/java/format/pom.xml
----------------------------------------------------------------------
diff --git a/java/format/pom.xml b/java/format/pom.xml
index 6b962e2..af8ff60 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>arrow-java-root</artifactId>
<groupId>org.apache.arrow</groupId>
- <version>0.5.0</version>
+ <version>0.6.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-format</artifactId>
http://git-wip-us.apache.org/repos/asf/arrow/blob/9b26ed84/java/memory/pom.xml
----------------------------------------------------------------------
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index e204cd6..9a8d2d7 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0</version>
+ <version>0.6.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-memory</artifactId>
<name>Arrow Memory</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/9b26ed84/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 7eaf27d..2613a44 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0</version>
+ <version>0.6.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
<url>https://github.com/apache/arrow</url>
- <tag>apache-arrow-0.5.0</tag>
+ <tag>HEAD</tag>
</scm>
<mailingLists>
http://git-wip-us.apache.org/repos/asf/arrow/blob/9b26ed84/java/tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 6d28c9f..9d067ef 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0</version>
+ <version>0.6.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-tools</artifactId>
<name>Arrow Tools</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/9b26ed84/java/vector/pom.xml
----------------------------------------------------------------------
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index c484935..e15ab9a 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0</version>
+ <version>0.6.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-vector</artifactId>
<name>Arrow Vectors</name>
[02/14] arrow git commit: [C++] Remove Plasma source tree for 0.5.0
release pending IP Clearance
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/dlmalloc.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/dlmalloc.c b/cpp/src/plasma/thirdparty/dlmalloc.c
deleted file mode 100644
index 84ccbd2..0000000
--- a/cpp/src/plasma/thirdparty/dlmalloc.c
+++ /dev/null
@@ -1,6281 +0,0 @@
-/*
- This is a version (aka dlmalloc) of malloc/free/realloc written by
- Doug Lea and released to the public domain, as explained at
- http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
- comments, complaints, performance data, etc to dl@cs.oswego.edu
-
-* Version 2.8.6 Wed Aug 29 06:57:58 2012 Doug Lea
- Note: There may be an updated version of this malloc obtainable at
- ftp://gee.cs.oswego.edu/pub/misc/malloc.c
- Check before installing!
-
-* Quickstart
-
- This library is all in one file to simplify the most common usage:
- ftp it, compile it (-O3), and link it into another program. All of
- the compile-time options default to reasonable values for use on
- most platforms. You might later want to step through various
- compile-time and dynamic tuning options.
-
- For convenience, an include file for code using this malloc is at:
- ftp://gee.cs.oswego.edu/pub/misc/malloc-2.8.6.h
- You don't really need this .h file unless you call functions not
- defined in your system include files. The .h file contains only the
- excerpts from this file needed for using this malloc on ANSI C/C++
- systems, so long as you haven't changed compile-time options about
- naming and tuning parameters. If you do, then you can create your
- own malloc.h that does include all settings by cutting at the point
- indicated below. Note that you may already by default be using a C
- library containing a malloc that is based on some version of this
- malloc (for example in linux). You might still want to use the one
- in this file to customize settings or to avoid overheads associated
- with library versions.
-
-* Vital statistics:
-
- Supported pointer/size_t representation: 4 or 8 bytes
- size_t MUST be an unsigned type of the same width as
- pointers. (If you are using an ancient system that declares
- size_t as a signed type, or need it to be a different width
- than pointers, you can use a previous release of this malloc
- (e.g. 2.7.2) supporting these.)
-
- Alignment: 8 bytes (minimum)
- This suffices for nearly all current machines and C compilers.
- However, you can define MALLOC_ALIGNMENT to be wider than this
- if necessary (up to 128bytes), at the expense of using more space.
-
- Minimum overhead per allocated chunk: 4 or 8 bytes (if 4byte sizes)
- 8 or 16 bytes (if 8byte sizes)
- Each malloced chunk has a hidden word of overhead holding size
- and status information, and additional cross-check word
- if FOOTERS is defined.
-
- Minimum allocated size: 4-byte ptrs: 16 bytes (including overhead)
- 8-byte ptrs: 32 bytes (including overhead)
-
- Even a request for zero bytes (i.e., malloc(0)) returns a
- pointer to something of the minimum allocatable size.
- The maximum overhead wastage (i.e., number of extra bytes
- allocated than were requested in malloc) is less than or equal
- to the minimum size, except for requests >= mmap_threshold that
- are serviced via mmap(), where the worst case wastage is about
- 32 bytes plus the remainder from a system page (the minimal
- mmap unit); typically 4096 or 8192 bytes.
-
- Security: static-safe; optionally more or less
- The "security" of malloc refers to the ability of malicious
- code to accentuate the effects of errors (for example, freeing
- space that is not currently malloc'ed or overwriting past the
- ends of chunks) in code that calls malloc. This malloc
- guarantees not to modify any memory locations below the base of
- heap, i.e., static variables, even in the presence of usage
- errors. The routines additionally detect most improper frees
- and reallocs. All this holds as long as the static bookkeeping
- for malloc itself is not corrupted by some other means. This
- is only one aspect of security -- these checks do not, and
- cannot, detect all possible programming errors.
-
- If FOOTERS is defined nonzero, then each allocated chunk
- carries an additional check word to verify that it was malloced
- from its space. These check words are the same within each
- execution of a program using malloc, but differ across
- executions, so externally crafted fake chunks cannot be
- freed. This improves security by rejecting frees/reallocs that
- could corrupt heap memory, in addition to the checks preventing
- writes to statics that are always on. This may further improve
- security at the expense of time and space overhead. (Note that
- FOOTERS may also be worth using with MSPACES.)
-
- By default detected errors cause the program to abort (calling
- "abort()"). You can override this to instead proceed past
- errors by defining PROCEED_ON_ERROR. In this case, a bad free
- has no effect, and a malloc that encounters a bad address
- caused by user overwrites will ignore the bad address by
- dropping pointers and indices to all known memory. This may
- be appropriate for programs that should continue if at all
- possible in the face of programming errors, although they may
- run out of memory because dropped memory is never reclaimed.
-
- If you don't like either of these options, you can define
- CORRUPTION_ERROR_ACTION and USAGE_ERROR_ACTION to do anything
- else. And if if you are sure that your program using malloc has
- no errors or vulnerabilities, you can define INSECURE to 1,
- which might (or might not) provide a small performance improvement.
-
- It is also possible to limit the maximum total allocatable
- space, using malloc_set_footprint_limit. This is not
- designed as a security feature in itself (calls to set limits
- are not screened or privileged), but may be useful as one
- aspect of a secure implementation.
-
- Thread-safety: NOT thread-safe unless USE_LOCKS defined non-zero
- When USE_LOCKS is defined, each public call to malloc, free,
- etc is surrounded with a lock. By default, this uses a plain
- pthread mutex, win32 critical section, or a spin-lock if if
- available for the platform and not disabled by setting
- USE_SPIN_LOCKS=0. However, if USE_RECURSIVE_LOCKS is defined,
- recursive versions are used instead (which are not required for
- base functionality but may be needed in layered extensions).
- Using a global lock is not especially fast, and can be a major
- bottleneck. It is designed only to provide minimal protection
- in concurrent environments, and to provide a basis for
- extensions. If you are using malloc in a concurrent program,
- consider instead using nedmalloc
- (http://www.nedprod.com/programs/portable/nedmalloc/) or
- ptmalloc (See http://www.malloc.de), which are derived from
- versions of this malloc.
-
- System requirements: Any combination of MORECORE and/or MMAP/MUNMAP
- This malloc can use unix sbrk or any emulation (invoked using
- the CALL_MORECORE macro) and/or mmap/munmap or any emulation
- (invoked using CALL_MMAP/CALL_MUNMAP) to get and release system
- memory. On most unix systems, it tends to work best if both
- MORECORE and MMAP are enabled. On Win32, it uses emulations
- based on VirtualAlloc. It also uses common C library functions
- like memset.
-
- Compliance: I believe it is compliant with the Single Unix Specification
- (See http://www.unix.org). Also SVID/XPG, ANSI C, and probably
- others as well.
-
-* Overview of algorithms
-
- This is not the fastest, most space-conserving, most portable, or
- most tunable malloc ever written. However it is among the fastest
- while also being among the most space-conserving, portable and
- tunable. Consistent balance across these factors results in a good
- general-purpose allocator for malloc-intensive programs.
-
- In most ways, this malloc is a best-fit allocator. Generally, it
- chooses the best-fitting existing chunk for a request, with ties
- broken in approximately least-recently-used order. (This strategy
- normally maintains low fragmentation.) However, for requests less
- than 256bytes, it deviates from best-fit when there is not an
- exactly fitting available chunk by preferring to use space adjacent
- to that used for the previous small request, as well as by breaking
- ties in approximately most-recently-used order. (These enhance
- locality of series of small allocations.) And for very large requests
- (>= 256Kb by default), it relies on system memory mapping
- facilities, if supported. (This helps avoid carrying around and
- possibly fragmenting memory used only for large chunks.)
-
- All operations (except malloc_stats and mallinfo) have execution
- times that are bounded by a constant factor of the number of bits in
- a size_t, not counting any clearing in calloc or copying in realloc,
- or actions surrounding MORECORE and MMAP that have times
- proportional to the number of non-contiguous regions returned by
- system allocation routines, which is often just 1. In real-time
- applications, you can optionally suppress segment traversals using
- NO_SEGMENT_TRAVERSAL, which assures bounded execution even when
- system allocators return non-contiguous spaces, at the typical
- expense of carrying around more memory and increased fragmentation.
-
- The implementation is not very modular and seriously overuses
- macros. Perhaps someday all C compilers will do as good a job
- inlining modular code as can now be done by brute-force expansion,
- but now, enough of them seem not to.
-
- Some compilers issue a lot of warnings about code that is
- dead/unreachable only on some platforms, and also about intentional
- uses of negation on unsigned types. All known cases of each can be
- ignored.
-
- For a longer but out of date high-level description, see
- http://gee.cs.oswego.edu/dl/html/malloc.html
-
-* MSPACES
- If MSPACES is defined, then in addition to malloc, free, etc.,
- this file also defines mspace_malloc, mspace_free, etc. These
- are versions of malloc routines that take an "mspace" argument
- obtained using create_mspace, to control all internal bookkeeping.
- If ONLY_MSPACES is defined, only these versions are compiled.
- So if you would like to use this allocator for only some allocations,
- and your system malloc for others, you can compile with
- ONLY_MSPACES and then do something like...
- static mspace mymspace = create_mspace(0,0); // for example
- #define mymalloc(bytes) mspace_malloc(mymspace, bytes)
-
- (Note: If you only need one instance of an mspace, you can instead
- use "USE_DL_PREFIX" to relabel the global malloc.)
-
- You can similarly create thread-local allocators by storing
- mspaces as thread-locals. For example:
- static __thread mspace tlms = 0;
- void* tlmalloc(size_t bytes) {
- if (tlms == 0) tlms = create_mspace(0, 0);
- return mspace_malloc(tlms, bytes);
- }
- void tlfree(void* mem) { mspace_free(tlms, mem); }
-
- Unless FOOTERS is defined, each mspace is completely independent.
- You cannot allocate from one and free to another (although
- conformance is only weakly checked, so usage errors are not always
- caught). If FOOTERS is defined, then each chunk carries around a tag
- indicating its originating mspace, and frees are directed to their
- originating spaces. Normally, this requires use of locks.
-
- ------------------------- Compile-time options ---------------------------
-
-Be careful in setting #define values for numerical constants of type
-size_t. On some systems, literal values are not automatically extended
-to size_t precision unless they are explicitly casted. You can also
-use the symbolic values MAX_SIZE_T, SIZE_T_ONE, etc below.
-
-WIN32 default: defined if _WIN32 defined
- Defining WIN32 sets up defaults for MS environment and compilers.
- Otherwise defaults are for unix. Beware that there seem to be some
- cases where this malloc might not be a pure drop-in replacement for
- Win32 malloc: Random-looking failures from Win32 GDI API's (eg;
- SetDIBits()) may be due to bugs in some video driver implementations
- when pixel buffers are malloc()ed, and the region spans more than
- one VirtualAlloc()ed region. Because dlmalloc uses a small (64Kb)
- default granularity, pixel buffers may straddle virtual allocation
- regions more often than when using the Microsoft allocator. You can
- avoid this by using VirtualAlloc() and VirtualFree() for all pixel
- buffers rather than using malloc(). If this is not possible,
- recompile this malloc with a larger DEFAULT_GRANULARITY. Note:
- in cases where MSC and gcc (cygwin) are known to differ on WIN32,
- conditions use _MSC_VER to distinguish them.
-
-DLMALLOC_EXPORT default: extern
- Defines how public APIs are declared. If you want to export via a
- Windows DLL, you might define this as
- #define DLMALLOC_EXPORT extern __declspec(dllexport)
- If you want a POSIX ELF shared object, you might use
- #define DLMALLOC_EXPORT extern __attribute__((visibility("default")))
-
-MALLOC_ALIGNMENT default: (size_t)(2 * sizeof(void *))
- Controls the minimum alignment for malloc'ed chunks. It must be a
- power of two and at least 8, even on machines for which smaller
- alignments would suffice. It may be defined as larger than this
- though. Note however that code and data structures are optimized for
- the case of 8-byte alignment.
-
-MSPACES default: 0 (false)
- If true, compile in support for independent allocation spaces.
- This is only supported if HAVE_MMAP is true.
-
-ONLY_MSPACES default: 0 (false)
- If true, only compile in mspace versions, not regular versions.
-
-USE_LOCKS default: 0 (false)
- Causes each call to each public routine to be surrounded with
- pthread or WIN32 mutex lock/unlock. (If set true, this can be
- overridden on a per-mspace basis for mspace versions.) If set to a
- non-zero value other than 1, locks are used, but their
- implementation is left out, so lock functions must be supplied manually,
- as described below.
-
-USE_SPIN_LOCKS default: 1 iff USE_LOCKS and spin locks available
- If true, uses custom spin locks for locking. This is currently
- supported only gcc >= 4.1, older gccs on x86 platforms, and recent
- MS compilers. Otherwise, posix locks or win32 critical sections are
- used.
-
-USE_RECURSIVE_LOCKS default: not defined
- If defined nonzero, uses recursive (aka reentrant) locks, otherwise
- uses plain mutexes. This is not required for malloc proper, but may
- be needed for layered allocators such as nedmalloc.
-
-LOCK_AT_FORK default: not defined
- If defined nonzero, performs pthread_atfork upon initialization
- to initialize child lock while holding parent lock. The implementation
- assumes that pthread locks (not custom locks) are being used. In other
- cases, you may need to customize the implementation.
-
-FOOTERS default: 0
- If true, provide extra checking and dispatching by placing
- information in the footers of allocated chunks. This adds
- space and time overhead.
-
-INSECURE default: 0
- If true, omit checks for usage errors and heap space overwrites.
-
-USE_DL_PREFIX default: NOT defined
- Causes compiler to prefix all public routines with the string 'dl'.
- This can be useful when you only want to use this malloc in one part
- of a program, using your regular system malloc elsewhere.
-
-MALLOC_INSPECT_ALL default: NOT defined
- If defined, compiles malloc_inspect_all and mspace_inspect_all, that
- perform traversal of all heap space. Unless access to these
- functions is otherwise restricted, you probably do not want to
- include them in secure implementations.
-
-ABORT default: defined as abort()
- Defines how to abort on failed checks. On most systems, a failed
- check cannot die with an "assert" or even print an informative
- message, because the underlying print routines in turn call malloc,
- which will fail again. Generally, the best policy is to simply call
- abort(). It's not very useful to do more than this because many
- errors due to overwriting will show up as address faults (null, odd
- addresses etc) rather than malloc-triggered checks, so will also
- abort. Also, most compilers know that abort() does not return, so
- can better optimize code conditionally calling it.
-
-PROCEED_ON_ERROR default: defined as 0 (false)
- Controls whether detected bad addresses cause them to bypassed
- rather than aborting. If set, detected bad arguments to free and
- realloc are ignored. And all bookkeeping information is zeroed out
- upon a detected overwrite of freed heap space, thus losing the
- ability to ever return it from malloc again, but enabling the
- application to proceed. If PROCEED_ON_ERROR is defined, the
- static variable malloc_corruption_error_count is compiled in
- and can be examined to see if errors have occurred. This option
- generates slower code than the default abort policy.
-
-DEBUG default: NOT defined
- The DEBUG setting is mainly intended for people trying to modify
- this code or diagnose problems when porting to new platforms.
- However, it may also be able to better isolate user errors than just
- using runtime checks. The assertions in the check routines spell
- out in more detail the assumptions and invariants underlying the
- algorithms. The checking is fairly extensive, and will slow down
- execution noticeably. Calling malloc_stats or mallinfo with DEBUG
- set will attempt to check every non-mmapped allocated and free chunk
- in the course of computing the summaries.
-
-ABORT_ON_ASSERT_FAILURE default: defined as 1 (true)
- Debugging assertion failures can be nearly impossible if your
- version of the assert macro causes malloc to be called, which will
- lead to a cascade of further failures, blowing the runtime stack.
- ABORT_ON_ASSERT_FAILURE cause assertions failures to call abort(),
- which will usually make debugging easier.
-
-MALLOC_FAILURE_ACTION default: sets errno to ENOMEM, or no-op on win32
- The action to take before "return 0" when malloc fails to be able to
- return memory because there is none available.
-
-HAVE_MORECORE default: 1 (true) unless win32 or ONLY_MSPACES
- True if this system supports sbrk or an emulation of it.
-
-MORECORE default: sbrk
- The name of the sbrk-style system routine to call to obtain more
- memory. See below for guidance on writing custom MORECORE
- functions. The type of the argument to sbrk/MORECORE varies across
- systems. It cannot be size_t, because it supports negative
- arguments, so it is normally the signed type of the same width as
- size_t (sometimes declared as "intptr_t"). It doesn't much matter
- though. Internally, we only call it with arguments less than half
- the max value of a size_t, which should work across all reasonable
- possibilities, although sometimes generating compiler warnings.
-
-MORECORE_CONTIGUOUS default: 1 (true) if HAVE_MORECORE
- If true, take advantage of fact that consecutive calls to MORECORE
- with positive arguments always return contiguous increasing
- addresses. This is true of unix sbrk. It does not hurt too much to
- set it true anyway, since malloc copes with non-contiguities.
- Setting it false when definitely non-contiguous saves time
- and possibly wasted space it would take to discover this though.
-
-MORECORE_CANNOT_TRIM default: NOT defined
- True if MORECORE cannot release space back to the system when given
- negative arguments. This is generally necessary only if you are
- using a hand-crafted MORECORE function that cannot handle negative
- arguments.
-
-NO_SEGMENT_TRAVERSAL default: 0
- If non-zero, suppresses traversals of memory segments
- returned by either MORECORE or CALL_MMAP. This disables
- merging of segments that are contiguous, and selectively
- releasing them to the OS if unused, but bounds execution times.
-
-HAVE_MMAP default: 1 (true)
- True if this system supports mmap or an emulation of it. If so, and
- HAVE_MORECORE is not true, MMAP is used for all system
- allocation. If set and HAVE_MORECORE is true as well, MMAP is
- primarily used to directly allocate very large blocks. It is also
- used as a backup strategy in cases where MORECORE fails to provide
- space from system. Note: A single call to MUNMAP is assumed to be
- able to unmap memory that may have be allocated using multiple calls
- to MMAP, so long as they are adjacent.
-
-HAVE_MREMAP default: 1 on linux, else 0
- If true realloc() uses mremap() to re-allocate large blocks and
- extend or shrink allocation spaces.
-
-MMAP_CLEARS default: 1 except on WINCE.
- True if mmap clears memory so calloc doesn't need to. This is true
- for standard unix mmap using /dev/zero and on WIN32 except for WINCE.
-
-USE_BUILTIN_FFS default: 0 (i.e., not used)
- Causes malloc to use the builtin ffs() function to compute indices.
- Some compilers may recognize and intrinsify ffs to be faster than the
- supplied C version. Also, the case of x86 using gcc is special-cased
- to an asm instruction, so is already as fast as it can be, and so
- this setting has no effect. Similarly for Win32 under recent MS compilers.
- (On most x86s, the asm version is only slightly faster than the C version.)
-
-malloc_getpagesize default: derive from system includes, or 4096.
- The system page size. To the extent possible, this malloc manages
- memory from the system in page-size units. This may be (and
- usually is) a function rather than a constant. This is ignored
- if WIN32, where page size is determined using getSystemInfo during
- initialization.
-
-USE_DEV_RANDOM default: 0 (i.e., not used)
- Causes malloc to use /dev/random to initialize secure magic seed for
- stamping footers. Otherwise, the current time is used.
-
-NO_MALLINFO default: 0
- If defined, don't compile "mallinfo". This can be a simple way
- of dealing with mismatches between system declarations and
- those in this file.
-
-MALLINFO_FIELD_TYPE default: size_t
- The type of the fields in the mallinfo struct. This was originally
- defined as "int" in SVID etc, but is more usefully defined as
- size_t. The value is used only if HAVE_USR_INCLUDE_MALLOC_H is not set
-
-NO_MALLOC_STATS default: 0
- If defined, don't compile "malloc_stats". This avoids calls to
- fprintf and bringing in stdio dependencies you might not want.
-
-REALLOC_ZERO_BYTES_FREES default: not defined
- This should be set if a call to realloc with zero bytes should
- be the same as a call to free. Some people think it should. Otherwise,
- since this malloc returns a unique pointer for malloc(0), so does
- realloc(p, 0).
-
-LACKS_UNISTD_H, LACKS_FCNTL_H, LACKS_SYS_PARAM_H, LACKS_SYS_MMAN_H
-LACKS_STRINGS_H, LACKS_STRING_H, LACKS_SYS_TYPES_H, LACKS_ERRNO_H
-LACKS_STDLIB_H LACKS_SCHED_H LACKS_TIME_H default: NOT defined unless on WIN32
- Define these if your system does not have these header files.
- You might need to manually insert some of the declarations they provide.
-
-DEFAULT_GRANULARITY default: page size if MORECORE_CONTIGUOUS,
- system_info.dwAllocationGranularity in WIN32,
- otherwise 64K.
- Also settable using mallopt(M_GRANULARITY, x)
- The unit for allocating and deallocating memory from the system. On
- most systems with contiguous MORECORE, there is no reason to
- make this more than a page. However, systems with MMAP tend to
- either require or encourage larger granularities. You can increase
- this value to prevent system allocation functions to be called so
- often, especially if they are slow. The value must be at least one
- page and must be a power of two. Setting to 0 causes initialization
- to either page size or win32 region size. (Note: In previous
- versions of malloc, the equivalent of this option was called
- "TOP_PAD")
-
-DEFAULT_TRIM_THRESHOLD default: 2MB
- Also settable using mallopt(M_TRIM_THRESHOLD, x)
- The maximum amount of unused top-most memory to keep before
- releasing via malloc_trim in free(). Automatic trimming is mainly
- useful in long-lived programs using contiguous MORECORE. Because
- trimming via sbrk can be slow on some systems, and can sometimes be
- wasteful (in cases where programs immediately afterward allocate
- more large chunks) the value should be high enough so that your
- overall system performance would improve by releasing this much
- memory. As a rough guide, you might set to a value close to the
- average size of a process (program) running on your system.
- Releasing this much memory would allow such a process to run in
- memory. Generally, it is worth tuning trim thresholds when a
- program undergoes phases where several large chunks are allocated
- and released in ways that can reuse each other's storage, perhaps
- mixed with phases where there are no such chunks at all. The trim
- value must be greater than page size to have any useful effect. To
- disable trimming completely, you can set to MAX_SIZE_T. Note that the trick
- some people use of mallocing a huge space and then freeing it at
- program startup, in an attempt to reserve system memory, doesn't
- have the intended effect under automatic trimming, since that memory
- will immediately be returned to the system.
-
-DEFAULT_MMAP_THRESHOLD default: 256K
- Also settable using mallopt(M_MMAP_THRESHOLD, x)
- The request size threshold for using MMAP to directly service a
- request. Requests of at least this size that cannot be allocated
- using already-existing space will be serviced via mmap. (If enough
- normal freed space already exists it is used instead.) Using mmap
- segregates relatively large chunks of memory so that they can be
- individually obtained and released from the host system. A request
- serviced through mmap is never reused by any other request (at least
- not directly; the system may just so happen to remap successive
- requests to the same locations). Segregating space in this way has
- the benefits that: Mmapped space can always be individually released
- back to the system, which helps keep the system level memory demands
- of a long-lived program low. Also, mapped memory doesn't become
- `locked' between other chunks, as can happen with normally allocated
- chunks, which means that even trimming via malloc_trim would not
- release them. However, it has the disadvantage that the space
- cannot be reclaimed, consolidated, and then used to service later
- requests, as happens with normal chunks. The advantages of mmap
- nearly always outweigh disadvantages for "large" chunks, but the
- value of "large" may vary across systems. The default is an
- empirically derived value that works well in most systems. You can
- disable mmap by setting to MAX_SIZE_T.
-
-MAX_RELEASE_CHECK_RATE default: 4095 unless not HAVE_MMAP
- The number of consolidated frees between checks to release
- unused segments when freeing. When using non-contiguous segments,
- especially with multiple mspaces, checking only for topmost space
- doesn't always suffice to trigger trimming. To compensate for this,
- free() will, with a period of MAX_RELEASE_CHECK_RATE (or the
- current number of segments, if greater) try to release unused
- segments to the OS when freeing chunks that result in
- consolidation. The best value for this parameter is a compromise
- between slowing down frees with relatively costly checks that
- rarely trigger versus holding on to unused memory. To effectively
- disable, set to MAX_SIZE_T. This may lead to a very slight speed
- improvement at the expense of carrying around more memory.
-*/
-
-/* Version identifier to allow people to support multiple versions */
-#ifndef DLMALLOC_VERSION
-#define DLMALLOC_VERSION 20806
-#endif /* DLMALLOC_VERSION */
-
-#ifndef DLMALLOC_EXPORT
-#define DLMALLOC_EXPORT extern
-#endif
-
-#ifndef WIN32
-#ifdef _WIN32
-#define WIN32 1
-#endif /* _WIN32 */
-#ifdef _WIN32_WCE
-#define LACKS_FCNTL_H
-#define WIN32 1
-#endif /* _WIN32_WCE */
-#endif /* WIN32 */
-#ifdef WIN32
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <tchar.h>
-#define HAVE_MMAP 1
-#define HAVE_MORECORE 0
-#define LACKS_UNISTD_H
-#define LACKS_SYS_PARAM_H
-#define LACKS_SYS_MMAN_H
-#define LACKS_STRING_H
-#define LACKS_STRINGS_H
-#define LACKS_SYS_TYPES_H
-#define LACKS_ERRNO_H
-#define LACKS_SCHED_H
-#ifndef MALLOC_FAILURE_ACTION
-#define MALLOC_FAILURE_ACTION
-#endif /* MALLOC_FAILURE_ACTION */
-#ifndef MMAP_CLEARS
-#ifdef _WIN32_WCE /* WINCE reportedly does not clear */
-#define MMAP_CLEARS 0
-#else
-#define MMAP_CLEARS 1
-#endif /* _WIN32_WCE */
-#endif /*MMAP_CLEARS */
-#endif /* WIN32 */
-
-#if defined(DARWIN) || defined(_DARWIN)
-/* Mac OSX docs advise not to use sbrk; it seems better to use mmap */
-#ifndef HAVE_MORECORE
-#define HAVE_MORECORE 0
-#define HAVE_MMAP 1
-/* OSX allocators provide 16 byte alignment */
-#ifndef MALLOC_ALIGNMENT
-#define MALLOC_ALIGNMENT ((size_t)16U)
-#endif
-#endif /* HAVE_MORECORE */
-#endif /* DARWIN */
-
-#ifndef LACKS_SYS_TYPES_H
-#include <sys/types.h> /* For size_t */
-#endif /* LACKS_SYS_TYPES_H */
-
-/* The maximum possible size_t value has all bits set */
-#define MAX_SIZE_T (~(size_t)0)
-
-#ifndef USE_LOCKS /* ensure true if spin or recursive locks set */
-#define USE_LOCKS ((defined(USE_SPIN_LOCKS) && USE_SPIN_LOCKS != 0) || \
- (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0))
-#endif /* USE_LOCKS */
-
-#if USE_LOCKS /* Spin locks for gcc >= 4.1, older gcc on x86, MSC >= 1310 */
-#if ((defined(__GNUC__) && \
- ((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) || \
- defined(__i386__) || defined(__x86_64__))) || \
- (defined(_MSC_VER) && _MSC_VER>=1310))
-#ifndef USE_SPIN_LOCKS
-#define USE_SPIN_LOCKS 1
-#endif /* USE_SPIN_LOCKS */
-#elif USE_SPIN_LOCKS
-#error "USE_SPIN_LOCKS defined without implementation"
-#endif /* ... locks available... */
-#elif !defined(USE_SPIN_LOCKS)
-#define USE_SPIN_LOCKS 0
-#endif /* USE_LOCKS */
-
-#ifndef ONLY_MSPACES
-#define ONLY_MSPACES 0
-#endif /* ONLY_MSPACES */
-#ifndef MSPACES
-#if ONLY_MSPACES
-#define MSPACES 1
-#else /* ONLY_MSPACES */
-#define MSPACES 0
-#endif /* ONLY_MSPACES */
-#endif /* MSPACES */
-#ifndef MALLOC_ALIGNMENT
-#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *)))
-#endif /* MALLOC_ALIGNMENT */
-#ifndef FOOTERS
-#define FOOTERS 0
-#endif /* FOOTERS */
-#ifndef ABORT
-#define ABORT abort()
-#endif /* ABORT */
-#ifndef ABORT_ON_ASSERT_FAILURE
-#define ABORT_ON_ASSERT_FAILURE 1
-#endif /* ABORT_ON_ASSERT_FAILURE */
-#ifndef PROCEED_ON_ERROR
-#define PROCEED_ON_ERROR 0
-#endif /* PROCEED_ON_ERROR */
-
-#ifndef INSECURE
-#define INSECURE 0
-#endif /* INSECURE */
-#ifndef MALLOC_INSPECT_ALL
-#define MALLOC_INSPECT_ALL 0
-#endif /* MALLOC_INSPECT_ALL */
-#ifndef HAVE_MMAP
-#define HAVE_MMAP 1
-#endif /* HAVE_MMAP */
-#ifndef MMAP_CLEARS
-#define MMAP_CLEARS 1
-#endif /* MMAP_CLEARS */
-#ifndef HAVE_MREMAP
-#ifdef linux
-#define HAVE_MREMAP 1
-#define _GNU_SOURCE /* Turns on mremap() definition */
-#else /* linux */
-#define HAVE_MREMAP 0
-#endif /* linux */
-#endif /* HAVE_MREMAP */
-#ifndef MALLOC_FAILURE_ACTION
-#define MALLOC_FAILURE_ACTION errno = ENOMEM;
-#endif /* MALLOC_FAILURE_ACTION */
-#ifndef HAVE_MORECORE
-#if ONLY_MSPACES
-#define HAVE_MORECORE 0
-#else /* ONLY_MSPACES */
-#define HAVE_MORECORE 1
-#endif /* ONLY_MSPACES */
-#endif /* HAVE_MORECORE */
-#if !HAVE_MORECORE
-#define MORECORE_CONTIGUOUS 0
-#else /* !HAVE_MORECORE */
-#define MORECORE_DEFAULT sbrk
-#ifndef MORECORE_CONTIGUOUS
-#define MORECORE_CONTIGUOUS 1
-#endif /* MORECORE_CONTIGUOUS */
-#endif /* HAVE_MORECORE */
-#ifndef DEFAULT_GRANULARITY
-#if (MORECORE_CONTIGUOUS || defined(WIN32))
-#define DEFAULT_GRANULARITY (0) /* 0 means to compute in init_mparams */
-#else /* MORECORE_CONTIGUOUS */
-#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U)
-#endif /* MORECORE_CONTIGUOUS */
-#endif /* DEFAULT_GRANULARITY */
-#ifndef DEFAULT_TRIM_THRESHOLD
-#ifndef MORECORE_CANNOT_TRIM
-#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U)
-#else /* MORECORE_CANNOT_TRIM */
-#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T
-#endif /* MORECORE_CANNOT_TRIM */
-#endif /* DEFAULT_TRIM_THRESHOLD */
-#ifndef DEFAULT_MMAP_THRESHOLD
-#if HAVE_MMAP
-#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U)
-#else /* HAVE_MMAP */
-#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
-#endif /* HAVE_MMAP */
-#endif /* DEFAULT_MMAP_THRESHOLD */
-#ifndef MAX_RELEASE_CHECK_RATE
-#if HAVE_MMAP
-#define MAX_RELEASE_CHECK_RATE 4095
-#else
-#define MAX_RELEASE_CHECK_RATE MAX_SIZE_T
-#endif /* HAVE_MMAP */
-#endif /* MAX_RELEASE_CHECK_RATE */
-#ifndef USE_BUILTIN_FFS
-#define USE_BUILTIN_FFS 0
-#endif /* USE_BUILTIN_FFS */
-#ifndef USE_DEV_RANDOM
-#define USE_DEV_RANDOM 0
-#endif /* USE_DEV_RANDOM */
-#ifndef NO_MALLINFO
-#define NO_MALLINFO 0
-#endif /* NO_MALLINFO */
-#ifndef MALLINFO_FIELD_TYPE
-#define MALLINFO_FIELD_TYPE size_t
-#endif /* MALLINFO_FIELD_TYPE */
-#ifndef NO_MALLOC_STATS
-#define NO_MALLOC_STATS 0
-#endif /* NO_MALLOC_STATS */
-#ifndef NO_SEGMENT_TRAVERSAL
-#define NO_SEGMENT_TRAVERSAL 0
-#endif /* NO_SEGMENT_TRAVERSAL */
-
-/*
- mallopt tuning options. SVID/XPG defines four standard parameter
- numbers for mallopt, normally defined in malloc.h. None of these
- are used in this malloc, so setting them has no effect. But this
- malloc does support the following options.
-*/
-
-#define M_TRIM_THRESHOLD (-1)
-#define M_GRANULARITY (-2)
-#define M_MMAP_THRESHOLD (-3)
-
-/* ------------------------ Mallinfo declarations ------------------------ */
-
-#if !NO_MALLINFO
-/*
- This version of malloc supports the standard SVID/XPG mallinfo
- routine that returns a struct containing usage properties and
- statistics. It should work on any system that has a
- /usr/include/malloc.h defining struct mallinfo. The main
- declaration needed is the mallinfo struct that is returned (by-copy)
- by mallinfo(). The malloinfo struct contains a bunch of fields that
- are not even meaningful in this version of malloc. These fields are
- are instead filled by mallinfo() with other numbers that might be of
- interest.
-
- HAVE_USR_INCLUDE_MALLOC_H should be set if you have a
- /usr/include/malloc.h file that includes a declaration of struct
- mallinfo. If so, it is included; else a compliant version is
- declared below. These must be precisely the same for mallinfo() to
- work. The original SVID version of this struct, defined on most
- systems with mallinfo, declares all fields as ints. But some others
- define as unsigned long. If your system defines the fields using a
- type of different width than listed here, you MUST #include your
- system version and #define HAVE_USR_INCLUDE_MALLOC_H.
-*/
-
-/* #define HAVE_USR_INCLUDE_MALLOC_H */
-
-#ifdef HAVE_USR_INCLUDE_MALLOC_H
-#include "/usr/include/malloc.h"
-#else /* HAVE_USR_INCLUDE_MALLOC_H */
-#ifndef STRUCT_MALLINFO_DECLARED
-/* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is defined */
-#define _STRUCT_MALLINFO
-#define STRUCT_MALLINFO_DECLARED 1
-struct mallinfo {
- MALLINFO_FIELD_TYPE arena; /* non-mmapped space allocated from system */
- MALLINFO_FIELD_TYPE ordblks; /* number of free chunks */
- MALLINFO_FIELD_TYPE smblks; /* always 0 */
- MALLINFO_FIELD_TYPE hblks; /* always 0 */
- MALLINFO_FIELD_TYPE hblkhd; /* space in mmapped regions */
- MALLINFO_FIELD_TYPE usmblks; /* maximum total allocated space */
- MALLINFO_FIELD_TYPE fsmblks; /* always 0 */
- MALLINFO_FIELD_TYPE uordblks; /* total allocated space */
- MALLINFO_FIELD_TYPE fordblks; /* total free space */
- MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */
-};
-#endif /* STRUCT_MALLINFO_DECLARED */
-#endif /* HAVE_USR_INCLUDE_MALLOC_H */
-#endif /* NO_MALLINFO */
-
-/*
- Try to persuade compilers to inline. The most critical functions for
- inlining are defined as macros, so these aren't used for them.
-*/
-
-#ifndef FORCEINLINE
- #if defined(__GNUC__)
-#define FORCEINLINE __inline __attribute__ ((always_inline))
- #elif defined(_MSC_VER)
- #define FORCEINLINE __forceinline
- #endif
-#endif
-#ifndef NOINLINE
- #if defined(__GNUC__)
- #define NOINLINE __attribute__ ((noinline))
- #elif defined(_MSC_VER)
- #define NOINLINE __declspec(noinline)
- #else
- #define NOINLINE
- #endif
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#ifndef FORCEINLINE
- #define FORCEINLINE inline
-#endif
-#endif /* __cplusplus */
-#ifndef FORCEINLINE
- #define FORCEINLINE
-#endif
-
-#if !ONLY_MSPACES
-
-/* ------------------- Declarations of public routines ------------------- */
-
-#ifndef USE_DL_PREFIX
-#define dlcalloc calloc
-#define dlfree free
-#define dlmalloc malloc
-#define dlmemalign memalign
-#define dlposix_memalign posix_memalign
-#define dlrealloc realloc
-#define dlrealloc_in_place realloc_in_place
-#define dlvalloc valloc
-#define dlpvalloc pvalloc
-#define dlmallinfo mallinfo
-#define dlmallopt mallopt
-#define dlmalloc_trim malloc_trim
-#define dlmalloc_stats malloc_stats
-#define dlmalloc_usable_size malloc_usable_size
-#define dlmalloc_footprint malloc_footprint
-#define dlmalloc_max_footprint malloc_max_footprint
-#define dlmalloc_footprint_limit malloc_footprint_limit
-#define dlmalloc_set_footprint_limit malloc_set_footprint_limit
-#define dlmalloc_inspect_all malloc_inspect_all
-#define dlindependent_calloc independent_calloc
-#define dlindependent_comalloc independent_comalloc
-#define dlbulk_free bulk_free
-#endif /* USE_DL_PREFIX */
-
-/*
- malloc(size_t n)
- Returns a pointer to a newly allocated chunk of at least n bytes, or
- null if no space is available, in which case errno is set to ENOMEM
- on ANSI C systems.
-
- If n is zero, malloc returns a minimum-sized chunk. (The minimum
- size is 16 bytes on most 32bit systems, and 32 bytes on 64bit
- systems.) Note that size_t is an unsigned type, so calls with
- arguments that would be negative if signed are interpreted as
- requests for huge amounts of space, which will often fail. The
- maximum supported value of n differs across systems, but is in all
- cases less than the maximum representable value of a size_t.
-*/
-DLMALLOC_EXPORT void* dlmalloc(size_t);
-
-/*
- free(void* p)
- Releases the chunk of memory pointed to by p, that had been previously
- allocated using malloc or a related routine such as realloc.
- It has no effect if p is null. If p was not malloced or already
- freed, free(p) will by default cause the current program to abort.
-*/
-DLMALLOC_EXPORT void dlfree(void*);
-
-/*
- calloc(size_t n_elements, size_t element_size);
- Returns a pointer to n_elements * element_size bytes, with all locations
- set to zero.
-*/
-DLMALLOC_EXPORT void* dlcalloc(size_t, size_t);
-
-/*
- realloc(void* p, size_t n)
- Returns a pointer to a chunk of size n that contains the same data
- as does chunk p up to the minimum of (n, p's size) bytes, or null
- if no space is available.
-
- The returned pointer may or may not be the same as p. The algorithm
- prefers extending p in most cases when possible, otherwise it
- employs the equivalent of a malloc-copy-free sequence.
-
- If p is null, realloc is equivalent to malloc.
-
- If space is not available, realloc returns null, errno is set (if on
- ANSI) and p is NOT freed.
-
- if n is for fewer bytes than already held by p, the newly unused
- space is lopped off and freed if possible. realloc with a size
- argument of zero (re)allocates a minimum-sized chunk.
-
- The old unix realloc convention of allowing the last-free'd chunk
- to be used as an argument to realloc is not supported.
-*/
-DLMALLOC_EXPORT void* dlrealloc(void*, size_t);
-
-/*
- realloc_in_place(void* p, size_t n)
- Resizes the space allocated for p to size n, only if this can be
- done without moving p (i.e., only if there is adjacent space
- available if n is greater than p's current allocated size, or n is
- less than or equal to p's size). This may be used instead of plain
- realloc if an alternative allocation strategy is needed upon failure
- to expand space; for example, reallocation of a buffer that must be
- memory-aligned or cleared. You can use realloc_in_place to trigger
- these alternatives only when needed.
-
- Returns p if successful; otherwise null.
-*/
-DLMALLOC_EXPORT void* dlrealloc_in_place(void*, size_t);
-
-/*
- memalign(size_t alignment, size_t n);
- Returns a pointer to a newly allocated chunk of n bytes, aligned
- in accord with the alignment argument.
-
- The alignment argument should be a power of two. If the argument is
- not a power of two, the nearest greater power is used.
- 8-byte alignment is guaranteed by normal malloc calls, so don't
- bother calling memalign with an argument of 8 or less.
-
- Overreliance on memalign is a sure way to fragment space.
-*/
-DLMALLOC_EXPORT void* dlmemalign(size_t, size_t);
-
-/*
- int posix_memalign(void** pp, size_t alignment, size_t n);
- Allocates a chunk of n bytes, aligned in accord with the alignment
- argument. Differs from memalign only in that it (1) assigns the
- allocated memory to *pp rather than returning it, (2) fails and
- returns EINVAL if the alignment is not a power of two (3) fails and
- returns ENOMEM if memory cannot be allocated.
-*/
-DLMALLOC_EXPORT int dlposix_memalign(void**, size_t, size_t);
-
-/*
- valloc(size_t n);
- Equivalent to memalign(pagesize, n), where pagesize is the page
- size of the system. If the pagesize is unknown, 4096 is used.
-*/
-DLMALLOC_EXPORT void* dlvalloc(size_t);
-
-/*
- mallopt(int parameter_number, int parameter_value)
- Sets tunable parameters The format is to provide a
- (parameter-number, parameter-value) pair. mallopt then sets the
- corresponding parameter to the argument value if it can (i.e., so
- long as the value is meaningful), and returns 1 if successful else
- 0. To workaround the fact that mallopt is specified to use int,
- not size_t parameters, the value -1 is specially treated as the
- maximum unsigned size_t value.
-
- SVID/XPG/ANSI defines four standard param numbers for mallopt,
- normally defined in malloc.h. None of these are use in this malloc,
- so setting them has no effect. But this malloc also supports other
- options in mallopt. See below for details. Briefly, supported
- parameters are as follows (listed defaults are for "typical"
- configurations).
-
- Symbol param # default allowed param values
- M_TRIM_THRESHOLD -1 2*1024*1024 any (-1 disables)
- M_GRANULARITY -2 page size any power of 2 >= page size
- M_MMAP_THRESHOLD -3 256*1024 any (or 0 if no MMAP support)
-*/
-DLMALLOC_EXPORT int dlmallopt(int, int);
-
-/*
- malloc_footprint();
- Returns the number of bytes obtained from the system. The total
- number of bytes allocated by malloc, realloc etc., is less than this
- value. Unlike mallinfo, this function returns only a precomputed
- result, so can be called frequently to monitor memory consumption.
- Even if locks are otherwise defined, this function does not use them,
- so results might not be up to date.
-*/
-DLMALLOC_EXPORT size_t dlmalloc_footprint(void);
-
-/*
- malloc_max_footprint();
- Returns the maximum number of bytes obtained from the system. This
- value will be greater than current footprint if deallocated space
- has been reclaimed by the system. The peak number of bytes allocated
- by malloc, realloc etc., is less than this value. Unlike mallinfo,
- this function returns only a precomputed result, so can be called
- frequently to monitor memory consumption. Even if locks are
- otherwise defined, this function does not use them, so results might
- not be up to date.
-*/
-DLMALLOC_EXPORT size_t dlmalloc_max_footprint(void);
-
-/*
- malloc_footprint_limit();
- Returns the number of bytes that the heap is allowed to obtain from
- the system, returning the last value returned by
- malloc_set_footprint_limit, or the maximum size_t value if
- never set. The returned value reflects a permission. There is no
- guarantee that this number of bytes can actually be obtained from
- the system.
-*/
-DLMALLOC_EXPORT size_t dlmalloc_footprint_limit();
-
-/*
- malloc_set_footprint_limit();
- Sets the maximum number of bytes to obtain from the system, causing
- failure returns from malloc and related functions upon attempts to
- exceed this value. The argument value may be subject to page
- rounding to an enforceable limit; this actual value is returned.
- Using an argument of the maximum possible size_t effectively
- disables checks. If the argument is less than or equal to the
- current malloc_footprint, then all future allocations that require
- additional system memory will fail. However, invocation cannot
- retroactively deallocate existing used memory.
-*/
-DLMALLOC_EXPORT size_t dlmalloc_set_footprint_limit(size_t bytes);
-
-#if MALLOC_INSPECT_ALL
-/*
- malloc_inspect_all(void(*handler)(void *start,
- void *end,
- size_t used_bytes,
- void* callback_arg),
- void* arg);
- Traverses the heap and calls the given handler for each managed
- region, skipping all bytes that are (or may be) used for bookkeeping
- purposes. Traversal does not include include chunks that have been
- directly memory mapped. Each reported region begins at the start
- address, and continues up to but not including the end address. The
- first used_bytes of the region contain allocated data. If
- used_bytes is zero, the region is unallocated. The handler is
- invoked with the given callback argument. If locks are defined, they
- are held during the entire traversal. It is a bad idea to invoke
- other malloc functions from within the handler.
-
- For example, to count the number of in-use chunks with size greater
- than 1000, you could write:
- static int count = 0;
- void count_chunks(void* start, void* end, size_t used, void* arg) {
- if (used >= 1000) ++count;
- }
- then:
- malloc_inspect_all(count_chunks, NULL);
-
- malloc_inspect_all is compiled only if MALLOC_INSPECT_ALL is defined.
-*/
-DLMALLOC_EXPORT void dlmalloc_inspect_all(void(*handler)(void*, void *, size_t, void*),
- void* arg);
-
-#endif /* MALLOC_INSPECT_ALL */
-
-#if !NO_MALLINFO
-/*
- mallinfo()
- Returns (by copy) a struct containing various summary statistics:
-
- arena: current total non-mmapped bytes allocated from system
- ordblks: the number of free chunks
- smblks: always zero.
- hblks: current number of mmapped regions
- hblkhd: total bytes held in mmapped regions
- usmblks: the maximum total allocated space. This will be greater
- than current total if trimming has occurred.
- fsmblks: always zero
- uordblks: current total allocated space (normal or mmapped)
- fordblks: total free space
- keepcost: the maximum number of bytes that could ideally be released
- back to system via malloc_trim. ("ideally" means that
- it ignores page restrictions etc.)
-
- Because these fields are ints, but internal bookkeeping may
- be kept as longs, the reported values may wrap around zero and
- thus be inaccurate.
-*/
-DLMALLOC_EXPORT struct mallinfo dlmallinfo(void);
-#endif /* NO_MALLINFO */
-
-/*
- independent_calloc(size_t n_elements, size_t element_size, void* chunks[]);
-
- independent_calloc is similar to calloc, but instead of returning a
- single cleared space, it returns an array of pointers to n_elements
- independent elements that can hold contents of size elem_size, each
- of which starts out cleared, and can be independently freed,
- realloc'ed etc. The elements are guaranteed to be adjacently
- allocated (this is not guaranteed to occur with multiple callocs or
- mallocs), which may also improve cache locality in some
- applications.
-
- The "chunks" argument is optional (i.e., may be null, which is
- probably the most typical usage). If it is null, the returned array
- is itself dynamically allocated and should also be freed when it is
- no longer needed. Otherwise, the chunks array must be of at least
- n_elements in length. It is filled in with the pointers to the
- chunks.
-
- In either case, independent_calloc returns this pointer array, or
- null if the allocation failed. If n_elements is zero and "chunks"
- is null, it returns a chunk representing an array with zero elements
- (which should be freed if not wanted).
-
- Each element must be freed when it is no longer needed. This can be
- done all at once using bulk_free.
-
- independent_calloc simplifies and speeds up implementations of many
- kinds of pools. It may also be useful when constructing large data
- structures that initially have a fixed number of fixed-sized nodes,
- but the number is not known at compile time, and some of the nodes
- may later need to be freed. For example:
-
- struct Node { int item; struct Node* next; };
-
- struct Node* build_list() {
- struct Node** pool;
- int n = read_number_of_nodes_needed();
- if (n <= 0) return 0;
- pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0);
- if (pool == 0) die();
- // organize into a linked list...
- struct Node* first = pool[0];
- for (i = 0; i < n-1; ++i)
- pool[i]->next = pool[i+1];
- free(pool); // Can now free the array (or not, if it is needed later)
- return first;
- }
-*/
-DLMALLOC_EXPORT void** dlindependent_calloc(size_t, size_t, void**);
-
-/*
- independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
-
- independent_comalloc allocates, all at once, a set of n_elements
- chunks with sizes indicated in the "sizes" array. It returns
- an array of pointers to these elements, each of which can be
- independently freed, realloc'ed etc. The elements are guaranteed to
- be adjacently allocated (this is not guaranteed to occur with
- multiple callocs or mallocs), which may also improve cache locality
- in some applications.
-
- The "chunks" argument is optional (i.e., may be null). If it is null
- the returned array is itself dynamically allocated and should also
- be freed when it is no longer needed. Otherwise, the chunks array
- must be of at least n_elements in length. It is filled in with the
- pointers to the chunks.
-
- In either case, independent_comalloc returns this pointer array, or
- null if the allocation failed. If n_elements is zero and chunks is
- null, it returns a chunk representing an array with zero elements
- (which should be freed if not wanted).
-
- Each element must be freed when it is no longer needed. This can be
- done all at once using bulk_free.
-
- independent_comallac differs from independent_calloc in that each
- element may have a different size, and also that it does not
- automatically clear elements.
-
- independent_comalloc can be used to speed up allocation in cases
- where several structs or objects must always be allocated at the
- same time. For example:
-
- struct Head { ... }
- struct Foot { ... }
-
- void send_message(char* msg) {
- int msglen = strlen(msg);
- size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) };
- void* chunks[3];
- if (independent_comalloc(3, sizes, chunks) == 0)
- die();
- struct Head* head = (struct Head*)(chunks[0]);
- char* body = (char*)(chunks[1]);
- struct Foot* foot = (struct Foot*)(chunks[2]);
- // ...
- }
-
- In general though, independent_comalloc is worth using only for
- larger values of n_elements. For small values, you probably won't
- detect enough difference from series of malloc calls to bother.
-
- Overuse of independent_comalloc can increase overall memory usage,
- since it cannot reuse existing noncontiguous small chunks that
- might be available for some of the elements.
-*/
-DLMALLOC_EXPORT void** dlindependent_comalloc(size_t, size_t*, void**);
-
-/*
- bulk_free(void* array[], size_t n_elements)
- Frees and clears (sets to null) each non-null pointer in the given
- array. This is likely to be faster than freeing them one-by-one.
- If footers are used, pointers that have been allocated in different
- mspaces are not freed or cleared, and the count of all such pointers
- is returned. For large arrays of pointers with poor locality, it
- may be worthwhile to sort this array before calling bulk_free.
-*/
-DLMALLOC_EXPORT size_t dlbulk_free(void**, size_t n_elements);
-
-/*
- pvalloc(size_t n);
- Equivalent to valloc(minimum-page-that-holds(n)), that is,
- round up n to nearest pagesize.
- */
-DLMALLOC_EXPORT void* dlpvalloc(size_t);
-
-/*
- malloc_trim(size_t pad);
-
- If possible, gives memory back to the system (via negative arguments
- to sbrk) if there is unused memory at the `high' end of the malloc
- pool or in unused MMAP segments. You can call this after freeing
- large blocks of memory to potentially reduce the system-level memory
- requirements of a program. However, it cannot guarantee to reduce
- memory. Under some allocation patterns, some large free blocks of
- memory will be locked between two used chunks, so they cannot be
- given back to the system.
-
- The `pad' argument to malloc_trim represents the amount of free
- trailing space to leave untrimmed. If this argument is zero, only
- the minimum amount of memory to maintain internal data structures
- will be left. Non-zero arguments can be supplied to maintain enough
- trailing space to service future expected allocations without having
- to re-obtain memory from the system.
-
- Malloc_trim returns 1 if it actually released any memory, else 0.
-*/
-DLMALLOC_EXPORT int dlmalloc_trim(size_t);
-
-/*
- malloc_stats();
- Prints on stderr the amount of space obtained from the system (both
- via sbrk and mmap), the maximum amount (which may be more than
- current if malloc_trim and/or munmap got called), and the current
- number of bytes allocated via malloc (or realloc, etc) but not yet
- freed. Note that this is the number of bytes allocated, not the
- number requested. It will be larger than the number requested
- because of alignment and bookkeeping overhead. Because it includes
- alignment wastage as being in use, this figure may be greater than
- zero even when no user-level chunks are allocated.
-
- The reported current and maximum system memory can be inaccurate if
- a program makes other calls to system memory allocation functions
- (normally sbrk) outside of malloc.
-
- malloc_stats prints only the most commonly interesting statistics.
- More information can be obtained by calling mallinfo.
-*/
-DLMALLOC_EXPORT void dlmalloc_stats(void);
-
-/*
- malloc_usable_size(void* p);
-
- Returns the number of bytes you can actually use in
- an allocated chunk, which may be more than you requested (although
- often not) due to alignment and minimum size constraints.
- You can use this many bytes without worrying about
- overwriting other allocated objects. This is not a particularly great
- programming practice. malloc_usable_size can be more useful in
- debugging and assertions, for example:
-
- p = malloc(n);
- assert(malloc_usable_size(p) >= 256);
-*/
-size_t dlmalloc_usable_size(void*);
-
-#endif /* ONLY_MSPACES */
-
-#if MSPACES
-
-/*
- mspace is an opaque type representing an independent
- region of space that supports mspace_malloc, etc.
-*/
-typedef void* mspace;
-
-/*
- create_mspace creates and returns a new independent space with the
- given initial capacity, or, if 0, the default granularity size. It
- returns null if there is no system memory available to create the
- space. If argument locked is non-zero, the space uses a separate
- lock to control access. The capacity of the space will grow
- dynamically as needed to service mspace_malloc requests. You can
- control the sizes of incremental increases of this space by
- compiling with a different DEFAULT_GRANULARITY or dynamically
- setting with mallopt(M_GRANULARITY, value).
-*/
-DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked);
-
-/*
- destroy_mspace destroys the given space, and attempts to return all
- of its memory back to the system, returning the total number of
- bytes freed. After destruction, the results of access to all memory
- used by the space become undefined.
-*/
-DLMALLOC_EXPORT size_t destroy_mspace(mspace msp);
-
-/*
- create_mspace_with_base uses the memory supplied as the initial base
- of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this
- space is used for bookkeeping, so the capacity must be at least this
- large. (Otherwise 0 is returned.) When this initial space is
- exhausted, additional memory will be obtained from the system.
- Destroying this space will deallocate all additionally allocated
- space (if possible) but not the initial base.
-*/
-DLMALLOC_EXPORT mspace create_mspace_with_base(void* base, size_t capacity, int locked);
-
-/*
- mspace_track_large_chunks controls whether requests for large chunks
- are allocated in their own untracked mmapped regions, separate from
- others in this mspace. By default large chunks are not tracked,
- which reduces fragmentation. However, such chunks are not
- necessarily released to the system upon destroy_mspace. Enabling
- tracking by setting to true may increase fragmentation, but avoids
- leakage when relying on destroy_mspace to release all memory
- allocated using this space. The function returns the previous
- setting.
-*/
-DLMALLOC_EXPORT int mspace_track_large_chunks(mspace msp, int enable);
-
-
-/*
- mspace_malloc behaves as malloc, but operates within
- the given space.
-*/
-DLMALLOC_EXPORT void* mspace_malloc(mspace msp, size_t bytes);
-
-/*
- mspace_free behaves as free, but operates within
- the given space.
-
- If compiled with FOOTERS==1, mspace_free is not actually needed.
- free may be called instead of mspace_free because freed chunks from
- any space are handled by their originating spaces.
-*/
-DLMALLOC_EXPORT void mspace_free(mspace msp, void* mem);
-
-/*
- mspace_realloc behaves as realloc, but operates within
- the given space.
-
- If compiled with FOOTERS==1, mspace_realloc is not actually
- needed. realloc may be called instead of mspace_realloc because
- realloced chunks from any space are handled by their originating
- spaces.
-*/
-DLMALLOC_EXPORT void* mspace_realloc(mspace msp, void* mem, size_t newsize);
-
-/*
- mspace_calloc behaves as calloc, but operates within
- the given space.
-*/
-DLMALLOC_EXPORT void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size);
-
-/*
- mspace_memalign behaves as memalign, but operates within
- the given space.
-*/
-DLMALLOC_EXPORT void* mspace_memalign(mspace msp, size_t alignment, size_t bytes);
-
-/*
- mspace_independent_calloc behaves as independent_calloc, but
- operates within the given space.
-*/
-DLMALLOC_EXPORT void** mspace_independent_calloc(mspace msp, size_t n_elements,
- size_t elem_size, void* chunks[]);
-
-/*
- mspace_independent_comalloc behaves as independent_comalloc, but
- operates within the given space.
-*/
-DLMALLOC_EXPORT void** mspace_independent_comalloc(mspace msp, size_t n_elements,
- size_t sizes[], void* chunks[]);
-
-/*
- mspace_footprint() returns the number of bytes obtained from the
- system for this space.
-*/
-DLMALLOC_EXPORT size_t mspace_footprint(mspace msp);
-
-/*
- mspace_max_footprint() returns the peak number of bytes obtained from the
- system for this space.
-*/
-DLMALLOC_EXPORT size_t mspace_max_footprint(mspace msp);
-
-
-#if !NO_MALLINFO
-/*
- mspace_mallinfo behaves as mallinfo, but reports properties of
- the given space.
-*/
-DLMALLOC_EXPORT struct mallinfo mspace_mallinfo(mspace msp);
-#endif /* NO_MALLINFO */
-
-/*
- malloc_usable_size(void* p) behaves the same as malloc_usable_size;
-*/
-DLMALLOC_EXPORT size_t mspace_usable_size(const void* mem);
-
-/*
- mspace_malloc_stats behaves as malloc_stats, but reports
- properties of the given space.
-*/
-DLMALLOC_EXPORT void mspace_malloc_stats(mspace msp);
-
-/*
- mspace_trim behaves as malloc_trim, but
- operates within the given space.
-*/
-DLMALLOC_EXPORT int mspace_trim(mspace msp, size_t pad);
-
-/*
- An alias for mallopt.
-*/
-DLMALLOC_EXPORT int mspace_mallopt(int, int);
-
-#endif /* MSPACES */
-
-#ifdef __cplusplus
-} /* end of extern "C" */
-#endif /* __cplusplus */
-
-/*
- ========================================================================
- To make a fully customizable malloc.h header file, cut everything
- above this line, put into file malloc.h, edit to suit, and #include it
- on the next line, as well as in programs that use this malloc.
- ========================================================================
-*/
-
-/* #include "malloc.h" */
-
-/*------------------------------ internal #includes ---------------------- */
-
-#ifdef _MSC_VER
-#pragma warning( disable : 4146 ) /* no "unsigned" warnings */
-#endif /* _MSC_VER */
-#if !NO_MALLOC_STATS
-#include <stdio.h> /* for printing in malloc_stats */
-#endif /* NO_MALLOC_STATS */
-#ifndef LACKS_ERRNO_H
-#include <errno.h> /* for MALLOC_FAILURE_ACTION */
-#endif /* LACKS_ERRNO_H */
-#ifdef DEBUG
-#if ABORT_ON_ASSERT_FAILURE
-#undef assert
-#define assert(x) if(!(x)) ABORT
-#else /* ABORT_ON_ASSERT_FAILURE */
-#include <assert.h>
-#endif /* ABORT_ON_ASSERT_FAILURE */
-#else /* DEBUG */
-#ifndef assert
-#define assert(x)
-#endif
-#define DEBUG 0
-#endif /* DEBUG */
-#if !defined(WIN32) && !defined(LACKS_TIME_H)
-#include <time.h> /* for magic initialization */
-#endif /* WIN32 */
-#ifndef LACKS_STDLIB_H
-#include <stdlib.h> /* for abort() */
-#endif /* LACKS_STDLIB_H */
-#ifndef LACKS_STRING_H
-#include <string.h> /* for memset etc */
-#endif /* LACKS_STRING_H */
-#if USE_BUILTIN_FFS
-#ifndef LACKS_STRINGS_H
-#include <strings.h> /* for ffs */
-#endif /* LACKS_STRINGS_H */
-#endif /* USE_BUILTIN_FFS */
-#if HAVE_MMAP
-#ifndef LACKS_SYS_MMAN_H
-/* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */
-#if (defined(linux) && !defined(__USE_GNU))
-#define __USE_GNU 1
-#include <sys/mman.h> /* for mmap */
-#undef __USE_GNU
-#else
-#include <sys/mman.h> /* for mmap */
-#endif /* linux */
-#endif /* LACKS_SYS_MMAN_H */
-#ifndef LACKS_FCNTL_H
-#include <fcntl.h>
-#endif /* LACKS_FCNTL_H */
-#endif /* HAVE_MMAP */
-#ifndef LACKS_UNISTD_H
-#include <unistd.h> /* for sbrk, sysconf */
-#else /* LACKS_UNISTD_H */
-#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__)
-extern void* sbrk(ptrdiff_t);
-#endif /* FreeBSD etc */
-#endif /* LACKS_UNISTD_H */
-
-/* Declarations for locking */
-#if USE_LOCKS
-#ifndef WIN32
-#if defined (__SVR4) && defined (__sun) /* solaris */
-#include <thread.h>
-#elif !defined(LACKS_SCHED_H)
-#include <sched.h>
-#endif /* solaris or LACKS_SCHED_H */
-#if (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0) || !USE_SPIN_LOCKS
-#include <pthread.h>
-#endif /* USE_RECURSIVE_LOCKS ... */
-#elif defined(_MSC_VER)
-#ifndef _M_AMD64
-/* These are already defined on AMD64 builds */
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-LONG __cdecl _InterlockedCompareExchange(LONG volatile *Dest, LONG Exchange, LONG Comp);
-LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value);
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-#endif /* _M_AMD64 */
-#pragma intrinsic (_InterlockedCompareExchange)
-#pragma intrinsic (_InterlockedExchange)
-#define interlockedcompareexchange _InterlockedCompareExchange
-#define interlockedexchange _InterlockedExchange
-#elif defined(WIN32) && defined(__GNUC__)
-#define interlockedcompareexchange(a, b, c) __sync_val_compare_and_swap(a, c, b)
-#define interlockedexchange __sync_lock_test_and_set
-#endif /* Win32 */
-#else /* USE_LOCKS */
-#endif /* USE_LOCKS */
-
-#ifndef LOCK_AT_FORK
-#define LOCK_AT_FORK 0
-#endif
-
-/* Declarations for bit scanning on win32 */
-#if defined(_MSC_VER) && _MSC_VER>=1300
-#ifndef BitScanForward /* Try to avoid pulling in WinNT.h */
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-unsigned char _BitScanForward(unsigned long *index, unsigned long mask);
-unsigned char _BitScanReverse(unsigned long *index, unsigned long mask);
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-
-#define BitScanForward _BitScanForward
-#define BitScanReverse _BitScanReverse
-#pragma intrinsic(_BitScanForward)
-#pragma intrinsic(_BitScanReverse)
-#endif /* BitScanForward */
-#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */
-
-#ifndef WIN32
-#ifndef malloc_getpagesize
-# ifdef _SC_PAGESIZE /* some SVR4 systems omit an underscore */
-# ifndef _SC_PAGE_SIZE
-# define _SC_PAGE_SIZE _SC_PAGESIZE
-# endif
-# endif
-# ifdef _SC_PAGE_SIZE
-# define malloc_getpagesize sysconf(_SC_PAGE_SIZE)
-# else
-# if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE)
- extern size_t getpagesize();
-# define malloc_getpagesize getpagesize()
-# else
-# ifdef WIN32 /* use supplied emulation of getpagesize */
-# define malloc_getpagesize getpagesize()
-# else
-# ifndef LACKS_SYS_PARAM_H
-# include <sys/param.h>
-# endif
-# ifdef EXEC_PAGESIZE
-# define malloc_getpagesize EXEC_PAGESIZE
-# else
-# ifdef NBPG
-# ifndef CLSIZE
-# define malloc_getpagesize NBPG
-# else
-# define malloc_getpagesize (NBPG * CLSIZE)
-# endif
-# else
-# ifdef NBPC
-# define malloc_getpagesize NBPC
-# else
-# ifdef PAGESIZE
-# define malloc_getpagesize PAGESIZE
-# else /* just guess */
-# define malloc_getpagesize ((size_t)4096U)
-# endif
-# endif
-# endif
-# endif
-# endif
-# endif
-# endif
-#endif
-#endif
-
-/* ------------------- size_t and alignment properties -------------------- */
-
-/* The byte and bit size of a size_t */
-#define SIZE_T_SIZE (sizeof(size_t))
-#define SIZE_T_BITSIZE (sizeof(size_t) << 3)
-
-/* Some constants coerced to size_t */
-/* Annoying but necessary to avoid errors on some platforms */
-#define SIZE_T_ZERO ((size_t)0)
-#define SIZE_T_ONE ((size_t)1)
-#define SIZE_T_TWO ((size_t)2)
-#define SIZE_T_FOUR ((size_t)4)
-#define TWO_SIZE_T_SIZES (SIZE_T_SIZE<<1)
-#define FOUR_SIZE_T_SIZES (SIZE_T_SIZE<<2)
-#define SIX_SIZE_T_SIZES (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES)
-#define HALF_MAX_SIZE_T (MAX_SIZE_T / 2U)
-
-/* The bit mask value corresponding to MALLOC_ALIGNMENT */
-#define CHUNK_ALIGN_MASK (MALLOC_ALIGNMENT - SIZE_T_ONE)
-
-/* True if address a has acceptable alignment */
-#define is_aligned(A) (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0)
-
-/* the number of bytes to offset an address to align it */
-#define align_offset(A)\
- ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\
- ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK))
-
-/* -------------------------- MMAP preliminaries ------------------------- */
-
-/*
- If HAVE_MORECORE or HAVE_MMAP are false, we just define calls and
- checks to fail so compiler optimizer can delete code rather than
- using so many "#if"s.
-*/
-
-
-/* MORECORE and MMAP must return MFAIL on failure */
-#define MFAIL ((void*)(MAX_SIZE_T))
-#define CMFAIL ((char*)(MFAIL)) /* defined for convenience */
-
-#if HAVE_MMAP
-
-#ifndef WIN32
-#define MUNMAP_DEFAULT(a, s) munmap((a), (s))
-#define MMAP_PROT (PROT_READ|PROT_WRITE)
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS MAP_ANON
-#endif /* MAP_ANON */
-#ifdef MAP_ANONYMOUS
-#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS)
-#define MMAP_DEFAULT(s) mmap(0, (s), MMAP_PROT, MMAP_FLAGS, -1, 0)
-#else /* MAP_ANONYMOUS */
-/*
- Nearly all versions of mmap support MAP_ANONYMOUS, so the following
- is unlikely to be needed, but is supplied just in case.
-*/
-#define MMAP_FLAGS (MAP_PRIVATE)
-static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */
-#define MMAP_DEFAULT(s) ((dev_zero_fd < 0) ? \
- (dev_zero_fd = open("/dev/zero", O_RDWR), \
- mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) : \
- mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0))
-#endif /* MAP_ANONYMOUS */
-
-#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s)
-
-#else /* WIN32 */
-
-/* Win32 MMAP via VirtualAlloc */
-static FORCEINLINE void* win32mmap(size_t size) {
- void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
- return (ptr != 0)? ptr: MFAIL;
-}
-
-/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
-static FORCEINLINE void* win32direct_mmap(size_t size) {
- void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN,
- PAGE_READWRITE);
- return (ptr != 0)? ptr: MFAIL;
-}
-
-/* This function supports releasing coalesed segments */
-static FORCEINLINE int win32munmap(void* ptr, size_t size) {
- MEMORY_BASIC_INFORMATION minfo;
- char* cptr = (char*)ptr;
- while (size) {
- if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0)
- return -1;
- if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr ||
- minfo.State != MEM_COMMIT || minfo.RegionSize > size)
- return -1;
- if (VirtualFree(cptr, 0, MEM_RELEASE) == 0)
- return -1;
- cptr += minfo.RegionSize;
- size -= minfo.RegionSize;
- }
- return 0;
-}
-
-#define MMAP_DEFAULT(s) win32mmap(s)
-#define MUNMAP_DEFAULT(a, s) win32munmap((a), (s))
-#define DIRECT_MMAP_DEFAULT(s) win32direct_mmap(s)
-#endif /* WIN32 */
-#endif /* HAVE_MMAP */
-
-#if HAVE_MREMAP
-#ifndef WIN32
-#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv))
-#endif /* WIN32 */
-#endif /* HAVE_MREMAP */
-
-/**
- * Define CALL_MORECORE
- */
-#if HAVE_MORECORE
- #ifdef MORECORE
- #define CALL_MORECORE(S) MORECORE(S)
- #else /* MORECORE */
- #define CALL_MORECORE(S) MORECORE_DEFAULT(S)
- #endif /* MORECORE */
-#else /* HAVE_MORECORE */
- #define CALL_MORECORE(S) MFAIL
-#endif /* HAVE_MORECORE */
-
-/**
- * Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP
- */
-#if HAVE_MMAP
- #define USE_MMAP_BIT (SIZE_T_ONE)
-
- #ifdef MMAP
- #define CALL_MMAP(s) MMAP(s)
- #else /* MMAP */
- #define CALL_MMAP(s) MMAP_DEFAULT(s)
- #endif /* MMAP */
- #ifdef MUNMAP
- #define CALL_MUNMAP(a, s) MUNMAP((a), (s))
- #else /* MUNMAP */
- #define CALL_MUNMAP(a, s) MUNMAP_DEFAULT((a), (s))
- #endif /* MUNMAP */
- #ifdef DIRECT_MMAP
- #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
- #else /* DIRECT_MMAP */
- #define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s)
- #endif /* DIRECT_MMAP */
-#else /* HAVE_MMAP */
- #define USE_MMAP_BIT (SIZE_T_ZERO)
-
- #define MMAP(s) MFAIL
- #define MUNMAP(a, s) (-1)
- #define DIRECT_MMAP(s) MFAIL
- #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
- #define CALL_MMAP(s) MMAP(s)
- #define CALL_MUNMAP(a, s) MUNMAP((a), (s))
-#endif /* HAVE_MMAP */
-
-/**
- * Define CALL_MREMAP
- */
-#if HAVE_MMAP && HAVE_MREMAP
- #ifdef MREMAP
- #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv))
- #else /* MREMAP */
- #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP_DEFAULT((addr), (osz), (nsz), (mv))
- #endif /* MREMAP */
-#else /* HAVE_MMAP && HAVE_MREMAP */
- #define CALL_MREMAP(addr, osz, nsz, mv) MFAIL
-#endif /* HAVE_MMAP && HAVE_MREMAP */
-
-/* mstate bit set if continguous morecore disabled or failed */
-#define USE_NONCONTIGUOUS_BIT (4U)
-
-/* segment bit set in create_mspace_with_base */
-#define EXTERN_BIT (8U)
-
-
-/* --------------------------- Lock preliminaries ------------------------ */
-
-/*
- When locks are defined, there is one global lock, plus
- one per-mspace lock.
-
- The global lock_ensures that mparams.magic and other unique
- mparams values are initialized only once. It also protects
- sequences of calls to MORECORE. In many cases sys_alloc requires
- two calls, that should not be interleaved with calls by other
- threads. This does not protect against direct calls to MORECORE
- by other threads not using this lock, so there is still code to
- cope the best we can on interference.
-
- Per-mspace locks surround calls to malloc, free, etc.
- By default, locks are simple non-reentrant mutexes.
-
- Because lock-protected regions generally have bounded times, it is
- OK to use the supplied simple spinlocks. Spinlocks are likely to
- improve performance for lightly contended applications, but worsen
- performance under heavy contention.
-
- If USE_LOCKS is > 1, the definitions of lock routines here are
- bypassed, in which case you will need to define the type MLOCK_T,
- and at least INITIAL_LOCK, DESTROY_LOCK, ACQUIRE_LOCK, RELEASE_LOCK
- and TRY_LOCK. You must also declare a
- static MLOCK_T malloc_global_mutex = { initialization values };.
-
-*/
-
-#if !USE_LOCKS
-#define USE_LOCK_BIT (0U)
-#define INITIAL_LOCK(l) (0)
-#define DESTROY_LOCK(l) (0)
-#define ACQUIRE_MALLOC_GLOBAL_LOCK()
-#define RELEASE_MALLOC_GLOBAL_LOCK()
-
-#else
-#if USE_LOCKS > 1
-/* ----------------------- User-defined locks ------------------------ */
-/* Define your own lock implementation here */
-/* #define INITIAL_LOCK(lk) ... */
-/* #define DESTROY_LOCK(lk) ... */
-/* #define ACQUIRE_LOCK(lk) ... */
-/* #define RELEASE_LOCK(lk) ... */
-/* #define TRY_LOCK(lk) ... */
-/* static MLOCK_T malloc_global_mutex = ... */
-
-#elif USE_SPIN_LOCKS
-
-/* First, define CAS_LOCK and CLEAR_LOCK on ints */
-/* Note CAS_LOCK defined to return 0 on success */
-
-#if defined(__GNUC__)&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))
-#define CAS_LOCK(sl) __sync_lock_test_and_set(sl, 1)
-#define CLEAR_LOCK(sl) __sync_lock_release(sl)
-
-#elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)))
-/* Custom spin locks for older gcc on x86 */
-static FORCEINLINE int x86_cas_lock(int *sl) {
- int ret;
- int val = 1;
- int cmp = 0;
- __asm__ __volatile__ ("lock; cmpxchgl %1, %2"
- : "=a" (ret)
- : "r" (val), "m" (*(sl)), "0"(cmp)
- : "memory", "cc");
- return ret;
-}
-
-static FORCEINLINE void x86_clear_lock(int* sl) {
- assert(*sl != 0);
- int prev = 0;
- int ret;
- __asm__ __volatile__ ("lock; xchgl %0, %1"
- : "=r" (ret)
- : "m" (*(sl)), "0"(prev)
- : "memory");
-}
-
-#define CAS_LOCK(sl) x86_cas_lock(sl)
-#define CLEAR_LOCK(sl) x86_clear_lock(sl)
-
-#else /* Win32 MSC */
-#define CAS_LOCK(sl) interlockedexchange(sl, (LONG)1)
-#define CLEAR_LOCK(sl) interlockedexchange (sl, (LONG)0)
-
-#endif /* ... gcc spins locks ... */
-
-/* How to yield for a spin lock */
-#define SPINS_PER_YIELD 63
-#if defined(_MSC_VER)
-#define SLEEP_EX_DURATION 50 /* delay for yield/sleep */
-#define SPIN_LOCK_YIELD SleepEx(SLEEP_EX_DURATION, FALSE)
-#elif defined (__SVR4) && defined (__sun) /* solaris */
-#define SPIN_LOCK_YIELD thr_yield();
-#elif !defined(LACKS_SCHED_H)
-#define SPIN_LOCK_YIELD sched_yield();
-#else
-#define SPIN_LOCK_YIELD
-#endif /* ... yield ... */
-
-#if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0
-/* Plain spin locks use single word (embedded in malloc_states) */
-static int spin_acquire_lock(int *sl) {
- int spins = 0;
- while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) {
- if ((++spins & SPINS_PER_YIELD) == 0) {
- SPIN_LOCK_YIELD;
- }
- }
- return 0;
-}
-
-#define MLOCK_T int
-#define TRY_LOCK(sl) !CAS_LOCK(sl)
-#define RELEASE_LOCK(sl) CLEAR_LOCK(sl)
-#define ACQUIRE_LOCK(sl) (CAS_LOCK(sl)? spin_acquire_lock(sl) : 0)
-#define INITIAL_LOCK(sl) (*sl = 0)
-#define DESTROY_LOCK(sl) (0)
-static MLOCK_T malloc_global_mutex = 0;
-
-#else /* USE_RECURSIVE_LOCKS */
-/* types for lock owners */
-#ifdef WIN32
-#define THREAD_ID_T DWORD
-#define CURRENT_THREAD GetCurrentThreadId()
-#define EQ_OWNER(X,Y) ((X) == (Y))
-#else
-/*
- Note: the following assume that pthread_t is a type that can be
- initialized to (casted) zero. If this is not the case, you will need to
- somehow redefine these or not use spin locks.
-*/
-#define THREAD_ID_T pthread_t
-#define CURRENT_THREAD pthread_self()
-#define EQ_OWNER(X,Y) pthread_equal(X, Y)
-#endif
-
-struct malloc_recursive_lock {
- int sl;
- unsigned int c;
- THREAD_ID_T threadid;
-};
-
-#define MLOCK_T struct malloc_recursive_lock
-static MLOCK_T malloc_global_mutex = { 0, 0, (THREAD_ID_T)0};
-
-static FORCEINLINE void recursive_release_lock(MLOCK_T *lk) {
- assert(lk->sl != 0);
- if (--lk->c == 0) {
- CLEAR_LOCK(&lk->sl);
- }
-}
-
-static FORCEINLINE int recursive_acquire_lock(MLOCK_T *lk) {
- THREAD_ID_T mythreadid = CURRENT_THREAD;
- int spins = 0;
- for (;;) {
- if (*((volatile int *)(&lk->sl)) == 0) {
- if (!CAS_LOCK(&lk->sl)) {
- lk->threadid = mythreadid;
- lk->c = 1;
- return 0;
- }
- }
- else if (EQ_OWNER(lk->threadid, mythreadid)) {
- ++lk->c;
- return 0;
- }
- if ((++spins & SPINS_PER_YIELD) == 0) {
- SPIN_LOCK_YIELD;
- }
- }
-}
-
-static FORCEINLINE int recursive_try_lock(MLOCK_T *lk) {
- THREAD_ID_T mythreadid = CURRENT_THREAD;
- if (*((volatile int *)(&lk->sl)) == 0) {
- if (!CAS_LOCK(&lk->sl)) {
- lk->threadid = mythreadid;
- lk->c = 1;
- return 1;
- }
- }
- else if (EQ_OWNER(lk->threadid, mythreadid)) {
- ++lk->c;
- return 1;
- }
- return 0;
-}
-
-#define RELEASE_LOCK(lk) recursive_release_lock(lk)
-#define TRY_LOCK(lk) recursive_try_lock(lk)
-#define ACQUIRE_LOCK(lk) recursive_acquire_lock(lk)
-#define INITIAL_LOCK(lk) ((lk)->threadid = (THREAD_ID_T)0, (lk)->sl = 0, (lk)->c = 0)
-#define DESTROY_LOCK(lk) (0)
-#endif /* USE_RECURSIVE_LOCKS */
-
-#elif defined(WIN32) /* Win32 critical sections */
-#define MLOCK_T CRITICAL_SECTION
-#define ACQUIRE_LOCK(lk) (EnterCriticalSection(lk), 0)
-#define RELEASE_LOCK(lk) LeaveCriticalSection(lk)
-#define TRY_LOCK(lk) TryEnterCriticalSection(lk)
-#define INITIAL_LOCK(lk) (!InitializeCriticalSectionAndSpinCount((lk), 0x80000000|4000))
-#define DESTROY_LOCK(lk) (DeleteCriticalSection(lk), 0)
-#define NEED_GLOBAL_LOCK_INIT
-
-static MLOCK_T malloc_global_mutex;
-static volatile LONG malloc_global_mutex_status;
-
-/* Use spin loop to initialize global lock */
-static void init_malloc_global_mutex() {
- for (;;) {
- long stat = malloc_global_mutex_status;
- if (stat > 0)
- return;
- /* transition to < 0 while initializing, then to > 0) */
- if (stat == 0 &&
- interlockedcompareexchange(&malloc_global_mutex_status, (LONG)-1, (LONG)0) == 0) {
- InitializeCriticalSection(&malloc_global_mutex);
- interlockedexchange(&malloc_global_mutex_status, (LONG)1);
- return;
- }
- SleepEx(0, FALSE);
- }
-}
-
-#else /* pthreads-based locks */
-#define MLOCK_T pthread_mutex_t
-#define ACQUIRE_LOCK(lk) pthread_mutex_lock(lk)
-#define RELEASE_LOCK(lk) pthread_mutex_unlock(lk)
-#define TRY_LOCK(lk) (!pthread_mutex_trylock(lk))
-#define INITIAL_LOCK(lk) pthread_init_lock(lk)
-#define DESTROY_LOCK(lk) pthread_mutex_destroy(lk)
-
-#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 && defined(linux) && !defined(PTHREAD_MUTEX_RECURSIVE)
-/* Cope with old-style linux recursive lock initialization by adding */
-/* skipped internal declaration from pthread.h */
-extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr,
- int __kind));
-#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP
-#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y)
-#endif /* USE_RECURSIVE_LOCKS ... */
-
-static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-static int pthread_init_lock (MLOCK_T *lk) {
- pthread_mutexattr_t attr;
- if (pthread_mutexattr_init(&attr)) return 1;
-#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0
- if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1;
-#endif
- if (pthread_mutex_init(lk, &attr)) return 1;
- if (pthread_mutexattr_destroy(&attr)) return 1;
- return 0;
-}
-
-#endif /* ... lock types ... */
-
-/* Common code for all lock types */
-#define USE_LOCK_BIT (2U)
-
-#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK
-#define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex);
-#endif
-
-#ifndef RELEASE_MALLOC_GLOBAL_LOCK
-#define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex);
-#endif
-
-#endif /* USE_LOCKS */
-
-/* ----------------------- Chunk representations ------------------------ */
-
-/*
- (The following includes lightly edited explanations by Colin Plumb.)
-
- The malloc_chunk declaration below is misleading (but accurate and
- necessary). It declares a "view" into memory allowing access to
- necessary fields at known offsets from a given base.
-
- Chunks of memory are maintained using a `boundary tag' method as
- originally described by Knuth. (See the paper by Paul Wilson
- ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a survey of such
- techniques.) Sizes of free chunks are stored both in the front of
- each chunk and at the end. This makes consolidating fragmented
- chunks into bigger chunks fast. The head fields also hold bits
- representing whether chunks are free or in use.
-
- Here are some pictures to make it clearer. They are "exploded" to
- show that the state of a chunk can be thought of as extending from
- the high 31 bits of the head field of its header through the
- prev_foot and PINUSE_BIT bit of the following chunk header.
-
- A chunk that's in use looks like:
-
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Size of previous chunk (if P = 0) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
- | Size of this chunk 1| +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | |
- +- -+
- | |
- +- -+
- | :
- +- size - sizeof(size_t) available payload bytes -+
- : |
- chunk-> +- -+
- | |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |1|
- | Size of next chunk (may or may not be in use) | +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
- And if it's free, it looks like this:
-
- chunk-> +- -+
- | User payload (must be in use, or we would have merged!) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
- | Size of this chunk 0| +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Next pointer |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Prev pointer |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | :
- +- size - sizeof(struct chunk) unused bytes -+
- : |
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Size of this chunk |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |0|
- | Size of next chunk (must be in use, or we would have merged)| +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | :
- +- User payload -+
- : |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- |0|
- +-+
- Note that since we always merge adjacent free chunks, the chunks
- adjacent to a free chunk must be in use.
-
- Given a pointer to a chunk (which can be derived trivially from the
- payload pointer) we can, in O(1) time, find out whether the adjacent
- chunks are free, and if so, unlink them from the lists that they
- are on and merge them with the current chunk.
-
- Chunks always begin on even word boundaries, so the mem portion
- (which is returned to the user) is also on an even word boundary, and
- thus at least double-word aligned.
-
- The P (PINUSE_BIT) bit, stored in the unused low-order bit of the
- chunk size (which is always a multiple of two words), is an in-use
- bit for the *previous* chunk. If that bit is *clear*, then the
- word before the current chunk size contains the previous chunk
- size, and can be used to find the front of the previous chunk.
- The very first chunk allocated always has this bit set, preventing
- access to non-existent (or non-owned) memory. If pinuse is set for
- any given chunk, then you CANNOT determine the size of the
- previous chunk, and might even get a memory addressing fault when
- trying to do so.
-
- The C (CINUSE_BIT) bit, stored in the unused second-lowest bit of
- the chunk size redundantly records whether the current chunk is
- inuse (unless the chunk is mmapped). This redundancy enables usage
- checks within free and realloc, and reduces indirection when freeing
- and consolidating chunks.
-
- Each freshly allocated chunk must have both cinuse and pinuse set.
- That is, each allocated chunk borders either a previously allocated
- and still in-use chunk, or the base of its memory arena. This is
- ensured by making all alloc
<TRUNCATED>
[07/14] arrow git commit: [maven-release-plugin] prepare release
apache-arrow-0.5.0
Posted by we...@apache.org.
[maven-release-plugin] prepare release apache-arrow-0.5.0
Change-Id: Icb6a8ac2d92a38190fbc89d88a0a97120646f843
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e9f76e12
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e9f76e12
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e9f76e12
Branch: refs/heads/master
Commit: e9f76e125b836d0fdc0a533e2fee3fca8bf4c1a1
Parents: 62ef2cd
Author: Wes McKinney <we...@twosigma.com>
Authored: Thu Jul 20 13:02:41 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Thu Jul 20 13:02:41 2017 -0400
----------------------------------------------------------------------
java/format/pom.xml | 2 +-
java/memory/pom.xml | 2 +-
java/pom.xml | 4 ++--
java/tools/pom.xml | 2 +-
java/vector/pom.xml | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/e9f76e12/java/format/pom.xml
----------------------------------------------------------------------
diff --git a/java/format/pom.xml b/java/format/pom.xml
index e092754..6b962e2 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>arrow-java-root</artifactId>
<groupId>org.apache.arrow</groupId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.5.0</version>
</parent>
<artifactId>arrow-format</artifactId>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e9f76e12/java/memory/pom.xml
----------------------------------------------------------------------
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index dc4d0da..e204cd6 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.5.0</version>
</parent>
<artifactId>arrow-memory</artifactId>
<name>Arrow Memory</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e9f76e12/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 1ec3d56..7eaf27d 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.5.0</version>
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
<url>https://github.com/apache/arrow</url>
- <tag>HEAD</tag>
+ <tag>apache-arrow-0.5.0</tag>
</scm>
<mailingLists>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e9f76e12/java/tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 53331f2..6d28c9f 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.5.0</version>
</parent>
<artifactId>arrow-tools</artifactId>
<name>Arrow Tools</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e9f76e12/java/vector/pom.xml
----------------------------------------------------------------------
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index a117a2f..c484935 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.5.0</version>
</parent>
<artifactId>arrow-vector</artifactId>
<name>Arrow Vectors</name>
[05/14] arrow git commit: [C++] Remove Plasma source tree for 0.5.0
release pending IP Clearance
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/fling.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/fling.cc b/cpp/src/plasma/fling.cc
deleted file mode 100644
index 79da4f4..0000000
--- a/cpp/src/plasma/fling.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright 2013 Sharvil Nanavati
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "plasma/fling.h"
-
-#include <string.h>
-
-void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len) {
- iov->iov_base = buf;
- iov->iov_len = 1;
-
- msg->msg_iov = iov;
- msg->msg_iovlen = 1;
- msg->msg_control = buf;
- msg->msg_controllen = buf_len;
- msg->msg_name = NULL;
- msg->msg_namelen = 0;
-}
-
-int send_fd(int conn, int fd) {
- struct msghdr msg;
- struct iovec iov;
- char buf[CMSG_SPACE(sizeof(int))];
- memset(&buf, 0, CMSG_SPACE(sizeof(int)));
-
- init_msg(&msg, &iov, buf, sizeof(buf));
-
- struct cmsghdr* header = CMSG_FIRSTHDR(&msg);
- header->cmsg_level = SOL_SOCKET;
- header->cmsg_type = SCM_RIGHTS;
- header->cmsg_len = CMSG_LEN(sizeof(int));
- *reinterpret_cast<int*>(CMSG_DATA(header)) = fd;
-
- // Send file descriptor.
- ssize_t r = sendmsg(conn, &msg, 0);
- if (r >= 0) {
- return 0;
- } else {
- return static_cast<int>(r);
- }
-}
-
-int recv_fd(int conn) {
- struct msghdr msg;
- struct iovec iov;
- char buf[CMSG_SPACE(sizeof(int))];
- init_msg(&msg, &iov, buf, sizeof(buf));
-
- if (recvmsg(conn, &msg, 0) == -1) return -1;
-
- int found_fd = -1;
- int oh_noes = 0;
- for (struct cmsghdr* header = CMSG_FIRSTHDR(&msg); header != NULL;
- header = CMSG_NXTHDR(&msg, header))
- if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) {
- ssize_t count =
- (header->cmsg_len - (CMSG_DATA(header) - (unsigned char*)header)) / sizeof(int);
- for (int i = 0; i < count; ++i) {
- int fd = (reinterpret_cast<int*>(CMSG_DATA(header)))[i];
- if (found_fd == -1) {
- found_fd = fd;
- } else {
- close(fd);
- oh_noes = 1;
- }
- }
- }
-
- // The sender sent us more than one file descriptor. We've closed
- // them all to prevent fd leaks but notify the caller that we got
- // a bad message.
- if (oh_noes) {
- close(found_fd);
- errno = EBADMSG;
- return -1;
- }
-
- return found_fd;
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/fling.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/fling.h b/cpp/src/plasma/fling.h
deleted file mode 100644
index 78ac9d1..0000000
--- a/cpp/src/plasma/fling.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2013 Sharvil Nanavati
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// FLING: Exchanging file descriptors over sockets
-//
-// This is a little library for sending file descriptors over a socket
-// between processes. The reason for doing that (as opposed to using
-// filenames to share the files) is so (a) no files remain in the
-// filesystem after all the processes terminate, (b) to make sure that
-// there are no name collisions and (c) to be able to control who has
-// access to the data.
-//
-// Most of the code is from https://github.com/sharvil/flingfd
-
-#include <errno.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-// This is neccessary for Mac OS X, see http://www.apuebook.com/faqs2e.html
-// (10).
-#if !defined(CMSG_SPACE) && !defined(CMSG_LEN)
-#define CMSG_SPACE(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + __DARWIN_ALIGN32(len))
-#define CMSG_LEN(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + (len))
-#endif
-
-void init_msg(struct msghdr* msg, struct iovec* iov, char* buf, size_t buf_len);
-
-// Send a file descriptor over a unix domain socket.
-//
-// @param conn Unix domain socket to send the file descriptor over.
-// @param fd File descriptor to send over.
-// @return Status code which is < 0 on failure.
-int send_fd(int conn, int fd);
-
-// Receive a file descriptor over a unix domain socket.
-//
-// @param conn Unix domain socket to receive the file descriptor from.
-// @return File descriptor or a value < 0 on failure.
-int recv_fd(int conn);
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/format/.gitignore
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/format/.gitignore b/cpp/src/plasma/format/.gitignore
deleted file mode 100644
index b2ddb05..0000000
--- a/cpp/src/plasma/format/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*_generated.h
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/format/common.fbs
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/format/common.fbs b/cpp/src/plasma/format/common.fbs
deleted file mode 100644
index 4d7d285..0000000
--- a/cpp/src/plasma/format/common.fbs
+++ /dev/null
@@ -1,34 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Object information data structure.
-table ObjectInfo {
- // Object ID of this object.
- object_id: string;
- // Number of bytes the content of this object occupies in memory.
- data_size: long;
- // Number of bytes the metadata of this object occupies in memory.
- metadata_size: long;
- // Unix epoch of when this object was created.
- create_time: long;
- // How long creation of this object took.
- construct_duration: long;
- // Hash of the object content.
- digest: string;
- // Specifies if this object was deleted or added.
- is_deletion: bool;
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/format/plasma.fbs
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs
deleted file mode 100644
index 23782ad..0000000
--- a/cpp/src/plasma/format/plasma.fbs
+++ /dev/null
@@ -1,291 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Plasma protocol specification
-
-enum MessageType:int {
- // Create a new object.
- PlasmaCreateRequest = 1,
- PlasmaCreateReply,
- // Seal an object.
- PlasmaSealRequest,
- PlasmaSealReply,
- // Get an object that is stored on the local Plasma store.
- PlasmaGetRequest,
- PlasmaGetReply,
- // Release an object.
- PlasmaReleaseRequest,
- PlasmaReleaseReply,
- // Delete an object.
- PlasmaDeleteRequest,
- PlasmaDeleteReply,
- // Get status of an object.
- PlasmaStatusRequest,
- PlasmaStatusReply,
- // See if the store contains an object (will be deprecated).
- PlasmaContainsRequest,
- PlasmaContainsReply,
- // Get information for a newly connecting client.
- PlasmaConnectRequest,
- PlasmaConnectReply,
- // Make room for new objects in the plasma store.
- PlasmaEvictRequest,
- PlasmaEvictReply,
- // Fetch objects from remote Plasma stores.
- PlasmaFetchRequest,
- // Wait for objects to be ready either from local or remote Plasma stores.
- PlasmaWaitRequest,
- PlasmaWaitReply,
- // Subscribe to a list of objects or to all objects.
- PlasmaSubscribeRequest,
- // Unsubscribe.
- PlasmaUnsubscribeRequest,
- // Sending and receiving data.
- // PlasmaDataRequest initiates sending the data, there will be one
- // such message per data transfer.
- PlasmaDataRequest,
- // PlasmaDataReply contains the actual data and is sent back to the
- // object store that requested the data. For each transfer, multiple
- // reply messages get sent. Each one contains a fixed number of bytes.
- PlasmaDataReply,
- // Object notifications.
- PlasmaNotification
-}
-
-enum PlasmaError:int {
- // Operation was successful.
- OK,
- // Trying to create an object that already exists.
- ObjectExists,
- // Trying to access an object that doesn't exist.
- ObjectNonexistent,
- // Trying to create an object but there isn't enough space in the store.
- OutOfMemory
-}
-
-// Plasma store messages
-
-struct PlasmaObjectSpec {
- // Index of the memory segment (= memory mapped file) that
- // this object is allocated in.
- segment_index: int;
- // Size in bytes of this segment (needed to call mmap).
- mmap_size: ulong;
- // The offset in bytes in the memory mapped file of the data.
- data_offset: ulong;
- // The size in bytes of the data.
- data_size: ulong;
- // The offset in bytes in the memory mapped file of the metadata.
- metadata_offset: ulong;
- // The size in bytes of the metadata.
- metadata_size: ulong;
-}
-
-table PlasmaCreateRequest {
- // ID of the object to be created.
- object_id: string;
- // The size of the object's data in bytes.
- data_size: ulong;
- // The size of the object's metadata in bytes.
- metadata_size: ulong;
-}
-
-table PlasmaCreateReply {
- // ID of the object that was created.
- object_id: string;
- // The object that is returned with this reply.
- plasma_object: PlasmaObjectSpec;
- // Error that occurred for this call.
- error: PlasmaError;
-}
-
-table PlasmaSealRequest {
- // ID of the object to be sealed.
- object_id: string;
- // Hash of the object data.
- digest: string;
-}
-
-table PlasmaSealReply {
- // ID of the object that was sealed.
- object_id: string;
- // Error code.
- error: PlasmaError;
-}
-
-table PlasmaGetRequest {
- // IDs of the objects stored at local Plasma store we are getting.
- object_ids: [string];
- // The number of milliseconds before the request should timeout.
- timeout_ms: long;
-}
-
-table PlasmaGetReply {
- // IDs of the objects being returned.
- // This number can be smaller than the number of requested
- // objects if not all requested objects are stored and sealed
- // in the local Plasma store.
- object_ids: [string];
- // Plasma object information, in the same order as their IDs.
- plasma_objects: [PlasmaObjectSpec];
- // The number of elements in both object_ids and plasma_objects arrays must agree.
-}
-
-table PlasmaReleaseRequest {
- // ID of the object to be released.
- object_id: string;
-}
-
-table PlasmaReleaseReply {
- // ID of the object that was released.
- object_id: string;
- // Error code.
- error: PlasmaError;
-}
-
-table PlasmaDeleteRequest {
- // ID of the object to be deleted.
- object_id: string;
-}
-
-table PlasmaDeleteReply {
- // ID of the object that was deleted.
- object_id: string;
- // Error code.
- error: PlasmaError;
-}
-
-table PlasmaStatusRequest {
- // IDs of the objects stored at local Plasma store we request the status of.
- object_ids: [string];
-}
-
-enum ObjectStatus:int {
- // Object is stored in the local Plasma Store.
- Local = 1,
- // Object is stored on a remote Plasma store, and it is not stored on the
- // local Plasma Store.
- Remote,
- // Object is not stored in the system.
- Nonexistent,
- // Object is currently transferred from a remote Plasma store the the local
- // Plasma Store.
- Transfer
-}
-
-table PlasmaStatusReply {
- // IDs of the objects being returned.
- object_ids: [string];
- // Status of the object.
- status: [ObjectStatus];
-}
-
-// PlasmaContains is a subset of PlasmaStatus which does not
-// involve the plasma manager, only the store. We should consider
-// unifying them in the future and deprecating PlasmaContains.
-
-table PlasmaContainsRequest {
- // ID of the object we are querying.
- object_id: string;
-}
-
-table PlasmaContainsReply {
- // ID of the object we are querying.
- object_id: string;
- // 1 if the object is in the store and 0 otherwise.
- has_object: int;
-}
-
-// PlasmaConnect is used by a plasma client the first time it connects with the
-// store. This is not really necessary, but is used to get some information
-// about the store such as its memory capacity.
-
-table PlasmaConnectRequest {
-}
-
-table PlasmaConnectReply {
- // The memory capacity of the store.
- memory_capacity: long;
-}
-
-table PlasmaEvictRequest {
- // Number of bytes that shall be freed.
- num_bytes: ulong;
-}
-
-table PlasmaEvictReply {
- // Number of bytes that have been freed.
- num_bytes: ulong;
-}
-
-table PlasmaFetchRequest {
- // IDs of objects to be gotten.
- object_ids: [string];
-}
-
-table ObjectRequestSpec {
- // ID of the object.
- object_id: string;
- // The type of the object. This specifies whether we
- // will be waiting for an object store in the local or
- // global Plasma store.
- type: int;
-}
-
-table PlasmaWaitRequest {
- // Array of object requests whose status we are asking for.
- object_requests: [ObjectRequestSpec];
- // Number of objects expected to be returned, if available.
- num_ready_objects: int;
- // timeout
- timeout: long;
-}
-
-table ObjectReply {
- // ID of the object.
- object_id: string;
- // The object status. This specifies where the object is stored.
- status: int;
-}
-
-table PlasmaWaitReply {
- // Array of object requests being returned.
- object_requests: [ObjectReply];
- // Number of objects expected to be returned, if available.
- num_ready_objects: int;
-}
-
-table PlasmaSubscribeRequest {
-}
-
-table PlasmaDataRequest {
- // ID of the object that is requested.
- object_id: string;
- // The host address where the data shall be sent to.
- address: string;
- // The port of the manager the data shall be sent to.
- port: int;
-}
-
-table PlasmaDataReply {
- // ID of the object that will be sent.
- object_id: string;
- // Size of the object data in bytes.
- object_size: ulong;
- // Size of the metadata in bytes.
- metadata_size: ulong;
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/io.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc
deleted file mode 100644
index 5875ebb..0000000
--- a/cpp/src/plasma/io.cc
+++ /dev/null
@@ -1,212 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/io.h"
-
-#include "plasma/common.h"
-
-using arrow::Status;
-
-/* Number of times we try binding to a socket. */
-#define NUM_BIND_ATTEMPTS 5
-#define BIND_TIMEOUT_MS 100
-
-/* Number of times we try connecting to a socket. */
-#define NUM_CONNECT_ATTEMPTS 50
-#define CONNECT_TIMEOUT_MS 100
-
-Status WriteBytes(int fd, uint8_t* cursor, size_t length) {
- ssize_t nbytes = 0;
- size_t bytesleft = length;
- size_t offset = 0;
- while (bytesleft > 0) {
- /* While we haven't written the whole message, write to the file descriptor,
- * advance the cursor, and decrease the amount left to write. */
- nbytes = write(fd, cursor + offset, bytesleft);
- if (nbytes < 0) {
- if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
- return Status::IOError(std::string(strerror(errno)));
- } else if (nbytes == 0) {
- return Status::IOError("Encountered unexpected EOF");
- }
- ARROW_CHECK(nbytes > 0);
- bytesleft -= nbytes;
- offset += nbytes;
- }
-
- return Status::OK();
-}
-
-Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes) {
- int64_t version = PLASMA_PROTOCOL_VERSION;
- RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)));
- RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&type), sizeof(type)));
- RETURN_NOT_OK(WriteBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)));
- return WriteBytes(fd, bytes, length * sizeof(char));
-}
-
-Status ReadBytes(int fd, uint8_t* cursor, size_t length) {
- ssize_t nbytes = 0;
- /* Termination condition: EOF or read 'length' bytes total. */
- size_t bytesleft = length;
- size_t offset = 0;
- while (bytesleft > 0) {
- nbytes = read(fd, cursor + offset, bytesleft);
- if (nbytes < 0) {
- if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { continue; }
- return Status::IOError(std::string(strerror(errno)));
- } else if (0 == nbytes) {
- return Status::IOError("Encountered unexpected EOF");
- }
- ARROW_CHECK(nbytes > 0);
- bytesleft -= nbytes;
- offset += nbytes;
- }
-
- return Status::OK();
-}
-
-Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer) {
- int64_t version;
- RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&version), sizeof(version)),
- *type = DISCONNECT_CLIENT);
- ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version;
- size_t length;
- RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(type), sizeof(*type)),
- *type = DISCONNECT_CLIENT);
- RETURN_NOT_OK_ELSE(ReadBytes(fd, reinterpret_cast<uint8_t*>(&length), sizeof(length)),
- *type = DISCONNECT_CLIENT);
- if (length > buffer->size()) { buffer->resize(length); }
- RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer->data(), length), *type = DISCONNECT_CLIENT);
- return Status::OK();
-}
-
-int bind_ipc_sock(const std::string& pathname, bool shall_listen) {
- struct sockaddr_un socket_address;
- int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
- if (socket_fd < 0) {
- ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
- return -1;
- }
- /* Tell the system to allow the port to be reused. */
- int on = 1;
- if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&on),
- sizeof(on)) < 0) {
- ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname;
- close(socket_fd);
- return -1;
- }
-
- unlink(pathname.c_str());
- memset(&socket_address, 0, sizeof(socket_address));
- socket_address.sun_family = AF_UNIX;
- if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
- ARROW_LOG(ERROR) << "Socket pathname is too long.";
- close(socket_fd);
- return -1;
- }
- strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
-
- if (bind(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) != 0) {
- ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname;
- close(socket_fd);
- return -1;
- }
- if (shall_listen && listen(socket_fd, 128) == -1) {
- ARROW_LOG(ERROR) << "Could not listen to socket " << pathname;
- close(socket_fd);
- return -1;
- }
- return socket_fd;
-}
-
-int connect_ipc_sock_retry(
- const std::string& pathname, int num_retries, int64_t timeout) {
- /* Pick the default values if the user did not specify. */
- if (num_retries < 0) { num_retries = NUM_CONNECT_ATTEMPTS; }
- if (timeout < 0) { timeout = CONNECT_TIMEOUT_MS; }
-
- int fd = -1;
- for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
- fd = connect_ipc_sock(pathname);
- if (fd >= 0) { break; }
- if (num_attempts == 0) {
- ARROW_LOG(ERROR) << "Connection to socket failed for pathname " << pathname;
- }
- /* Sleep for timeout milliseconds. */
- usleep(static_cast<int>(timeout * 1000));
- }
- /* If we could not connect to the socket, exit. */
- if (fd == -1) { ARROW_LOG(FATAL) << "Could not connect to socket " << pathname; }
- return fd;
-}
-
-int connect_ipc_sock(const std::string& pathname) {
- struct sockaddr_un socket_address;
- int socket_fd;
-
- socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
- if (socket_fd < 0) {
- ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
- return -1;
- }
-
- memset(&socket_address, 0, sizeof(socket_address));
- socket_address.sun_family = AF_UNIX;
- if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
- ARROW_LOG(ERROR) << "Socket pathname is too long.";
- return -1;
- }
- strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
-
- if (connect(socket_fd, (struct sockaddr*)&socket_address, sizeof(socket_address)) !=
- 0) {
- close(socket_fd);
- return -1;
- }
-
- return socket_fd;
-}
-
-int AcceptClient(int socket_fd) {
- int client_fd = accept(socket_fd, NULL, NULL);
- if (client_fd < 0) {
- ARROW_LOG(ERROR) << "Error reading from socket.";
- return -1;
- }
- return client_fd;
-}
-
-uint8_t* read_message_async(int sock) {
- int64_t size;
- Status s = ReadBytes(sock, reinterpret_cast<uint8_t*>(&size), sizeof(int64_t));
- if (!s.ok()) {
- /* The other side has closed the socket. */
- ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
- close(sock);
- return NULL;
- }
- uint8_t* message = reinterpret_cast<uint8_t*>(malloc(size));
- s = ReadBytes(sock, message, size);
- if (!s.ok()) {
- /* The other side has closed the socket. */
- ARROW_LOG(DEBUG) << "Socket has been closed, or some other error has occurred.";
- close(sock);
- return NULL;
- }
- return message;
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/io.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/io.h b/cpp/src/plasma/io.h
deleted file mode 100644
index 43c3fb5..0000000
--- a/cpp/src/plasma/io.h
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_IO_H
-#define PLASMA_IO_H
-
-#include <inttypes.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include <string>
-#include <vector>
-
-#include "arrow/status.h"
-
-// TODO(pcm): Replace our own custom message header (message type,
-// message length, plasma protocol verion) with one that is serialized
-// using flatbuffers.
-#define PLASMA_PROTOCOL_VERSION 0x0000000000000000
-#define DISCONNECT_CLIENT 0
-
-arrow::Status WriteBytes(int fd, uint8_t* cursor, size_t length);
-
-arrow::Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t* bytes);
-
-arrow::Status ReadBytes(int fd, uint8_t* cursor, size_t length);
-
-arrow::Status ReadMessage(int fd, int64_t* type, std::vector<uint8_t>* buffer);
-
-int bind_ipc_sock(const std::string& pathname, bool shall_listen);
-
-int connect_ipc_sock(const std::string& pathname);
-
-int connect_ipc_sock_retry(const std::string& pathname, int num_retries, int64_t timeout);
-
-int AcceptClient(int socket_fd);
-
-uint8_t* read_message_async(int sock);
-
-#endif // PLASMA_IO_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/malloc.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/malloc.cc b/cpp/src/plasma/malloc.cc
deleted file mode 100644
index 97c9a16..0000000
--- a/cpp/src/plasma/malloc.cc
+++ /dev/null
@@ -1,178 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/malloc.h"
-
-#include <assert.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <unistd.h>
-
-#include <unordered_map>
-
-#include "plasma/common.h"
-
-extern "C" {
-void* fake_mmap(size_t);
-int fake_munmap(void*, int64_t);
-
-#define MMAP(s) fake_mmap(s)
-#define MUNMAP(a, s) fake_munmap(a, s)
-#define DIRECT_MMAP(s) fake_mmap(s)
-#define DIRECT_MUNMAP(a, s) fake_munmap(a, s)
-#define USE_DL_PREFIX
-#define HAVE_MORECORE 0
-#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
-#define DEFAULT_GRANULARITY ((size_t)128U * 1024U)
-
-#include "thirdparty/dlmalloc.c" // NOLINT
-
-#undef MMAP
-#undef MUNMAP
-#undef DIRECT_MMAP
-#undef DIRECT_MUNMAP
-#undef USE_DL_PREFIX
-#undef HAVE_MORECORE
-#undef DEFAULT_GRANULARITY
-}
-
-struct mmap_record {
- int fd;
- int64_t size;
-};
-
-namespace {
-
-/** Hashtable that contains one entry per segment that we got from the OS
- * via mmap. Associates the address of that segment with its file descriptor
- * and size. */
-std::unordered_map<void*, mmap_record> mmap_records;
-
-} /* namespace */
-
-constexpr int GRANULARITY_MULTIPLIER = 2;
-
-static void* pointer_advance(void* p, ptrdiff_t n) {
- return (unsigned char*)p + n;
-}
-
-static void* pointer_retreat(void* p, ptrdiff_t n) {
- return (unsigned char*)p - n;
-}
-
-static ptrdiff_t pointer_distance(void const* pfrom, void const* pto) {
- return (unsigned char const*)pto - (unsigned char const*)pfrom;
-}
-
-/* Create a buffer. This is creating a temporary file and then
- * immediately unlinking it so we do not leave traces in the system. */
-int create_buffer(int64_t size) {
- int fd;
-#ifdef _WIN32
- if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
- (DWORD)((uint64_t)size >> (CHAR_BIT * sizeof(DWORD))), (DWORD)(uint64_t)size,
- NULL)) {
- fd = -1;
- }
-#else
-#ifdef __linux__
- constexpr char file_template[] = "/dev/shm/plasmaXXXXXX";
-#else
- constexpr char file_template[] = "/tmp/plasmaXXXXXX";
-#endif
- char file_name[32];
- strncpy(file_name, file_template, 32);
- fd = mkstemp(file_name);
- if (fd < 0) return -1;
- FILE* file = fdopen(fd, "a+");
- if (!file) {
- close(fd);
- return -1;
- }
- if (unlink(file_name) != 0) {
- ARROW_LOG(FATAL) << "unlink error";
- return -1;
- }
- if (ftruncate(fd, (off_t)size) != 0) {
- ARROW_LOG(FATAL) << "ftruncate error";
- return -1;
- }
-#endif
- return fd;
-}
-
-void* fake_mmap(size_t size) {
- /* Add sizeof(size_t) so that the returned pointer is deliberately not
- * page-aligned. This ensures that the segments of memory returned by
- * fake_mmap are never contiguous. */
- size += sizeof(size_t);
-
- int fd = create_buffer(size);
- ARROW_CHECK(fd >= 0) << "Failed to create buffer during mmap";
- void* pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
- if (pointer == MAP_FAILED) { return pointer; }
-
- /* Increase dlmalloc's allocation granularity directly. */
- mparams.granularity *= GRANULARITY_MULTIPLIER;
-
- mmap_record& record = mmap_records[pointer];
- record.fd = fd;
- record.size = size;
-
- /* We lie to dlmalloc about where mapped memory actually lives. */
- pointer = pointer_advance(pointer, sizeof(size_t));
- ARROW_LOG(DEBUG) << pointer << " = fake_mmap(" << size << ")";
- return pointer;
-}
-
-int fake_munmap(void* addr, int64_t size) {
- ARROW_LOG(DEBUG) << "fake_munmap(" << addr << ", " << size << ")";
- addr = pointer_retreat(addr, sizeof(size_t));
- size += sizeof(size_t);
-
- auto entry = mmap_records.find(addr);
-
- if (entry == mmap_records.end() || entry->second.size != size) {
- /* Reject requests to munmap that don't directly match previous
- * calls to mmap, to prevent dlmalloc from trimming. */
- return -1;
- }
-
- int r = munmap(addr, size);
- if (r == 0) { close(entry->second.fd); }
-
- mmap_records.erase(entry);
- return r;
-}
-
-void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_size, ptrdiff_t* offset) {
- /* TODO(rshin): Implement a more efficient search through mmap_records. */
- for (const auto& entry : mmap_records) {
- if (addr >= entry.first && addr < pointer_advance(entry.first, entry.second.size)) {
- *fd = entry.second.fd;
- *map_size = entry.second.size;
- *offset = pointer_distance(entry.first, addr);
- return;
- }
- }
- *fd = -1;
- *map_size = 0;
- *offset = 0;
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/malloc.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/malloc.h b/cpp/src/plasma/malloc.h
deleted file mode 100644
index b4af2c8..0000000
--- a/cpp/src/plasma/malloc.h
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_MALLOC_H
-#define PLASMA_MALLOC_H
-
-#include <inttypes.h>
-#include <stddef.h>
-
-void get_malloc_mapinfo(void* addr, int* fd, int64_t* map_length, ptrdiff_t* offset);
-
-#endif // MALLOC_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/plasma.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/plasma.cc b/cpp/src/plasma/plasma.cc
deleted file mode 100644
index 559d8e7..0000000
--- a/cpp/src/plasma/plasma.cc
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/plasma.h"
-
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "plasma/common.h"
-#include "plasma/protocol.h"
-
-int warn_if_sigpipe(int status, int client_sock) {
- if (status >= 0) { return 0; }
- if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
- ARROW_LOG(WARNING) << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when "
- "sending a message to client on fd "
- << client_sock << ". The client on the other end may "
- "have hung up.";
- return errno;
- }
- ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock << ".";
- return -1; // This is never reached.
-}
-
-/**
- * This will create a new ObjectInfo buffer. The first sizeof(int64_t) bytes
- * of this buffer are the length of the remaining message and the
- * remaining message is a serialized version of the object info.
- *
- * @param object_info The object info to be serialized
- * @return The object info buffer. It is the caller's responsibility to free
- * this buffer with "delete" after it has been used.
- */
-uint8_t* create_object_info_buffer(ObjectInfoT* object_info) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreateObjectInfo(fbb, object_info);
- fbb.Finish(message);
- uint8_t* notification = new uint8_t[sizeof(int64_t) + fbb.GetSize()];
- *(reinterpret_cast<int64_t*>(notification)) = fbb.GetSize();
- memcpy(notification + sizeof(int64_t), fbb.GetBufferPointer(), fbb.GetSize());
- return notification;
-}
-
-ObjectTableEntry* get_object_table_entry(
- PlasmaStoreInfo* store_info, const ObjectID& object_id) {
- auto it = store_info->objects.find(object_id);
- if (it == store_info->objects.end()) { return NULL; }
- return it->second.get();
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/plasma.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/plasma.h b/cpp/src/plasma/plasma.h
deleted file mode 100644
index 275d0c7..0000000
--- a/cpp/src/plasma/plasma.h
+++ /dev/null
@@ -1,191 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_PLASMA_H
-#define PLASMA_PLASMA_H
-
-#include <errno.h>
-#include <inttypes.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h> // pid_t
-
-#include <unordered_map>
-#include <unordered_set>
-
-#include "arrow/status.h"
-#include "arrow/util/logging.h"
-#include "format/common_generated.h"
-#include "plasma/common.h"
-
-#define HANDLE_SIGPIPE(s, fd_) \
- do { \
- Status _s = (s); \
- if (!_s.ok()) { \
- if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) { \
- ARROW_LOG(WARNING) \
- << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \
- "sending a message to client on fd " \
- << fd_ << ". " \
- "The client on the other end may have hung up."; \
- } else { \
- return _s; \
- } \
- } \
- } while (0);
-
-/// Allocation granularity used in plasma for object allocation.
-#define BLOCK_SIZE 64
-
-/// Size of object hash digests.
-constexpr int64_t kDigestSize = sizeof(uint64_t);
-
-struct Client;
-
-/// Object request data structure. Used in the plasma_wait_for_objects()
-/// argument.
-typedef struct {
- /// The ID of the requested object. If ID_NIL request any object.
- ObjectID object_id;
- /// Request associated to the object. It can take one of the following values:
- /// - PLASMA_QUERY_LOCAL: return if or when the object is available in the
- /// local Plasma Store.
- /// - PLASMA_QUERY_ANYWHERE: return if or when the object is available in
- /// the system (i.e., either in the local or a remote Plasma Store).
- int type;
- /// Object status. Same as the status returned by plasma_status() function
- /// call. This is filled in by plasma_wait_for_objects1():
- /// - ObjectStatus_Local: object is ready at the local Plasma Store.
- /// - ObjectStatus_Remote: object is ready at a remote Plasma Store.
- /// - ObjectStatus_Nonexistent: object does not exist in the system.
- /// - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
- /// for being transferred or it is transferring.
- int status;
-} ObjectRequest;
-
-/// Mapping from object IDs to type and status of the request.
-typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectRequestMap;
-
-/// Handle to access memory mapped file and map it into client address space.
-typedef struct {
- /// The file descriptor of the memory mapped file in the store. It is used as
- /// a unique identifier of the file in the client to look up the corresponding
- /// file descriptor on the client's side.
- int store_fd;
- /// The size in bytes of the memory mapped file.
- int64_t mmap_size;
-} object_handle;
-
-// TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
-typedef struct {
- /// Handle for memory mapped file the object is stored in.
- object_handle handle;
- /// The offset in bytes in the memory mapped file of the data.
- ptrdiff_t data_offset;
- /// The offset in bytes in the memory mapped file of the metadata.
- ptrdiff_t metadata_offset;
- /// The size in bytes of the data.
- int64_t data_size;
- /// The size in bytes of the metadata.
- int64_t metadata_size;
-} PlasmaObject;
-
-typedef enum {
- /// Object was created but not sealed in the local Plasma Store.
- PLASMA_CREATED = 1,
- /// Object is sealed and stored in the local Plasma Store.
- PLASMA_SEALED
-} object_state;
-
-typedef enum {
- /// The object was not found.
- OBJECT_NOT_FOUND = 0,
- /// The object was found.
- OBJECT_FOUND = 1
-} object_status;
-
-typedef enum {
- /// Query for object in the local plasma store.
- PLASMA_QUERY_LOCAL = 1,
- /// Query for object in the local plasma store or in a remote plasma store.
- PLASMA_QUERY_ANYWHERE
-} object_request_type;
-
-/// This type is used by the Plasma store. It is here because it is exposed to
-/// the eviction policy.
-struct ObjectTableEntry {
- /// Object id of this object.
- ObjectID object_id;
- /// Object info like size, creation time and owner.
- ObjectInfoT info;
- /// Memory mapped file containing the object.
- int fd;
- /// Size of the underlying map.
- int64_t map_size;
- /// Offset from the base of the mmap.
- ptrdiff_t offset;
- /// Pointer to the object data. Needed to free the object.
- uint8_t* pointer;
- /// Set of clients currently using this object.
- std::unordered_set<Client*> clients;
- /// The state of the object, e.g., whether it is open or sealed.
- object_state state;
- /// The digest of the object. Used to see if two objects are the same.
- unsigned char digest[kDigestSize];
-};
-
-/// The plasma store information that is exposed to the eviction policy.
-struct PlasmaStoreInfo {
- /// Objects that are in the Plasma store.
- std::unordered_map<ObjectID, std::unique_ptr<ObjectTableEntry>, UniqueIDHasher> objects;
- /// The amount of memory (in bytes) that we allow to be allocated in the
- /// store.
- int64_t memory_capacity;
-};
-
-/// Get an entry from the object table and return NULL if the object_id
-/// is not present.
-///
-/// @param store_info The PlasmaStoreInfo that contains the object table.
-/// @param object_id The object_id of the entry we are looking for.
-/// @return The entry associated with the object_id or NULL if the object_id
-/// is not present.
-ObjectTableEntry* get_object_table_entry(
- PlasmaStoreInfo* store_info, const ObjectID& object_id);
-
-/// Print a warning if the status is less than zero. This should be used to check
-/// the success of messages sent to plasma clients. We print a warning instead of
-/// failing because the plasma clients are allowed to die. This is used to handle
-/// situations where the store writes to a client file descriptor, and the client
-/// may already have disconnected. If we have processed the disconnection and
-/// closed the file descriptor, we should get a BAD FILE DESCRIPTOR error. If we
-/// have not, then we should get a SIGPIPE. If we write to a TCP socket that
-/// isn't connected yet, then we should get an ECONNRESET.
-///
-/// @param status The status to check. If it is less less than zero, we will
-/// print a warning.
-/// @param client_sock The client socket. This is just used to print some extra
-/// information.
-/// @return The errno set.
-int warn_if_sigpipe(int status, int client_sock);
-
-uint8_t* create_object_info_buffer(ObjectInfoT* object_info);
-
-#endif // PLASMA_PLASMA_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/protocol.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc
deleted file mode 100644
index 246aa29..0000000
--- a/cpp/src/plasma/protocol.cc
+++ /dev/null
@@ -1,502 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/protocol.h"
-
-#include "flatbuffers/flatbuffers.h"
-#include "format/plasma_generated.h"
-
-#include "plasma/common.h"
-#include "plasma/io.h"
-
-using flatbuffers::uoffset_t;
-
-flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
-to_flatbuffer(flatbuffers::FlatBufferBuilder* fbb, const ObjectID* object_ids,
- int64_t num_objects) {
- std::vector<flatbuffers::Offset<flatbuffers::String>> results;
- for (int64_t i = 0; i < num_objects; i++) {
- results.push_back(fbb->CreateString(object_ids[i].binary()));
- }
- return fbb->CreateVector(results);
-}
-
-Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer) {
- int64_t type;
- RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
- ARROW_CHECK(type == message_type) << "type = " << type
- << ", message_type = " << message_type;
- return Status::OK();
-}
-
-template <typename Message>
-Status PlasmaSend(int sock, int64_t message_type, flatbuffers::FlatBufferBuilder* fbb,
- const Message& message) {
- fbb->Finish(message);
- return WriteMessage(sock, message_type, fbb->GetSize(), fbb->GetBufferPointer());
-}
-
-// Create messages.
-
-Status SendCreateRequest(
- int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaCreateRequest(
- fbb, fbb.CreateString(object_id.binary()), data_size, metadata_size);
- return PlasmaSend(sock, MessageType_PlasmaCreateRequest, &fbb, message);
-}
-
-Status ReadCreateRequest(
- uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
- *data_size = message->data_size();
- *metadata_size = message->metadata_size();
- *object_id = ObjectID::from_binary(message->object_id()->str());
- return Status::OK();
-}
-
-Status SendCreateReply(
- int sock, ObjectID object_id, PlasmaObject* object, int error_code) {
- flatbuffers::FlatBufferBuilder fbb;
- PlasmaObjectSpec plasma_object(object->handle.store_fd, object->handle.mmap_size,
- object->data_offset, object->data_size, object->metadata_offset,
- object->metadata_size);
- auto message = CreatePlasmaCreateReply(
- fbb, fbb.CreateString(object_id.binary()), &plasma_object, (PlasmaError)error_code);
- return PlasmaSend(sock, MessageType_PlasmaCreateReply, &fbb, message);
-}
-
-Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- object->handle.store_fd = message->plasma_object()->segment_index();
- object->handle.mmap_size = message->plasma_object()->mmap_size();
- object->data_offset = message->plasma_object()->data_offset();
- object->data_size = message->plasma_object()->data_size();
- object->metadata_offset = message->plasma_object()->metadata_offset();
- object->metadata_size = message->plasma_object()->metadata_size();
- return plasma_error_status(message->error());
-}
-
-// Seal messages.
-
-Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest) {
- flatbuffers::FlatBufferBuilder fbb;
- auto digest_string = fbb.CreateString(reinterpret_cast<char*>(digest), kDigestSize);
- auto message =
- CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()), digest_string);
- return PlasmaSend(sock, MessageType_PlasmaSealRequest, &fbb, message);
-}
-
-Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaSealRequest>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- ARROW_CHECK(message->digest()->size() == kDigestSize);
- memcpy(digest, message->digest()->data(), kDigestSize);
- return Status::OK();
-}
-
-Status SendSealReply(int sock, ObjectID object_id, int error) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaSealReply(
- fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
- return PlasmaSend(sock, MessageType_PlasmaSealReply, &fbb, message);
-}
-
-Status ReadSealReply(uint8_t* data, ObjectID* object_id) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaSealReply>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- return plasma_error_status(message->error());
-}
-
-// Release messages.
-
-Status SendReleaseRequest(int sock, ObjectID object_id) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary()));
- return PlasmaSend(sock, MessageType_PlasmaReleaseRequest, &fbb, message);
-}
-
-Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaReleaseRequest>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- return Status::OK();
-}
-
-Status SendReleaseReply(int sock, ObjectID object_id, int error) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaReleaseReply(
- fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
- return PlasmaSend(sock, MessageType_PlasmaReleaseReply, &fbb, message);
-}
-
-Status ReadReleaseReply(uint8_t* data, ObjectID* object_id) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- return plasma_error_status(message->error());
-}
-
-// Delete messages.
-
-Status SendDeleteRequest(int sock, ObjectID object_id) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaDeleteRequest(fbb, fbb.CreateString(object_id.binary()));
- return PlasmaSend(sock, MessageType_PlasmaDeleteRequest, &fbb, message);
-}
-
-Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- return Status::OK();
-}
-
-Status SendDeleteReply(int sock, ObjectID object_id, int error) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaDeleteReply(
- fbb, fbb.CreateString(object_id.binary()), (PlasmaError)error);
- return PlasmaSend(sock, MessageType_PlasmaDeleteReply, &fbb, message);
-}
-
-Status ReadDeleteReply(uint8_t* data, ObjectID* object_id) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaDeleteReply>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- return plasma_error_status(message->error());
-}
-
-// Satus messages.
-
-Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message =
- CreatePlasmaStatusRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
- return PlasmaSend(sock, MessageType_PlasmaStatusRequest, &fbb, message);
-}
-
-Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaStatusRequest>(data);
- for (uoffset_t i = 0; i < num_objects; ++i) {
- object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
- }
- return Status::OK();
-}
-
-Status SendStatusReply(
- int sock, ObjectID object_ids[], int object_status[], int64_t num_objects) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message =
- CreatePlasmaStatusReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
- fbb.CreateVector(object_status, num_objects));
- return PlasmaSend(sock, MessageType_PlasmaStatusReply, &fbb, message);
-}
-
-int64_t ReadStatusReply_num_objects(uint8_t* data) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
- return message->object_ids()->size();
-}
-
-Status ReadStatusReply(
- uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
- for (uoffset_t i = 0; i < num_objects; ++i) {
- object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
- }
- for (uoffset_t i = 0; i < num_objects; ++i) {
- object_status[i] = message->status()->data()[i];
- }
- return Status::OK();
-}
-
-// Contains messages.
-
-Status SendContainsRequest(int sock, ObjectID object_id) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaContainsRequest(fbb, fbb.CreateString(object_id.binary()));
- return PlasmaSend(sock, MessageType_PlasmaContainsRequest, &fbb, message);
-}
-
-Status ReadContainsRequest(uint8_t* data, ObjectID* object_id) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaContainsRequest>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- return Status::OK();
-}
-
-Status SendContainsReply(int sock, ObjectID object_id, bool has_object) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message =
- CreatePlasmaContainsReply(fbb, fbb.CreateString(object_id.binary()), has_object);
- return PlasmaSend(sock, MessageType_PlasmaContainsReply, &fbb, message);
-}
-
-Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaContainsReply>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- *has_object = message->has_object();
- return Status::OK();
-}
-
-// Connect messages.
-
-Status SendConnectRequest(int sock) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaConnectRequest(fbb);
- return PlasmaSend(sock, MessageType_PlasmaConnectRequest, &fbb, message);
-}
-
-Status ReadConnectRequest(uint8_t* data) {
- return Status::OK();
-}
-
-Status SendConnectReply(int sock, int64_t memory_capacity) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaConnectReply(fbb, memory_capacity);
- return PlasmaSend(sock, MessageType_PlasmaConnectReply, &fbb, message);
-}
-
-Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaConnectReply>(data);
- *memory_capacity = message->memory_capacity();
- return Status::OK();
-}
-
-// Evict messages.
-
-Status SendEvictRequest(int sock, int64_t num_bytes) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaEvictRequest(fbb, num_bytes);
- return PlasmaSend(sock, MessageType_PlasmaEvictRequest, &fbb, message);
-}
-
-Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaEvictRequest>(data);
- *num_bytes = message->num_bytes();
- return Status::OK();
-}
-
-Status SendEvictReply(int sock, int64_t num_bytes) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaEvictReply(fbb, num_bytes);
- return PlasmaSend(sock, MessageType_PlasmaEvictReply, &fbb, message);
-}
-
-Status ReadEvictReply(uint8_t* data, int64_t& num_bytes) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaEvictReply>(data);
- num_bytes = message->num_bytes();
- return Status::OK();
-}
-
-// Get messages.
-
-Status SendGetRequest(
- int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaGetRequest(
- fbb, to_flatbuffer(&fbb, object_ids, num_objects), timeout_ms);
- return PlasmaSend(sock, MessageType_PlasmaGetRequest, &fbb, message);
-}
-
-Status ReadGetRequest(
- uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
- for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
- auto object_id = message->object_ids()->Get(i)->str();
- object_ids.push_back(ObjectID::from_binary(object_id));
- }
- *timeout_ms = message->timeout_ms();
- return Status::OK();
-}
-
-Status SendGetReply(int sock, ObjectID object_ids[],
- std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
- int64_t num_objects) {
- flatbuffers::FlatBufferBuilder fbb;
- std::vector<PlasmaObjectSpec> objects;
-
- for (int i = 0; i < num_objects; ++i) {
- const PlasmaObject& object = plasma_objects[object_ids[i]];
- objects.push_back(PlasmaObjectSpec(object.handle.store_fd, object.handle.mmap_size,
- object.data_offset, object.data_size, object.metadata_offset,
- object.metadata_size));
- }
- auto message = CreatePlasmaGetReply(fbb, to_flatbuffer(&fbb, object_ids, num_objects),
- fbb.CreateVectorOfStructs(objects.data(), num_objects));
- return PlasmaSend(sock, MessageType_PlasmaGetReply, &fbb, message);
-}
-
-Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
- int64_t num_objects) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
- for (uoffset_t i = 0; i < num_objects; ++i) {
- object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
- }
- for (uoffset_t i = 0; i < num_objects; ++i) {
- const PlasmaObjectSpec* object = message->plasma_objects()->Get(i);
- plasma_objects[i].handle.store_fd = object->segment_index();
- plasma_objects[i].handle.mmap_size = object->mmap_size();
- plasma_objects[i].data_offset = object->data_offset();
- plasma_objects[i].data_size = object->data_size();
- plasma_objects[i].metadata_offset = object->metadata_offset();
- plasma_objects[i].metadata_size = object->metadata_size();
- }
- return Status::OK();
-}
-
-// Fetch messages.
-
-Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message =
- CreatePlasmaFetchRequest(fbb, to_flatbuffer(&fbb, object_ids, num_objects));
- return PlasmaSend(sock, MessageType_PlasmaFetchRequest, &fbb, message);
-}
-
-Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaFetchRequest>(data);
- for (uoffset_t i = 0; i < message->object_ids()->size(); ++i) {
- object_ids.push_back(ObjectID::from_binary(message->object_ids()->Get(i)->str()));
- }
- return Status::OK();
-}
-
-// Wait messages.
-
-Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
- int num_ready_objects, int64_t timeout_ms) {
- flatbuffers::FlatBufferBuilder fbb;
-
- std::vector<flatbuffers::Offset<ObjectRequestSpec>> object_request_specs;
- for (int i = 0; i < num_requests; i++) {
- object_request_specs.push_back(CreateObjectRequestSpec(fbb,
- fbb.CreateString(object_requests[i].object_id.binary()),
- object_requests[i].type));
- }
-
- auto message = CreatePlasmaWaitRequest(
- fbb, fbb.CreateVector(object_request_specs), num_ready_objects, timeout_ms);
- return PlasmaSend(sock, MessageType_PlasmaWaitRequest, &fbb, message);
-}
-
-Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
- int64_t* timeout_ms, int* num_ready_objects) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaWaitRequest>(data);
- *num_ready_objects = message->num_ready_objects();
- *timeout_ms = message->timeout();
-
- for (uoffset_t i = 0; i < message->object_requests()->size(); i++) {
- ObjectID object_id =
- ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
- ObjectRequest object_request({object_id, message->object_requests()->Get(i)->type(),
- ObjectStatus_Nonexistent});
- object_requests[object_id] = object_request;
- }
- return Status::OK();
-}
-
-Status SendWaitReply(
- int sock, const ObjectRequestMap& object_requests, int num_ready_objects) {
- flatbuffers::FlatBufferBuilder fbb;
-
- std::vector<flatbuffers::Offset<ObjectReply>> object_replies;
- for (const auto& entry : object_requests) {
- const auto& object_request = entry.second;
- object_replies.push_back(CreateObjectReply(
- fbb, fbb.CreateString(object_request.object_id.binary()), object_request.status));
- }
-
- auto message = CreatePlasmaWaitReply(
- fbb, fbb.CreateVector(object_replies.data(), num_ready_objects), num_ready_objects);
- return PlasmaSend(sock, MessageType_PlasmaWaitReply, &fbb, message);
-}
-
-Status ReadWaitReply(
- uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects) {
- DCHECK(data);
-
- auto message = flatbuffers::GetRoot<PlasmaWaitReply>(data);
- *num_ready_objects = message->num_ready_objects();
- for (int i = 0; i < *num_ready_objects; i++) {
- object_requests[i].object_id =
- ObjectID::from_binary(message->object_requests()->Get(i)->object_id()->str());
- object_requests[i].status = message->object_requests()->Get(i)->status();
- }
- return Status::OK();
-}
-
-// Subscribe messages.
-
-Status SendSubscribeRequest(int sock) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaSubscribeRequest(fbb);
- return PlasmaSend(sock, MessageType_PlasmaSubscribeRequest, &fbb, message);
-}
-
-// Data messages.
-
-Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port) {
- flatbuffers::FlatBufferBuilder fbb;
- auto addr = fbb.CreateString(address, strlen(address));
- auto message =
- CreatePlasmaDataRequest(fbb, fbb.CreateString(object_id.binary()), addr, port);
- return PlasmaSend(sock, MessageType_PlasmaDataRequest, &fbb, message);
-}
-
-Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaDataRequest>(data);
- DCHECK(message->object_id()->size() == sizeof(ObjectID));
- *object_id = ObjectID::from_binary(message->object_id()->str());
- *address = strdup(message->address()->c_str());
- *port = message->port();
- return Status::OK();
-}
-
-Status SendDataReply(
- int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size) {
- flatbuffers::FlatBufferBuilder fbb;
- auto message = CreatePlasmaDataReply(
- fbb, fbb.CreateString(object_id.binary()), object_size, metadata_size);
- return PlasmaSend(sock, MessageType_PlasmaDataReply, &fbb, message);
-}
-
-Status ReadDataReply(
- uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size) {
- DCHECK(data);
- auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
- *object_id = ObjectID::from_binary(message->object_id()->str());
- *object_size = (int64_t)message->object_size();
- *metadata_size = (int64_t)message->metadata_size();
- return Status::OK();
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/protocol.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h
deleted file mode 100644
index 5d9d136..0000000
--- a/cpp/src/plasma/protocol.h
+++ /dev/null
@@ -1,170 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_PROTOCOL_H
-#define PLASMA_PROTOCOL_H
-
-#include <vector>
-
-#include "arrow/status.h"
-#include "format/plasma_generated.h"
-#include "plasma/plasma.h"
-
-using arrow::Status;
-
-/* Plasma receive message. */
-
-Status PlasmaReceive(int sock, int64_t message_type, std::vector<uint8_t>* buffer);
-
-/* Plasma Create message functions. */
-
-Status SendCreateRequest(
- int sock, ObjectID object_id, int64_t data_size, int64_t metadata_size);
-
-Status ReadCreateRequest(
- uint8_t* data, ObjectID* object_id, int64_t* data_size, int64_t* metadata_size);
-
-Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int error);
-
-Status ReadCreateReply(uint8_t* data, ObjectID* object_id, PlasmaObject* object);
-
-/* Plasma Seal message functions. */
-
-Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest);
-
-Status ReadSealRequest(uint8_t* data, ObjectID* object_id, unsigned char* digest);
-
-Status SendSealReply(int sock, ObjectID object_id, int error);
-
-Status ReadSealReply(uint8_t* data, ObjectID* object_id);
-
-/* Plasma Get message functions. */
-
-Status SendGetRequest(
- int sock, const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms);
-
-Status ReadGetRequest(
- uint8_t* data, std::vector<ObjectID>& object_ids, int64_t* timeout_ms);
-
-Status SendGetReply(int sock, ObjectID object_ids[],
- std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher>& plasma_objects,
- int64_t num_objects);
-
-Status ReadGetReply(uint8_t* data, ObjectID object_ids[], PlasmaObject plasma_objects[],
- int64_t num_objects);
-
-/* Plasma Release message functions. */
-
-Status SendReleaseRequest(int sock, ObjectID object_id);
-
-Status ReadReleaseRequest(uint8_t* data, ObjectID* object_id);
-
-Status SendReleaseReply(int sock, ObjectID object_id, int error);
-
-Status ReadReleaseReply(uint8_t* data, ObjectID* object_id);
-
-/* Plasma Delete message functions. */
-
-Status SendDeleteRequest(int sock, ObjectID object_id);
-
-Status ReadDeleteRequest(uint8_t* data, ObjectID* object_id);
-
-Status SendDeleteReply(int sock, ObjectID object_id, int error);
-
-Status ReadDeleteReply(uint8_t* data, ObjectID* object_id);
-
-/* Satus messages. */
-
-Status SendStatusRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
-
-Status ReadStatusRequest(uint8_t* data, ObjectID object_ids[], int64_t num_objects);
-
-Status SendStatusReply(
- int sock, ObjectID object_ids[], int object_status[], int64_t num_objects);
-
-int64_t ReadStatusReply_num_objects(uint8_t* data);
-
-Status ReadStatusReply(
- uint8_t* data, ObjectID object_ids[], int object_status[], int64_t num_objects);
-
-/* Plasma Constains message functions. */
-
-Status SendContainsRequest(int sock, ObjectID object_id);
-
-Status ReadContainsRequest(uint8_t* data, ObjectID* object_id);
-
-Status SendContainsReply(int sock, ObjectID object_id, bool has_object);
-
-Status ReadContainsReply(uint8_t* data, ObjectID* object_id, bool* has_object);
-
-/* Plasma Connect message functions. */
-
-Status SendConnectRequest(int sock);
-
-Status ReadConnectRequest(uint8_t* data);
-
-Status SendConnectReply(int sock, int64_t memory_capacity);
-
-Status ReadConnectReply(uint8_t* data, int64_t* memory_capacity);
-
-/* Plasma Evict message functions (no reply so far). */
-
-Status SendEvictRequest(int sock, int64_t num_bytes);
-
-Status ReadEvictRequest(uint8_t* data, int64_t* num_bytes);
-
-Status SendEvictReply(int sock, int64_t num_bytes);
-
-Status ReadEvictReply(uint8_t* data, int64_t& num_bytes);
-
-/* Plasma Fetch Remote message functions. */
-
-Status SendFetchRequest(int sock, const ObjectID* object_ids, int64_t num_objects);
-
-Status ReadFetchRequest(uint8_t* data, std::vector<ObjectID>& object_ids);
-
-/* Plasma Wait message functions. */
-
-Status SendWaitRequest(int sock, ObjectRequest object_requests[], int64_t num_requests,
- int num_ready_objects, int64_t timeout_ms);
-
-Status ReadWaitRequest(uint8_t* data, ObjectRequestMap& object_requests,
- int64_t* timeout_ms, int* num_ready_objects);
-
-Status SendWaitReply(
- int sock, const ObjectRequestMap& object_requests, int num_ready_objects);
-
-Status ReadWaitReply(
- uint8_t* data, ObjectRequest object_requests[], int* num_ready_objects);
-
-/* Plasma Subscribe message functions. */
-
-Status SendSubscribeRequest(int sock);
-
-/* Data messages. */
-
-Status SendDataRequest(int sock, ObjectID object_id, const char* address, int port);
-
-Status ReadDataRequest(uint8_t* data, ObjectID* object_id, char** address, int* port);
-
-Status SendDataReply(
- int sock, ObjectID object_id, int64_t object_size, int64_t metadata_size);
-
-Status ReadDataReply(
- uint8_t* data, ObjectID* object_id, int64_t* object_size, int64_t* metadata_size);
-
-#endif /* PLASMA_PROTOCOL */
[11/14] arrow git commit: [C++] Restore Plasma source tree after
0.5.0 release
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/ae/ae_evport.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae_evport.c b/cpp/src/plasma/thirdparty/ae/ae_evport.c
new file mode 100644
index 0000000..5c317be
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_evport.c
@@ -0,0 +1,320 @@
+/* ae.c module for illumos event ports.
+ *
+ * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <assert.h>
+#include <errno.h>
+#include <port.h>
+#include <poll.h>
+
+#include <sys/types.h>
+#include <sys/time.h>
+
+#include <stdio.h>
+
+static int evport_debug = 0;
+
+/*
+ * This file implements the ae API using event ports, present on Solaris-based
+ * systems since Solaris 10. Using the event port interface, we associate file
+ * descriptors with the port. Each association also includes the set of poll(2)
+ * events that the consumer is interested in (e.g., POLLIN and POLLOUT).
+ *
+ * There's one tricky piece to this implementation: when we return events via
+ * aeApiPoll, the corresponding file descriptors become dissociated from the
+ * port. This is necessary because poll events are level-triggered, so if the
+ * fd didn't become dissociated, it would immediately fire another event since
+ * the underlying state hasn't changed yet. We must re-associate the file
+ * descriptor, but only after we know that our caller has actually read from it.
+ * The ae API does not tell us exactly when that happens, but we do know that
+ * it must happen by the time aeApiPoll is called again. Our solution is to
+ * keep track of the last fds returned by aeApiPoll and re-associate them next
+ * time aeApiPoll is invoked.
+ *
+ * To summarize, in this module, each fd association is EITHER (a) represented
+ * only via the in-kernel association OR (b) represented by pending_fds and
+ * pending_masks. (b) is only true for the last fds we returned from aeApiPoll,
+ * and only until we enter aeApiPoll again (at which point we restore the
+ * in-kernel association).
+ */
+#define MAX_EVENT_BATCHSZ 512
+
+typedef struct aeApiState {
+ int portfd; /* event port */
+ int npending; /* # of pending fds */
+ int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */
+ int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+ int i;
+ aeApiState *state = zmalloc(sizeof(aeApiState));
+ if (!state) return -1;
+
+ state->portfd = port_create();
+ if (state->portfd == -1) {
+ zfree(state);
+ return -1;
+ }
+
+ state->npending = 0;
+
+ for (i = 0; i < MAX_EVENT_BATCHSZ; i++) {
+ state->pending_fds[i] = -1;
+ state->pending_masks[i] = AE_NONE;
+ }
+
+ eventLoop->apidata = state;
+ return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+ /* Nothing to resize here. */
+ return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+ aeApiState *state = eventLoop->apidata;
+
+ close(state->portfd);
+ zfree(state);
+}
+
+static int aeApiLookupPending(aeApiState *state, int fd) {
+ int i;
+
+ for (i = 0; i < state->npending; i++) {
+ if (state->pending_fds[i] == fd)
+ return (i);
+ }
+
+ return (-1);
+}
+
+/*
+ * Helper function to invoke port_associate for the given fd and mask.
+ */
+static int aeApiAssociate(const char *where, int portfd, int fd, int mask) {
+ int events = 0;
+ int rv, err;
+
+ if (mask & AE_READABLE)
+ events |= POLLIN;
+ if (mask & AE_WRITABLE)
+ events |= POLLOUT;
+
+ if (evport_debug)
+ fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events);
+
+ rv = port_associate(portfd, PORT_SOURCE_FD, fd, events,
+ (void *)(uintptr_t)mask);
+ err = errno;
+
+ if (evport_debug)
+ fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err));
+
+ if (rv == -1) {
+ fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err));
+
+ if (err == EAGAIN)
+ fprintf(stderr, "aeApiAssociate: event port limit exceeded.");
+ }
+
+ return rv;
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+ aeApiState *state = eventLoop->apidata;
+ int fullmask, pfd;
+
+ if (evport_debug)
+ fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask);
+
+ /*
+ * Since port_associate's "events" argument replaces any existing events, we
+ * must be sure to include whatever events are already associated when
+ * we call port_associate() again.
+ */
+ fullmask = mask | eventLoop->events[fd].mask;
+ pfd = aeApiLookupPending(state, fd);
+
+ if (pfd != -1) {
+ /*
+ * This fd was recently returned from aeApiPoll. It should be safe to
+ * assume that the consumer has processed that poll event, but we play
+ * it safer by simply updating pending_mask. The fd will be
+ * re-associated as usual when aeApiPoll is called again.
+ */
+ if (evport_debug)
+ fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd);
+ state->pending_masks[pfd] |= fullmask;
+ return 0;
+ }
+
+ return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask));
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+ aeApiState *state = eventLoop->apidata;
+ int fullmask, pfd;
+
+ if (evport_debug)
+ fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask);
+
+ pfd = aeApiLookupPending(state, fd);
+
+ if (pfd != -1) {
+ if (evport_debug)
+ fprintf(stderr, "deleting event from pending fd %d\n", fd);
+
+ /*
+ * This fd was just returned from aeApiPoll, so it's not currently
+ * associated with the port. All we need to do is update
+ * pending_mask appropriately.
+ */
+ state->pending_masks[pfd] &= ~mask;
+
+ if (state->pending_masks[pfd] == AE_NONE)
+ state->pending_fds[pfd] = -1;
+
+ return;
+ }
+
+ /*
+ * The fd is currently associated with the port. Like with the add case
+ * above, we must look at the full mask for the file descriptor before
+ * updating that association. We don't have a good way of knowing what the
+ * events are without looking into the eventLoop state directly. We rely on
+ * the fact that our caller has already updated the mask in the eventLoop.
+ */
+
+ fullmask = eventLoop->events[fd].mask;
+ if (fullmask == AE_NONE) {
+ /*
+ * We're removing *all* events, so use port_dissociate to remove the
+ * association completely. Failure here indicates a bug.
+ */
+ if (evport_debug)
+ fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd);
+
+ if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) {
+ perror("aeApiDelEvent: port_dissociate");
+ abort(); /* will not return */
+ }
+ } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd,
+ fullmask) != 0) {
+ /*
+ * ENOMEM is a potentially transient condition, but the kernel won't
+ * generally return it unless things are really bad. EAGAIN indicates
+ * we've reached an resource limit, for which it doesn't make sense to
+ * retry (counter-intuitively). All other errors indicate a bug. In any
+ * of these cases, the best we can do is to abort.
+ */
+ abort(); /* will not return */
+ }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+ aeApiState *state = eventLoop->apidata;
+ struct timespec timeout, *tsp;
+ int mask, i;
+ uint_t nevents;
+ port_event_t event[MAX_EVENT_BATCHSZ];
+
+ /*
+ * If we've returned fd events before, we must re-associate them with the
+ * port now, before calling port_get(). See the block comment at the top of
+ * this file for an explanation of why.
+ */
+ for (i = 0; i < state->npending; i++) {
+ if (state->pending_fds[i] == -1)
+ /* This fd has since been deleted. */
+ continue;
+
+ if (aeApiAssociate("aeApiPoll", state->portfd,
+ state->pending_fds[i], state->pending_masks[i]) != 0) {
+ /* See aeApiDelEvent for why this case is fatal. */
+ abort();
+ }
+
+ state->pending_masks[i] = AE_NONE;
+ state->pending_fds[i] = -1;
+ }
+
+ state->npending = 0;
+
+ if (tvp != NULL) {
+ timeout.tv_sec = tvp->tv_sec;
+ timeout.tv_nsec = tvp->tv_usec * 1000;
+ tsp = &timeout;
+ } else {
+ tsp = NULL;
+ }
+
+ /*
+ * port_getn can return with errno == ETIME having returned some events (!).
+ * So if we get ETIME, we check nevents, too.
+ */
+ nevents = 1;
+ if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents,
+ tsp) == -1 && (errno != ETIME || nevents == 0)) {
+ if (errno == ETIME || errno == EINTR)
+ return 0;
+
+ /* Any other error indicates a bug. */
+ perror("aeApiPoll: port_get");
+ abort();
+ }
+
+ state->npending = nevents;
+
+ for (i = 0; i < nevents; i++) {
+ mask = 0;
+ if (event[i].portev_events & POLLIN)
+ mask |= AE_READABLE;
+ if (event[i].portev_events & POLLOUT)
+ mask |= AE_WRITABLE;
+
+ eventLoop->fired[i].fd = event[i].portev_object;
+ eventLoop->fired[i].mask = mask;
+
+ if (evport_debug)
+ fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n",
+ (int)event[i].portev_object, mask);
+
+ state->pending_fds[i] = event[i].portev_object;
+ state->pending_masks[i] = (uintptr_t)event[i].portev_user;
+ }
+
+ return nevents;
+}
+
+static char *aeApiName(void) {
+ return "evport";
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae_kqueue.c b/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
new file mode 100644
index 0000000..6796f4c
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
@@ -0,0 +1,138 @@
+/* Kqueue(2)-based ae.c module
+ *
+ * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/time.h>
+
+typedef struct aeApiState {
+ int kqfd;
+ struct kevent *events;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+ aeApiState *state = zmalloc(sizeof(aeApiState));
+
+ if (!state) return -1;
+ state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize);
+ if (!state->events) {
+ zfree(state);
+ return -1;
+ }
+ state->kqfd = kqueue();
+ if (state->kqfd == -1) {
+ zfree(state->events);
+ zfree(state);
+ return -1;
+ }
+ eventLoop->apidata = state;
+ return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+ aeApiState *state = eventLoop->apidata;
+
+ state->events = zrealloc(state->events, sizeof(struct kevent)*setsize);
+ return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+ aeApiState *state = eventLoop->apidata;
+
+ close(state->kqfd);
+ zfree(state->events);
+ zfree(state);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+ aeApiState *state = eventLoop->apidata;
+ struct kevent ke;
+
+ if (mask & AE_READABLE) {
+ EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
+ if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
+ }
+ if (mask & AE_WRITABLE) {
+ EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL);
+ if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
+ }
+ return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+ aeApiState *state = eventLoop->apidata;
+ struct kevent ke;
+
+ if (mask & AE_READABLE) {
+ EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
+ kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
+ }
+ if (mask & AE_WRITABLE) {
+ EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
+ kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
+ }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+ aeApiState *state = eventLoop->apidata;
+ int retval, numevents = 0;
+
+ if (tvp != NULL) {
+ struct timespec timeout;
+ timeout.tv_sec = tvp->tv_sec;
+ timeout.tv_nsec = tvp->tv_usec * 1000;
+ retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
+ &timeout);
+ } else {
+ retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
+ NULL);
+ }
+
+ if (retval > 0) {
+ int j;
+
+ numevents = retval;
+ for(j = 0; j < numevents; j++) {
+ int mask = 0;
+ struct kevent *e = state->events+j;
+
+ if (e->filter == EVFILT_READ) mask |= AE_READABLE;
+ if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
+ eventLoop->fired[j].fd = e->ident;
+ eventLoop->fired[j].mask = mask;
+ }
+ }
+ return numevents;
+}
+
+static char *aeApiName(void) {
+ return "kqueue";
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/ae/ae_select.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae_select.c b/cpp/src/plasma/thirdparty/ae/ae_select.c
new file mode 100644
index 0000000..c039a8e
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_select.c
@@ -0,0 +1,106 @@
+/* Select()-based ae.c module.
+ *
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/select.h>
+#include <string.h>
+
+typedef struct aeApiState {
+ fd_set rfds, wfds;
+ /* We need to have a copy of the fd sets as it's not safe to reuse
+ * FD sets after select(). */
+ fd_set _rfds, _wfds;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+ aeApiState *state = zmalloc(sizeof(aeApiState));
+
+ if (!state) return -1;
+ FD_ZERO(&state->rfds);
+ FD_ZERO(&state->wfds);
+ eventLoop->apidata = state;
+ return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+ /* Just ensure we have enough room in the fd_set type. */
+ if (setsize >= FD_SETSIZE) return -1;
+ return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+ zfree(eventLoop->apidata);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+ aeApiState *state = eventLoop->apidata;
+
+ if (mask & AE_READABLE) FD_SET(fd,&state->rfds);
+ if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds);
+ return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
+ aeApiState *state = eventLoop->apidata;
+
+ if (mask & AE_READABLE) FD_CLR(fd,&state->rfds);
+ if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds);
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+ aeApiState *state = eventLoop->apidata;
+ int retval, j, numevents = 0;
+
+ memcpy(&state->_rfds,&state->rfds,sizeof(fd_set));
+ memcpy(&state->_wfds,&state->wfds,sizeof(fd_set));
+
+ retval = select(eventLoop->maxfd+1,
+ &state->_rfds,&state->_wfds,NULL,tvp);
+ if (retval > 0) {
+ for (j = 0; j <= eventLoop->maxfd; j++) {
+ int mask = 0;
+ aeFileEvent *fe = &eventLoop->events[j];
+
+ if (fe->mask == AE_NONE) continue;
+ if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds))
+ mask |= AE_READABLE;
+ if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds))
+ mask |= AE_WRITABLE;
+ eventLoop->fired[numevents].fd = j;
+ eventLoop->fired[numevents].mask = mask;
+ numevents++;
+ }
+ }
+ return numevents;
+}
+
+static char *aeApiName(void) {
+ return "select";
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/ae/config.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/config.h b/cpp/src/plasma/thirdparty/ae/config.h
new file mode 100644
index 0000000..4f8e1ea
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/config.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __CONFIG_H
+#define __CONFIG_H
+
+#ifdef __APPLE__
+#include <AvailabilityMacros.h>
+#endif
+
+/* Test for polling API */
+#ifdef __linux__
+#define HAVE_EPOLL 1
+#endif
+
+#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__)
+#define HAVE_KQUEUE 1
+#endif
+
+#ifdef __sun
+#include <sys/feature_tests.h>
+#ifdef _DTRACE_VERSION
+#define HAVE_EVPORT 1
+#endif
+#endif
+
+
+#endif
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/ae/zmalloc.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/zmalloc.h b/cpp/src/plasma/thirdparty/ae/zmalloc.h
new file mode 100644
index 0000000..6c27dd4
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/zmalloc.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _ZMALLOC_H
+#define _ZMALLOC_H
+
+#ifndef zmalloc
+#define zmalloc malloc
+#endif
+
+#ifndef zfree
+#define zfree free
+#endif
+
+#ifndef zrealloc
+#define zrealloc realloc
+#endif
+
+#endif /* _ZMALLOC_H */
[04/14] arrow git commit: [C++] Remove Plasma source tree for 0.5.0
release pending IP Clearance
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/store.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
deleted file mode 100644
index 9394e3d..0000000
--- a/cpp/src/plasma/store.cc
+++ /dev/null
@@ -1,683 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// PLASMA STORE: This is a simple object store server process
-//
-// It accepts incoming client connections on a unix domain socket
-// (name passed in via the -s option of the executable) and uses a
-// single thread to serve the clients. Each client establishes a
-// connection and can create objects, wait for objects and seal
-// objects through that connection.
-//
-// It keeps a hash table that maps object_ids (which are 20 byte long,
-// just enough to store and SHA1 hash) to memory mapped files.
-
-#include "plasma/store.h"
-
-#include <assert.h>
-#include <fcntl.h>
-#include <getopt.h>
-#include <limits.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/statvfs.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include <deque>
-#include <string>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-
-#include "format/common_generated.h"
-#include "plasma/common.h"
-#include "plasma/fling.h"
-#include "plasma/io.h"
-#include "plasma/malloc.h"
-
-extern "C" {
-void* dlmalloc(size_t bytes);
-void* dlmemalign(size_t alignment, size_t bytes);
-void dlfree(void* mem);
-size_t dlmalloc_set_footprint_limit(size_t bytes);
-}
-
-struct GetRequest {
- GetRequest(Client* client, const std::vector<ObjectID>& object_ids);
- /// The client that called get.
- Client* client;
- /// The ID of the timer that will time out and cause this wait to return to
- /// the client if it hasn't already returned.
- int64_t timer;
- /// The object IDs involved in this request. This is used in the reply.
- std::vector<ObjectID> object_ids;
- /// The object information for the objects in this request. This is used in
- /// the reply.
- std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> objects;
- /// The minimum number of objects to wait for in this request.
- int64_t num_objects_to_wait_for;
- /// The number of object requests in this wait request that are already
- /// satisfied.
- int64_t num_satisfied;
-};
-
-GetRequest::GetRequest(Client* client, const std::vector<ObjectID>& object_ids)
- : client(client),
- timer(-1),
- object_ids(object_ids.begin(), object_ids.end()),
- objects(object_ids.size()),
- num_satisfied(0) {
- std::unordered_set<ObjectID, UniqueIDHasher> unique_ids(
- object_ids.begin(), object_ids.end());
- num_objects_to_wait_for = unique_ids.size();
-}
-
-Client::Client(int fd) : fd(fd) {}
-
-PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory)
- : loop_(loop), eviction_policy_(&store_info_) {
- store_info_.memory_capacity = system_memory;
-}
-
-// TODO(pcm): Get rid of this destructor by using RAII to clean up data.
-PlasmaStore::~PlasmaStore() {
- for (const auto& element : pending_notifications_) {
- auto object_notifications = element.second.object_notifications;
- for (size_t i = 0; i < object_notifications.size(); ++i) {
- uint8_t* notification = reinterpret_cast<uint8_t*>(object_notifications.at(i));
- uint8_t* data = notification;
- // TODO(pcm): Get rid of this delete.
- delete[] data;
- }
- }
-}
-
-// If this client is not already using the object, add the client to the
-// object's list of clients, otherwise do nothing.
-void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client* client) {
- // Check if this client is already using the object.
- if (entry->clients.find(client) != entry->clients.end()) { return; }
- // If there are no other clients using this object, notify the eviction policy
- // that the object is being used.
- if (entry->clients.size() == 0) {
- // Tell the eviction policy that this object is being used.
- std::vector<ObjectID> objects_to_evict;
- eviction_policy_.begin_object_access(entry->object_id, &objects_to_evict);
- delete_objects(objects_to_evict);
- }
- // Add the client pointer to the list of clients using this object.
- entry->clients.insert(client);
-}
-
-// Create a new object buffer in the hash table.
-int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
- int64_t metadata_size, Client* client, PlasmaObject* result) {
- ARROW_LOG(DEBUG) << "creating object " << object_id.hex();
- if (store_info_.objects.count(object_id) != 0) {
- // There is already an object with the same ID in the Plasma Store, so
- // ignore this requst.
- return PlasmaError_ObjectExists;
- }
- // Try to evict objects until there is enough space.
- uint8_t* pointer;
- do {
- // Allocate space for the new object. We use dlmemalign instead of dlmalloc
- // in order to align the allocated region to a 64-byte boundary. This is not
- // strictly necessary, but it is an optimization that could speed up the
- // computation of a hash of the data (see compute_object_hash_parallel in
- // plasma_client.cc). Note that even though this pointer is 64-byte aligned,
- // it is not guaranteed that the corresponding pointer in the client will be
- // 64-byte aligned, but in practice it often will be.
- pointer =
- reinterpret_cast<uint8_t*>(dlmemalign(BLOCK_SIZE, data_size + metadata_size));
- if (pointer == NULL) {
- // Tell the eviction policy how much space we need to create this object.
- std::vector<ObjectID> objects_to_evict;
- bool success =
- eviction_policy_.require_space(data_size + metadata_size, &objects_to_evict);
- delete_objects(objects_to_evict);
- // Return an error to the client if not enough space could be freed to
- // create the object.
- if (!success) { return PlasmaError_OutOfMemory; }
- }
- } while (pointer == NULL);
- int fd;
- int64_t map_size;
- ptrdiff_t offset;
- get_malloc_mapinfo(pointer, &fd, &map_size, &offset);
- assert(fd != -1);
-
- auto entry = std::unique_ptr<ObjectTableEntry>(new ObjectTableEntry());
- entry->object_id = object_id;
- entry->info.object_id = object_id.binary();
- entry->info.data_size = data_size;
- entry->info.metadata_size = metadata_size;
- entry->pointer = pointer;
- // TODO(pcm): Set the other fields.
- entry->fd = fd;
- entry->map_size = map_size;
- entry->offset = offset;
- entry->state = PLASMA_CREATED;
-
- store_info_.objects[object_id] = std::move(entry);
- result->handle.store_fd = fd;
- result->handle.mmap_size = map_size;
- result->data_offset = offset;
- result->metadata_offset = offset + data_size;
- result->data_size = data_size;
- result->metadata_size = metadata_size;
- // Notify the eviction policy that this object was created. This must be done
- // immediately before the call to add_client_to_object_clients so that the
- // eviction policy does not have an opportunity to evict the object.
- eviction_policy_.object_created(object_id);
- // Record that this client is using this object.
- add_client_to_object_clients(store_info_.objects[object_id].get(), client);
- return PlasmaError_OK;
-}
-
-void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
- DCHECK(object != NULL);
- DCHECK(entry != NULL);
- DCHECK(entry->state == PLASMA_SEALED);
- object->handle.store_fd = entry->fd;
- object->handle.mmap_size = entry->map_size;
- object->data_offset = entry->offset;
- object->metadata_offset = entry->offset + entry->info.data_size;
- object->data_size = entry->info.data_size;
- object->metadata_size = entry->info.metadata_size;
-}
-
-void PlasmaStore::return_from_get(GetRequest* get_req) {
- // Send the get reply to the client.
- Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], get_req->objects,
- get_req->object_ids.size());
- warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd);
- // If we successfully sent the get reply message to the client, then also send
- // the file descriptors.
- if (s.ok()) {
- // Send all of the file descriptors for the present objects.
- for (const auto& object_id : get_req->object_ids) {
- PlasmaObject& object = get_req->objects[object_id];
- // We use the data size to indicate whether the object is present or not.
- if (object.data_size != -1) {
- int error_code = send_fd(get_req->client->fd, object.handle.store_fd);
- // If we failed to send the file descriptor, loop until we have sent it
- // successfully. TODO(rkn): This is problematic for two reasons. First
- // of all, sending the file descriptor should just succeed without any
- // errors, but sometimes I see a "Message too long" error number.
- // Second, looping like this allows a client to potentially block the
- // plasma store event loop which should never happen.
- while (error_code < 0) {
- if (errno == EMSGSIZE) {
- ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying.";
- error_code = send_fd(get_req->client->fd, object.handle.store_fd);
- continue;
- }
- warn_if_sigpipe(error_code, get_req->client->fd);
- break;
- }
- }
- }
- }
-
- // Remove the get request from each of the relevant object_get_requests hash
- // tables if it is present there. It should only be present there if the get
- // request timed out.
- for (ObjectID& object_id : get_req->object_ids) {
- auto& get_requests = object_get_requests_[object_id];
- // Erase get_req from the vector.
- auto it = std::find(get_requests.begin(), get_requests.end(), get_req);
- if (it != get_requests.end()) { get_requests.erase(it); }
- }
- // Remove the get request.
- if (get_req->timer != -1) { ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK); }
- delete get_req;
-}
-
-void PlasmaStore::update_object_get_requests(const ObjectID& object_id) {
- std::vector<GetRequest*>& get_requests = object_get_requests_[object_id];
- size_t index = 0;
- size_t num_requests = get_requests.size();
- for (size_t i = 0; i < num_requests; ++i) {
- GetRequest* get_req = get_requests[index];
- auto entry = get_object_table_entry(&store_info_, object_id);
- ARROW_CHECK(entry != NULL);
-
- PlasmaObject_init(&get_req->objects[object_id], entry);
- get_req->num_satisfied += 1;
- // Record the fact that this client will be using this object and will
- // be responsible for releasing this object.
- add_client_to_object_clients(entry, get_req->client);
-
- // If this get request is done, reply to the client.
- if (get_req->num_satisfied == get_req->num_objects_to_wait_for) {
- return_from_get(get_req);
- } else {
- // The call to return_from_get will remove the current element in the
- // array, so we only increment the counter in the else branch.
- index += 1;
- }
- }
-
- DCHECK(index == get_requests.size());
- // Remove the array of get requests for this object, since no one should be
- // waiting for this object anymore.
- object_get_requests_.erase(object_id);
-}
-
-void PlasmaStore::process_get_request(
- Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms) {
- // Create a get request for this object.
- GetRequest* get_req = new GetRequest(client, object_ids);
-
- for (auto object_id : object_ids) {
- // Check if this object is already present locally. If so, record that the
- // object is being used and mark it as accounted for.
- auto entry = get_object_table_entry(&store_info_, object_id);
- if (entry && entry->state == PLASMA_SEALED) {
- // Update the get request to take into account the present object.
- PlasmaObject_init(&get_req->objects[object_id], entry);
- get_req->num_satisfied += 1;
- // If necessary, record that this client is using this object. In the case
- // where entry == NULL, this will be called from seal_object.
- add_client_to_object_clients(entry, client);
- } else {
- // Add a placeholder plasma object to the get request to indicate that the
- // object is not present. This will be parsed by the client. We set the
- // data size to -1 to indicate that the object is not present.
- get_req->objects[object_id].data_size = -1;
- // Add the get request to the relevant data structures.
- object_get_requests_[object_id].push_back(get_req);
- }
- }
-
- // If all of the objects are present already or if the timeout is 0, return to
- // the client.
- if (get_req->num_satisfied == get_req->num_objects_to_wait_for || timeout_ms == 0) {
- return_from_get(get_req);
- } else if (timeout_ms != -1) {
- // Set a timer that will cause the get request to return to the client. Note
- // that a timeout of -1 is used to indicate that no timer should be set.
- get_req->timer = loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) {
- return_from_get(get_req);
- return kEventLoopTimerDone;
- });
- }
-}
-
-int PlasmaStore::remove_client_from_object_clients(
- ObjectTableEntry* entry, Client* client) {
- auto it = entry->clients.find(client);
- if (it != entry->clients.end()) {
- entry->clients.erase(it);
- // If no more clients are using this object, notify the eviction policy
- // that the object is no longer being used.
- if (entry->clients.size() == 0) {
- // Tell the eviction policy that this object is no longer being used.
- std::vector<ObjectID> objects_to_evict;
- eviction_policy_.end_object_access(entry->object_id, &objects_to_evict);
- delete_objects(objects_to_evict);
- }
- // Return 1 to indicate that the client was removed.
- return 1;
- } else {
- // Return 0 to indicate that the client was not removed.
- return 0;
- }
-}
-
-void PlasmaStore::release_object(const ObjectID& object_id, Client* client) {
- auto entry = get_object_table_entry(&store_info_, object_id);
- ARROW_CHECK(entry != NULL);
- // Remove the client from the object's array of clients.
- ARROW_CHECK(remove_client_from_object_clients(entry, client) == 1);
-}
-
-// Check if an object is present.
-int PlasmaStore::contains_object(const ObjectID& object_id) {
- auto entry = get_object_table_entry(&store_info_, object_id);
- return entry && (entry->state == PLASMA_SEALED) ? OBJECT_FOUND : OBJECT_NOT_FOUND;
-}
-
-// Seal an object that has been created in the hash table.
-void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[]) {
- ARROW_LOG(DEBUG) << "sealing object " << object_id.hex();
- auto entry = get_object_table_entry(&store_info_, object_id);
- ARROW_CHECK(entry != NULL);
- ARROW_CHECK(entry->state == PLASMA_CREATED);
- // Set the state of object to SEALED.
- entry->state = PLASMA_SEALED;
- // Set the object digest.
- entry->info.digest = std::string(reinterpret_cast<char*>(&digest[0]), kDigestSize);
- // Inform all subscribers that a new object has been sealed.
- push_notification(&entry->info);
-
- // Update all get requests that involve this object.
- update_object_get_requests(object_id);
-}
-
-void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
- for (const auto& object_id : object_ids) {
- ARROW_LOG(DEBUG) << "deleting object " << object_id.hex();
- auto entry = get_object_table_entry(&store_info_, object_id);
- // TODO(rkn): This should probably not fail, but should instead throw an
- // error. Maybe we should also support deleting objects that have been
- // created but not sealed.
- ARROW_CHECK(entry != NULL) << "To delete an object it must be in the object table.";
- ARROW_CHECK(entry->state == PLASMA_SEALED)
- << "To delete an object it must have been sealed.";
- ARROW_CHECK(entry->clients.size() == 0)
- << "To delete an object, there must be no clients currently using it.";
- dlfree(entry->pointer);
- store_info_.objects.erase(object_id);
- // Inform all subscribers that the object has been deleted.
- ObjectInfoT notification;
- notification.object_id = object_id.binary();
- notification.is_deletion = true;
- push_notification(¬ification);
- }
-}
-
-void PlasmaStore::connect_client(int listener_sock) {
- int client_fd = AcceptClient(listener_sock);
- // This is freed in disconnect_client.
- Client* client = new Client(client_fd);
- // Add a callback to handle events on this socket.
- // TODO(pcm): Check return value.
- loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) {
- Status s = process_message(client);
- if (!s.ok()) { ARROW_LOG(FATAL) << "Failed to process file event: " << s; }
- });
- ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
-}
-
-void PlasmaStore::disconnect_client(Client* client) {
- ARROW_CHECK(client != NULL);
- ARROW_CHECK(client->fd > 0);
- loop_->remove_file_event(client->fd);
- // Close the socket.
- close(client->fd);
- ARROW_LOG(INFO) << "Disconnecting client on fd " << client->fd;
- // If this client was using any objects, remove it from the appropriate
- // lists.
- for (const auto& entry : store_info_.objects) {
- remove_client_from_object_clients(entry.second.get(), client);
- }
- // Note, the store may still attempt to send a message to the disconnected
- // client (for example, when an object ID that the client was waiting for
- // is ready). In these cases, the attempt to send the message will fail, but
- // the store should just ignore the failure.
- delete client;
-}
-
-/// Send notifications about sealed objects to the subscribers. This is called
-/// in seal_object. If the socket's send buffer is full, the notification will
-/// be
-/// buffered, and this will be called again when the send buffer has room.
-///
-/// @param client The client to send the notification to.
-/// @return Void.
-void PlasmaStore::send_notifications(int client_fd) {
- auto it = pending_notifications_.find(client_fd);
-
- int num_processed = 0;
- bool closed = false;
- // Loop over the array of pending notifications and send as many of them as
- // possible.
- for (size_t i = 0; i < it->second.object_notifications.size(); ++i) {
- uint8_t* notification =
- reinterpret_cast<uint8_t*>(it->second.object_notifications.at(i));
- // Decode the length, which is the first bytes of the message.
- int64_t size = *(reinterpret_cast<int64_t*>(notification));
-
- // Attempt to send a notification about this object ID.
- ssize_t nbytes = send(client_fd, notification, sizeof(int64_t) + size, 0);
- if (nbytes >= 0) {
- ARROW_CHECK(nbytes == static_cast<ssize_t>(sizeof(int64_t)) + size);
- } else if (nbytes == -1 &&
- (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
- ARROW_LOG(DEBUG) << "The socket's send buffer is full, so we are caching this "
- "notification and will send it later.";
- // Add a callback to the event loop to send queued notifications whenever
- // there is room in the socket's send buffer. Callbacks can be added
- // more than once here and will be overwritten. The callback is removed
- // at the end of the method.
- // TODO(pcm): Introduce status codes and check in case the file descriptor
- // is added twice.
- loop_->add_file_event(client_fd, kEventLoopWrite,
- [this, client_fd](int events) { send_notifications(client_fd); });
- break;
- } else {
- ARROW_LOG(WARNING) << "Failed to send notification to client on fd " << client_fd;
- if (errno == EPIPE) {
- closed = true;
- break;
- }
- }
- num_processed += 1;
- // The corresponding malloc happened in create_object_info_buffer
- // within push_notification.
- delete[] notification;
- }
- // Remove the sent notifications from the array.
- it->second.object_notifications.erase(it->second.object_notifications.begin(),
- it->second.object_notifications.begin() + num_processed);
-
- // Stop sending notifications if the pipe was broken.
- if (closed) {
- close(client_fd);
- pending_notifications_.erase(client_fd);
- }
-
- // If we have sent all notifications, remove the fd from the event loop.
- if (it->second.object_notifications.empty()) { loop_->remove_file_event(client_fd); }
-}
-
-void PlasmaStore::push_notification(ObjectInfoT* object_info) {
- for (auto& element : pending_notifications_) {
- uint8_t* notification = create_object_info_buffer(object_info);
- element.second.object_notifications.push_back(notification);
- send_notifications(element.first);
- // The notification gets freed in send_notifications when the notification
- // is sent over the socket.
- }
-}
-
-// Subscribe to notifications about sealed objects.
-void PlasmaStore::subscribe_to_updates(Client* client) {
- ARROW_LOG(DEBUG) << "subscribing to updates on fd " << client->fd;
- // TODO(rkn): The store could block here if the client doesn't send a file
- // descriptor.
- int fd = recv_fd(client->fd);
- if (fd < 0) {
- // This may mean that the client died before sending the file descriptor.
- ARROW_LOG(WARNING) << "Failed to receive file descriptor from client on fd "
- << client->fd << ".";
- return;
- }
-
- // Create a new array to buffer notifications that can't be sent to the
- // subscriber yet because the socket send buffer is full. TODO(rkn): the queue
- // never gets freed.
- // TODO(pcm): Is the following neccessary?
- pending_notifications_[fd];
-
- // Push notifications to the new subscriber about existing objects.
- for (const auto& entry : store_info_.objects) {
- push_notification(&entry.second->info);
- }
- send_notifications(fd);
-}
-
-Status PlasmaStore::process_message(Client* client) {
- int64_t type;
- Status s = ReadMessage(client->fd, &type, &input_buffer_);
- ARROW_CHECK(s.ok() || s.IsIOError());
-
- uint8_t* input = input_buffer_.data();
- ObjectID object_id;
- PlasmaObject object;
- // TODO(pcm): Get rid of the following.
- memset(&object, 0, sizeof(object));
-
- // Process the different types of requests.
- switch (type) {
- case MessageType_PlasmaCreateRequest: {
- int64_t data_size;
- int64_t metadata_size;
- RETURN_NOT_OK(ReadCreateRequest(input, &object_id, &data_size, &metadata_size));
- int error_code =
- create_object(object_id, data_size, metadata_size, client, &object);
- HANDLE_SIGPIPE(
- SendCreateReply(client->fd, object_id, &object, error_code), client->fd);
- if (error_code == PlasmaError_OK) {
- warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
- }
- } break;
- case MessageType_PlasmaGetRequest: {
- std::vector<ObjectID> object_ids_to_get;
- int64_t timeout_ms;
- RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms));
- process_get_request(client, object_ids_to_get, timeout_ms);
- } break;
- case MessageType_PlasmaReleaseRequest:
- RETURN_NOT_OK(ReadReleaseRequest(input, &object_id));
- release_object(object_id, client);
- break;
- case MessageType_PlasmaContainsRequest:
- RETURN_NOT_OK(ReadContainsRequest(input, &object_id));
- if (contains_object(object_id) == OBJECT_FOUND) {
- HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd);
- } else {
- HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd);
- }
- break;
- case MessageType_PlasmaSealRequest: {
- unsigned char digest[kDigestSize];
- RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0]));
- seal_object(object_id, &digest[0]);
- } break;
- case MessageType_PlasmaEvictRequest: {
- // This code path should only be used for testing.
- int64_t num_bytes;
- RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes));
- std::vector<ObjectID> objects_to_evict;
- int64_t num_bytes_evicted =
- eviction_policy_.choose_objects_to_evict(num_bytes, &objects_to_evict);
- delete_objects(objects_to_evict);
- HANDLE_SIGPIPE(SendEvictReply(client->fd, num_bytes_evicted), client->fd);
- } break;
- case MessageType_PlasmaSubscribeRequest:
- subscribe_to_updates(client);
- break;
- case MessageType_PlasmaConnectRequest: {
- HANDLE_SIGPIPE(
- SendConnectReply(client->fd, store_info_.memory_capacity), client->fd);
- } break;
- case DISCONNECT_CLIENT:
- ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd;
- disconnect_client(client);
- break;
- default:
- // This code should be unreachable.
- ARROW_CHECK(0);
- }
- return Status::OK();
-}
-
-// Report "success" to valgrind.
-void signal_handler(int signal) {
- if (signal == SIGTERM) { exit(0); }
-}
-
-void start_server(char* socket_name, int64_t system_memory) {
- // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
- // to a client that has already died, the store could die.
- signal(SIGPIPE, SIG_IGN);
- // Create the event loop.
- EventLoop loop;
- PlasmaStore store(&loop, system_memory);
- int socket = bind_ipc_sock(socket_name, true);
- ARROW_CHECK(socket >= 0);
- // TODO(pcm): Check return value.
- loop.add_file_event(socket, kEventLoopRead,
- [&store, socket](int events) { store.connect_client(socket); });
- loop.run();
-}
-
-int main(int argc, char* argv[]) {
- signal(SIGTERM, signal_handler);
- char* socket_name = NULL;
- int64_t system_memory = -1;
- int c;
- while ((c = getopt(argc, argv, "s:m:")) != -1) {
- switch (c) {
- case 's':
- socket_name = optarg;
- break;
- case 'm': {
- char extra;
- int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra);
- ARROW_CHECK(scanned == 1);
- ARROW_LOG(INFO) << "Allowing the Plasma store to use up to "
- << static_cast<double>(system_memory) / 1000000000
- << "GB of memory.";
- break;
- }
- default:
- exit(-1);
- }
- }
- if (!socket_name) {
- ARROW_LOG(FATAL) << "please specify socket for incoming connections with -s switch";
- }
- if (system_memory == -1) {
- ARROW_LOG(FATAL) << "please specify the amount of system memory with -m switch";
- }
-#ifdef __linux__
- // On Linux, check that the amount of memory available in /dev/shm is large
- // enough to accommodate the request. If it isn't, then fail.
- int shm_fd = open("/dev/shm", O_RDONLY);
- struct statvfs shm_vfs_stats;
- fstatvfs(shm_fd, &shm_vfs_stats);
- // The value shm_vfs_stats.f_bsize is the block size, and the value
- // shm_vfs_stats.f_bavail is the number of available blocks.
- int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail;
- close(shm_fd);
- if (system_memory > shm_mem_avail) {
- ARROW_LOG(FATAL) << "System memory request exceeds memory available in /dev/shm. The "
- "request is for "
- << system_memory << " bytes, and the amount available is "
- << shm_mem_avail
- << " bytes. You may be able to free up space by deleting files in "
- "/dev/shm. If you are inside a Docker container, you may need to "
- "pass "
- "an argument with the flag '--shm-size' to 'docker run'.";
- }
-#endif
- // Make it so dlmalloc fails if we try to request more memory than is
- // available.
- dlmalloc_set_footprint_limit((size_t)system_memory);
- ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
- start_server(socket_name, system_memory);
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/store.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
deleted file mode 100644
index 8bd9426..0000000
--- a/cpp/src/plasma/store.h
+++ /dev/null
@@ -1,169 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_STORE_H
-#define PLASMA_STORE_H
-
-#include <deque>
-#include <vector>
-
-#include "plasma/common.h"
-#include "plasma/events.h"
-#include "plasma/eviction_policy.h"
-#include "plasma/plasma.h"
-#include "plasma/protocol.h"
-
-struct GetRequest;
-
-struct NotificationQueue {
- /// The object notifications for clients. We notify the client about the
- /// objects in the order that the objects were sealed or deleted.
- std::deque<uint8_t*> object_notifications;
-};
-
-/// Contains all information that is associated with a Plasma store client.
-struct Client {
- explicit Client(int fd);
-
- /// The file descriptor used to communicate with the client.
- int fd;
-};
-
-class PlasmaStore {
- public:
- PlasmaStore(EventLoop* loop, int64_t system_memory);
-
- ~PlasmaStore();
-
- /// Create a new object. The client must do a call to release_object to tell
- /// the store when it is done with the object.
- ///
- /// @param object_id Object ID of the object to be created.
- /// @param data_size Size in bytes of the object to be created.
- /// @param metadata_size Size in bytes of the object metadata.
- /// @return One of the following error codes:
- /// - PlasmaError_OK, if the object was created successfully.
- /// - PlasmaError_ObjectExists, if an object with this ID is already
- /// present in the store. In this case, the client should not call
- /// plasma_release.
- /// - PlasmaError_OutOfMemory, if the store is out of memory and
- /// cannot create the object. In this case, the client should not call
- /// plasma_release.
- int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
- Client* client, PlasmaObject* result);
-
- /// Delete objects that have been created in the hash table. This should only
- /// be called on objects that are returned by the eviction policy to evict.
- ///
- /// @param object_ids Object IDs of the objects to be deleted.
- /// @return Void.
- void delete_objects(const std::vector<ObjectID>& object_ids);
-
- /// Process a get request from a client. This method assumes that we will
- /// eventually have these objects sealed. If one of the objects has not yet
- /// been sealed, the client that requested the object will be notified when it
- /// is sealed.
- ///
- /// For each object, the client must do a call to release_object to tell the
- /// store when it is done with the object.
- ///
- /// @param client The client making this request.
- /// @param object_ids Object IDs of the objects to be gotten.
- /// @param timeout_ms The timeout for the get request in milliseconds.
- /// @return Void.
- void process_get_request(
- Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms);
-
- /// Seal an object. The object is now immutable and can be accessed with get.
- ///
- /// @param object_id Object ID of the object to be sealed.
- /// @param digest The digest of the object. This is used to tell if two
- /// objects
- /// with the same object ID are the same.
- /// @return Void.
- void seal_object(const ObjectID& object_id, unsigned char digest[]);
-
- /// Check if the plasma store contains an object:
- ///
- /// @param object_id Object ID that will be checked.
- /// @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if
- /// not
- int contains_object(const ObjectID& object_id);
-
- /// Record the fact that a particular client is no longer using an object.
- ///
- /// @param object_id The object ID of the object that is being released.
- /// @param client The client making this request.
- /// @param Void.
- void release_object(const ObjectID& object_id, Client* client);
-
- /// Subscribe a file descriptor to updates about new sealed objects.
- ///
- /// @param client The client making this request.
- /// @return Void.
- void subscribe_to_updates(Client* client);
-
- /// Connect a new client to the PlasmaStore.
- ///
- /// @param listener_sock The socket that is listening to incoming connections.
- /// @return Void.
- void connect_client(int listener_sock);
-
- /// Disconnect a client from the PlasmaStore.
- ///
- /// @param client The client that is disconnected.
- /// @return Void.
- void disconnect_client(Client* client);
-
- void send_notifications(int client_fd);
-
- Status process_message(Client* client);
-
- private:
- void push_notification(ObjectInfoT* object_notification);
-
- void add_client_to_object_clients(ObjectTableEntry* entry, Client* client);
-
- void return_from_get(GetRequest* get_req);
-
- void update_object_get_requests(const ObjectID& object_id);
-
- int remove_client_from_object_clients(ObjectTableEntry* entry, Client* client);
-
- /// Event loop of the plasma store.
- EventLoop* loop_;
- /// The plasma store information, including the object tables, that is exposed
- /// to the eviction policy.
- PlasmaStoreInfo store_info_;
- /// The state that is managed by the eviction policy.
- EvictionPolicy eviction_policy_;
- /// Input buffer. This is allocated only once to avoid mallocs for every
- /// call to process_message.
- std::vector<uint8_t> input_buffer_;
- /// A hash table mapping object IDs to a vector of the get requests that are
- /// waiting for the object to arrive.
- std::unordered_map<ObjectID, std::vector<GetRequest*>, UniqueIDHasher>
- object_get_requests_;
- /// The pending notifications that have not been sent to subscribers because
- /// the socket send buffers were full. This is a hash table from client file
- /// descriptor to an array of object_ids to send to that client.
- /// TODO(pcm): Consider putting this into the Client data structure and
- /// reorganize the code slightly.
- std::unordered_map<int, NotificationQueue> pending_notifications_;
-};
-
-#endif // PLASMA_STORE_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/test/client_tests.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
deleted file mode 100644
index 29b5b13..0000000
--- a/cpp/src/plasma/test/client_tests.cc
+++ /dev/null
@@ -1,132 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-#include <assert.h>
-#include <signal.h>
-#include <stdlib.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "plasma/client.h"
-#include "plasma/common.h"
-#include "plasma/plasma.h"
-#include "plasma/protocol.h"
-
-std::string g_test_executable; // NOLINT
-
-class TestPlasmaStore : public ::testing::Test {
- public:
- // TODO(pcm): At the moment, stdout of the test gets mixed up with
- // stdout of the object store. Consider changing that.
- void SetUp() {
- std::string plasma_directory =
- g_test_executable.substr(0, g_test_executable.find_last_of("/"));
- std::string plasma_command =
- plasma_directory +
- "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &";
- system(plasma_command.c_str());
- ARROW_CHECK_OK(client_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
- }
- virtual void Finish() {
- ARROW_CHECK_OK(client_.Disconnect());
- system("killall plasma_store &");
- }
-
- protected:
- PlasmaClient client_;
-};
-
-TEST_F(TestPlasmaStore, ContainsTest) {
- ObjectID object_id = ObjectID::from_random();
-
- // Test for object non-existence.
- bool has_object;
- ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
- ASSERT_EQ(has_object, false);
-
- // Test for the object being in local Plasma store.
- // First create object.
- int64_t data_size = 100;
- uint8_t metadata[] = {5};
- int64_t metadata_size = sizeof(metadata);
- uint8_t* data;
- ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
- ARROW_CHECK_OK(client_.Seal(object_id));
- // Avoid race condition of Plasma Manager waiting for notification.
- ObjectBuffer object_buffer;
- ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
- ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
- ASSERT_EQ(has_object, true);
-}
-
-TEST_F(TestPlasmaStore, GetTest) {
- ObjectID object_id = ObjectID::from_random();
- ObjectBuffer object_buffer;
-
- // Test for object non-existence.
- ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer));
- ASSERT_EQ(object_buffer.data_size, -1);
-
- // Test for the object being in local Plasma store.
- // First create object.
- int64_t data_size = 4;
- uint8_t metadata[] = {5};
- int64_t metadata_size = sizeof(metadata);
- uint8_t* data;
- ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
- for (int64_t i = 0; i < data_size; i++) {
- data[i] = static_cast<uint8_t>(i % 4);
- }
- ARROW_CHECK_OK(client_.Seal(object_id));
-
- ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
- for (int64_t i = 0; i < data_size; i++) {
- ASSERT_EQ(data[i], object_buffer.data[i]);
- }
-}
-
-TEST_F(TestPlasmaStore, MultipleGetTest) {
- ObjectID object_id1 = ObjectID::from_random();
- ObjectID object_id2 = ObjectID::from_random();
- ObjectID object_ids[2] = {object_id1, object_id2};
- ObjectBuffer object_buffer[2];
-
- int64_t data_size = 4;
- uint8_t metadata[] = {5};
- int64_t metadata_size = sizeof(metadata);
- uint8_t* data;
- ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data));
- data[0] = 1;
- ARROW_CHECK_OK(client_.Seal(object_id1));
-
- ARROW_CHECK_OK(client_.Create(object_id2, data_size, metadata, metadata_size, &data));
- data[0] = 2;
- ARROW_CHECK_OK(client_.Seal(object_id2));
-
- ARROW_CHECK_OK(client_.Get(object_ids, 2, -1, object_buffer));
- ASSERT_EQ(object_buffer[0].data[0], 1);
- ASSERT_EQ(object_buffer[1].data[0], 2);
-}
-
-int main(int argc, char** argv) {
- ::testing::InitGoogleTest(&argc, argv);
- g_test_executable = std::string(argv[0]);
- return RUN_ALL_TESTS();
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/test/run_tests.sh
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/test/run_tests.sh b/cpp/src/plasma/test/run_tests.sh
deleted file mode 100644
index 958bd08..0000000
--- a/cpp/src/plasma/test/run_tests.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Cause the script to exit if a single command fails.
-set -e
-
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
-sleep 1
-./src/plasma/manager_tests
-killall plasma_store
-./src/plasma/serialization_tests
-
-# Start the Redis shards.
-./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
-redis_pid1=$!
-./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
-redis_pid2=$!
-sleep 1
-
-# Flush the redis server
-./src/common/thirdparty/redis/src/redis-cli flushall
-# Register the shard location with the primary shard.
-./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
-./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
-sleep 1
-./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
-plasma1_pid=$!
-./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
-plasma2_pid=$!
-./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 &
-plasma3_pid=$!
-./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
-plasma4_pid=$!
-sleep 1
-
-./src/plasma/client_tests
-
-kill $plasma4_pid
-kill $plasma3_pid
-kill $plasma2_pid
-kill $plasma1_pid
-kill $redis_pid1
-wait $redis_pid1
-kill $redis_pid2
-wait $redis_pid2
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/test/run_valgrind.sh
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/test/run_valgrind.sh b/cpp/src/plasma/test/run_valgrind.sh
deleted file mode 100644
index 0472194..0000000
--- a/cpp/src/plasma/test/run_valgrind.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Cause the script to exit if a single command fails.
-set -e
-
-./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
-sleep 1
-valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests
-killall plasma_store
-valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/test/serialization_tests.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
deleted file mode 100644
index 325cead..0000000
--- a/cpp/src/plasma/test/serialization_tests.cc
+++ /dev/null
@@ -1,388 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "plasma/common.h"
-#include "plasma/io.h"
-#include "plasma/plasma.h"
-#include "plasma/protocol.h"
-
-/**
- * Create a temporary file. Needs to be closed by the caller.
- *
- * @return File descriptor of the file.
- */
-int create_temp_file(void) {
- static char temp[] = "/tmp/tempfileXXXXXX";
- char file_name[32];
- strncpy(file_name, temp, 32);
- return mkstemp(file_name);
-}
-
-/**
- * Seek to the beginning of a file and read a message from it.
- *
- * @param fd File descriptor of the file.
- * @param message type Message type that we expect in the file.
- *
- * @return Pointer to the content of the message. Needs to be freed by the
- * caller.
- */
-std::vector<uint8_t> read_message_from_file(int fd, int message_type) {
- /* Go to the beginning of the file. */
- lseek(fd, 0, SEEK_SET);
- int64_t type;
- std::vector<uint8_t> data;
- ARROW_CHECK_OK(ReadMessage(fd, &type, &data));
- ARROW_CHECK(type == message_type);
- return data;
-}
-
-PlasmaObject random_plasma_object(void) {
- unsigned int seed = static_cast<unsigned int>(time(NULL));
- int random = rand_r(&seed);
- PlasmaObject object;
- memset(&object, 0, sizeof(object));
- object.handle.store_fd = random + 7;
- object.handle.mmap_size = random + 42;
- object.data_offset = random + 1;
- object.metadata_offset = random + 2;
- object.data_size = random + 3;
- object.metadata_size = random + 4;
- return object;
-}
-
-TEST(PlasmaSerialization, CreateRequest) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- int64_t data_size1 = 42;
- int64_t metadata_size1 = 11;
- ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateRequest);
- ObjectID object_id2;
- int64_t data_size2;
- int64_t metadata_size2;
- ARROW_CHECK_OK(
- ReadCreateRequest(data.data(), &object_id2, &data_size2, &metadata_size2));
- ASSERT_EQ(data_size1, data_size2);
- ASSERT_EQ(metadata_size1, metadata_size2);
- ASSERT_EQ(object_id1, object_id2);
- close(fd);
-}
-
-TEST(PlasmaSerialization, CreateReply) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- PlasmaObject object1 = random_plasma_object();
- ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateReply);
- ObjectID object_id2;
- PlasmaObject object2;
- memset(&object2, 0, sizeof(object2));
- ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2));
- ASSERT_EQ(object_id1, object_id2);
- ASSERT_EQ(memcmp(&object1, &object2, sizeof(object1)), 0);
- close(fd);
-}
-
-TEST(PlasmaSerialization, SealRequest) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- unsigned char digest1[kDigestSize];
- memset(&digest1[0], 7, kDigestSize);
- ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0]));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealRequest);
- ObjectID object_id2;
- unsigned char digest2[kDigestSize];
- ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0]));
- ASSERT_EQ(object_id1, object_id2);
- ASSERT_EQ(memcmp(&digest1[0], &digest2[0], kDigestSize), 0);
- close(fd);
-}
-
-TEST(PlasmaSerialization, SealReply) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealReply);
- ObjectID object_id2;
- Status s = ReadSealReply(data.data(), &object_id2);
- ASSERT_EQ(object_id1, object_id2);
- ASSERT_TRUE(s.IsPlasmaObjectExists());
- close(fd);
-}
-
-TEST(PlasmaSerialization, GetRequest) {
- int fd = create_temp_file();
- ObjectID object_ids[2];
- object_ids[0] = ObjectID::from_random();
- object_ids[1] = ObjectID::from_random();
- int64_t timeout_ms = 1234;
- ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetRequest);
- std::vector<ObjectID> object_ids_return;
- int64_t timeout_ms_return;
- ARROW_CHECK_OK(ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return));
- ASSERT_EQ(object_ids[0], object_ids_return[0]);
- ASSERT_EQ(object_ids[1], object_ids_return[1]);
- ASSERT_EQ(timeout_ms, timeout_ms_return);
- close(fd);
-}
-
-TEST(PlasmaSerialization, GetReply) {
- int fd = create_temp_file();
- ObjectID object_ids[2];
- object_ids[0] = ObjectID::from_random();
- object_ids[1] = ObjectID::from_random();
- std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> plasma_objects;
- plasma_objects[object_ids[0]] = random_plasma_object();
- plasma_objects[object_ids[1]] = random_plasma_object();
- ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetReply);
- ObjectID object_ids_return[2];
- PlasmaObject plasma_objects_return[2];
- memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
- ARROW_CHECK_OK(
- ReadGetReply(data.data(), object_ids_return, &plasma_objects_return[0], 2));
- ASSERT_EQ(object_ids[0], object_ids_return[0]);
- ASSERT_EQ(object_ids[1], object_ids_return[1]);
- ASSERT_EQ(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
- sizeof(PlasmaObject)),
- 0);
- ASSERT_EQ(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
- sizeof(PlasmaObject)),
- 0);
- close(fd);
-}
-
-TEST(PlasmaSerialization, ReleaseRequest) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1));
- std::vector<uint8_t> data =
- read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
- ObjectID object_id2;
- ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2));
- ASSERT_EQ(object_id1, object_id2);
- close(fd);
-}
-
-TEST(PlasmaSerialization, ReleaseReply) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaReleaseReply);
- ObjectID object_id2;
- Status s = ReadReleaseReply(data.data(), &object_id2);
- ASSERT_EQ(object_id1, object_id2);
- ASSERT_TRUE(s.IsPlasmaObjectExists());
- close(fd);
-}
-
-TEST(PlasmaSerialization, DeleteRequest) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
- ObjectID object_id2;
- ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2));
- ASSERT_EQ(object_id1, object_id2);
- close(fd);
-}
-
-TEST(PlasmaSerialization, DeleteReply) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- int error1 = PlasmaError_ObjectExists;
- ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteReply);
- ObjectID object_id2;
- Status s = ReadDeleteReply(data.data(), &object_id2);
- ASSERT_EQ(object_id1, object_id2);
- ASSERT_TRUE(s.IsPlasmaObjectExists());
- close(fd);
-}
-
-TEST(PlasmaSerialization, StatusRequest) {
- int fd = create_temp_file();
- int64_t num_objects = 2;
- ObjectID object_ids[num_objects];
- object_ids[0] = ObjectID::from_random();
- object_ids[1] = ObjectID::from_random();
- ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusRequest);
- ObjectID object_ids_read[num_objects];
- ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects));
- ASSERT_EQ(object_ids[0], object_ids_read[0]);
- ASSERT_EQ(object_ids[1], object_ids_read[1]);
- close(fd);
-}
-
-TEST(PlasmaSerialization, StatusReply) {
- int fd = create_temp_file();
- ObjectID object_ids[2];
- object_ids[0] = ObjectID::from_random();
- object_ids[1] = ObjectID::from_random();
- int object_statuses[2] = {42, 43};
- ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusReply);
- int64_t num_objects = ReadStatusReply_num_objects(data.data());
- ObjectID object_ids_read[num_objects];
- int object_statuses_read[num_objects];
- ARROW_CHECK_OK(
- ReadStatusReply(data.data(), object_ids_read, object_statuses_read, num_objects));
- ASSERT_EQ(object_ids[0], object_ids_read[0]);
- ASSERT_EQ(object_ids[1], object_ids_read[1]);
- ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
- ASSERT_EQ(object_statuses[1], object_statuses_read[1]);
- close(fd);
-}
-
-TEST(PlasmaSerialization, EvictRequest) {
- int fd = create_temp_file();
- int64_t num_bytes = 111;
- ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictRequest);
- int64_t num_bytes_received;
- ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received));
- ASSERT_EQ(num_bytes, num_bytes_received);
- close(fd);
-}
-
-TEST(PlasmaSerialization, EvictReply) {
- int fd = create_temp_file();
- int64_t num_bytes = 111;
- ARROW_CHECK_OK(SendEvictReply(fd, num_bytes));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictReply);
- int64_t num_bytes_received;
- ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received));
- ASSERT_EQ(num_bytes, num_bytes_received);
- close(fd);
-}
-
-TEST(PlasmaSerialization, FetchRequest) {
- int fd = create_temp_file();
- ObjectID object_ids[2];
- object_ids[0] = ObjectID::from_random();
- object_ids[1] = ObjectID::from_random();
- ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2));
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaFetchRequest);
- std::vector<ObjectID> object_ids_read;
- ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read));
- ASSERT_EQ(object_ids[0], object_ids_read[0]);
- ASSERT_EQ(object_ids[1], object_ids_read[1]);
- close(fd);
-}
-
-TEST(PlasmaSerialization, WaitRequest) {
- int fd = create_temp_file();
- const int num_objects_in = 2;
- ObjectRequest object_requests_in[num_objects_in] = {
- ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}),
- ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})};
- const int num_ready_objects_in = 1;
- int64_t timeout_ms = 1000;
-
- ARROW_CHECK_OK(SendWaitRequest(
- fd, &object_requests_in[0], num_objects_in, num_ready_objects_in, timeout_ms));
- /* Read message back. */
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitRequest);
- int num_ready_objects_out;
- int64_t timeout_ms_read;
- ObjectRequestMap object_requests_out;
- ARROW_CHECK_OK(ReadWaitRequest(
- data.data(), object_requests_out, &timeout_ms_read, &num_ready_objects_out));
- ASSERT_EQ(num_objects_in, object_requests_out.size());
- ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
- for (int i = 0; i < num_objects_in; i++) {
- const ObjectID& object_id = object_requests_in[i].object_id;
- ASSERT_EQ(1, object_requests_out.count(object_id));
- const auto& entry = object_requests_out.find(object_id);
- ASSERT_TRUE(entry != object_requests_out.end());
- ASSERT_EQ(entry->second.object_id, object_requests_in[i].object_id);
- ASSERT_EQ(entry->second.type, object_requests_in[i].type);
- }
- close(fd);
-}
-
-TEST(PlasmaSerialization, WaitReply) {
- int fd = create_temp_file();
- const int num_objects_in = 2;
- /* Create a map with two ObjectRequests in it. */
- ObjectRequestMap objects_in(num_objects_in);
- ObjectID id1 = ObjectID::from_random();
- objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local});
- ObjectID id2 = ObjectID::from_random();
- objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent});
-
- ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in));
- /* Read message back. */
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitReply);
- ObjectRequest objects_out[2];
- int num_objects_out;
- ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out));
- ASSERT_EQ(num_objects_in, num_objects_out);
- for (int i = 0; i < num_objects_out; i++) {
- /* Each object request must appear exactly once. */
- ASSERT_EQ(objects_in.count(objects_out[i].object_id), 1);
- const auto& entry = objects_in.find(objects_out[i].object_id);
- ASSERT_TRUE(entry != objects_in.end());
- ASSERT_EQ(entry->second.object_id, objects_out[i].object_id);
- ASSERT_EQ(entry->second.status, objects_out[i].status);
- }
- close(fd);
-}
-
-TEST(PlasmaSerialization, DataRequest) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- const char* address1 = "address1";
- int port1 = 12345;
- ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1));
- /* Reading message back. */
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataRequest);
- ObjectID object_id2;
- char* address2;
- int port2;
- ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2));
- ASSERT_EQ(object_id1, object_id2);
- ASSERT_EQ(strcmp(address1, address2), 0);
- ASSERT_EQ(port1, port2);
- free(address2);
- close(fd);
-}
-
-TEST(PlasmaSerialization, DataReply) {
- int fd = create_temp_file();
- ObjectID object_id1 = ObjectID::from_random();
- int64_t object_size1 = 146;
- int64_t metadata_size1 = 198;
- ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1));
- /* Reading message back. */
- std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataReply);
- ObjectID object_id2;
- int64_t object_size2;
- int64_t metadata_size2;
- ARROW_CHECK_OK(ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2));
- ASSERT_EQ(object_id1, object_id2);
- ASSERT_EQ(object_size1, object_size2);
- ASSERT_EQ(metadata_size1, metadata_size2);
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/ae/ae.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae.c b/cpp/src/plasma/thirdparty/ae/ae.c
deleted file mode 100644
index e66808a..0000000
--- a/cpp/src/plasma/thirdparty/ae/ae.c
+++ /dev/null
@@ -1,465 +0,0 @@
-/* A simple event-driven programming library. Originally I wrote this code
- * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
- * it in form of a library for easy reuse.
- *
- * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Redis nor the names of its contributors may be used
- * to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <stdio.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <poll.h>
-#include <string.h>
-#include <time.h>
-#include <errno.h>
-
-#include "ae.h"
-#include "zmalloc.h"
-#include "config.h"
-
-/* Include the best multiplexing layer supported by this system.
- * The following should be ordered by performances, descending. */
-#ifdef HAVE_EVPORT
-#include "ae_evport.c"
-#else
- #ifdef HAVE_EPOLL
- #include "ae_epoll.c"
- #else
- #ifdef HAVE_KQUEUE
- #include "ae_kqueue.c"
- #else
- #include "ae_select.c"
- #endif
- #endif
-#endif
-
-aeEventLoop *aeCreateEventLoop(int setsize) {
- aeEventLoop *eventLoop;
- int i;
-
- if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
- eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
- eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
- if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
- eventLoop->setsize = setsize;
- eventLoop->lastTime = time(NULL);
- eventLoop->timeEventHead = NULL;
- eventLoop->timeEventNextId = 0;
- eventLoop->stop = 0;
- eventLoop->maxfd = -1;
- eventLoop->beforesleep = NULL;
- if (aeApiCreate(eventLoop) == -1) goto err;
- /* Events with mask == AE_NONE are not set. So let's initialize the
- * vector with it. */
- for (i = 0; i < setsize; i++)
- eventLoop->events[i].mask = AE_NONE;
- return eventLoop;
-
-err:
- if (eventLoop) {
- zfree(eventLoop->events);
- zfree(eventLoop->fired);
- zfree(eventLoop);
- }
- return NULL;
-}
-
-/* Return the current set size. */
-int aeGetSetSize(aeEventLoop *eventLoop) {
- return eventLoop->setsize;
-}
-
-/* Resize the maximum set size of the event loop.
- * If the requested set size is smaller than the current set size, but
- * there is already a file descriptor in use that is >= the requested
- * set size minus one, AE_ERR is returned and the operation is not
- * performed at all.
- *
- * Otherwise AE_OK is returned and the operation is successful. */
-int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
- int i;
-
- if (setsize == eventLoop->setsize) return AE_OK;
- if (eventLoop->maxfd >= setsize) return AE_ERR;
- if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
-
- eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
- eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
- eventLoop->setsize = setsize;
-
- /* Make sure that if we created new slots, they are initialized with
- * an AE_NONE mask. */
- for (i = eventLoop->maxfd+1; i < setsize; i++)
- eventLoop->events[i].mask = AE_NONE;
- return AE_OK;
-}
-
-void aeDeleteEventLoop(aeEventLoop *eventLoop) {
- aeApiFree(eventLoop);
- zfree(eventLoop->events);
- zfree(eventLoop->fired);
- zfree(eventLoop);
-}
-
-void aeStop(aeEventLoop *eventLoop) {
- eventLoop->stop = 1;
-}
-
-int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
- aeFileProc *proc, void *clientData)
-{
- if (fd >= eventLoop->setsize) {
- errno = ERANGE;
- return AE_ERR;
- }
- aeFileEvent *fe = &eventLoop->events[fd];
-
- if (aeApiAddEvent(eventLoop, fd, mask) == -1)
- return AE_ERR;
- fe->mask |= mask;
- if (mask & AE_READABLE) fe->rfileProc = proc;
- if (mask & AE_WRITABLE) fe->wfileProc = proc;
- fe->clientData = clientData;
- if (fd > eventLoop->maxfd)
- eventLoop->maxfd = fd;
- return AE_OK;
-}
-
-void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
-{
- if (fd >= eventLoop->setsize) return;
- aeFileEvent *fe = &eventLoop->events[fd];
- if (fe->mask == AE_NONE) return;
-
- aeApiDelEvent(eventLoop, fd, mask);
- fe->mask = fe->mask & (~mask);
- if (fd == eventLoop->maxfd && fe->mask == AE_NONE) {
- /* Update the max fd */
- int j;
-
- for (j = eventLoop->maxfd-1; j >= 0; j--)
- if (eventLoop->events[j].mask != AE_NONE) break;
- eventLoop->maxfd = j;
- }
-}
-
-int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
- if (fd >= eventLoop->setsize) return 0;
- aeFileEvent *fe = &eventLoop->events[fd];
-
- return fe->mask;
-}
-
-static void aeGetTime(long *seconds, long *milliseconds)
-{
- struct timeval tv;
-
- gettimeofday(&tv, NULL);
- *seconds = tv.tv_sec;
- *milliseconds = tv.tv_usec/1000;
-}
-
-static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) {
- long cur_sec, cur_ms, when_sec, when_ms;
-
- aeGetTime(&cur_sec, &cur_ms);
- when_sec = cur_sec + milliseconds/1000;
- when_ms = cur_ms + milliseconds%1000;
- if (when_ms >= 1000) {
- when_sec ++;
- when_ms -= 1000;
- }
- *sec = when_sec;
- *ms = when_ms;
-}
-
-long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
- aeTimeProc *proc, void *clientData,
- aeEventFinalizerProc *finalizerProc)
-{
- long long id = eventLoop->timeEventNextId++;
- aeTimeEvent *te;
-
- te = zmalloc(sizeof(*te));
- if (te == NULL) return AE_ERR;
- te->id = id;
- aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms);
- te->timeProc = proc;
- te->finalizerProc = finalizerProc;
- te->clientData = clientData;
- te->next = eventLoop->timeEventHead;
- eventLoop->timeEventHead = te;
- return id;
-}
-
-int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
-{
- aeTimeEvent *te = eventLoop->timeEventHead;
- while(te) {
- if (te->id == id) {
- te->id = AE_DELETED_EVENT_ID;
- return AE_OK;
- }
- te = te->next;
- }
- return AE_ERR; /* NO event with the specified ID found */
-}
-
-/* Search the first timer to fire.
- * This operation is useful to know how many time the select can be
- * put in sleep without to delay any event.
- * If there are no timers NULL is returned.
- *
- * Note that's O(N) since time events are unsorted.
- * Possible optimizations (not needed by Redis so far, but...):
- * 1) Insert the event in order, so that the nearest is just the head.
- * Much better but still insertion or deletion of timers is O(N).
- * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)).
- */
-static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
-{
- aeTimeEvent *te = eventLoop->timeEventHead;
- aeTimeEvent *nearest = NULL;
-
- while(te) {
- if (!nearest || te->when_sec < nearest->when_sec ||
- (te->when_sec == nearest->when_sec &&
- te->when_ms < nearest->when_ms))
- nearest = te;
- te = te->next;
- }
- return nearest;
-}
-
-/* Process time events */
-static int processTimeEvents(aeEventLoop *eventLoop) {
- int processed = 0;
- aeTimeEvent *te, *prev;
- long long maxId;
- time_t now = time(NULL);
-
- /* If the system clock is moved to the future, and then set back to the
- * right value, time events may be delayed in a random way. Often this
- * means that scheduled operations will not be performed soon enough.
- *
- * Here we try to detect system clock skews, and force all the time
- * events to be processed ASAP when this happens: the idea is that
- * processing events earlier is less dangerous than delaying them
- * indefinitely, and practice suggests it is. */
- if (now < eventLoop->lastTime) {
- te = eventLoop->timeEventHead;
- while(te) {
- te->when_sec = 0;
- te = te->next;
- }
- }
- eventLoop->lastTime = now;
-
- prev = NULL;
- te = eventLoop->timeEventHead;
- maxId = eventLoop->timeEventNextId-1;
- while(te) {
- long now_sec, now_ms;
- long long id;
-
- /* Remove events scheduled for deletion. */
- if (te->id == AE_DELETED_EVENT_ID) {
- aeTimeEvent *next = te->next;
- if (prev == NULL)
- eventLoop->timeEventHead = te->next;
- else
- prev->next = te->next;
- if (te->finalizerProc)
- te->finalizerProc(eventLoop, te->clientData);
- zfree(te);
- te = next;
- continue;
- }
-
- /* Make sure we don't process time events created by time events in
- * this iteration. Note that this check is currently useless: we always
- * add new timers on the head, however if we change the implementation
- * detail, this check may be useful again: we keep it here for future
- * defense. */
- if (te->id > maxId) {
- te = te->next;
- continue;
- }
- aeGetTime(&now_sec, &now_ms);
- if (now_sec > te->when_sec ||
- (now_sec == te->when_sec && now_ms >= te->when_ms))
- {
- int retval;
-
- id = te->id;
- retval = te->timeProc(eventLoop, id, te->clientData);
- processed++;
- if (retval != AE_NOMORE) {
- aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
- } else {
- te->id = AE_DELETED_EVENT_ID;
- }
- }
- prev = te;
- te = te->next;
- }
- return processed;
-}
-
-/* Process every pending time event, then every pending file event
- * (that may be registered by time event callbacks just processed).
- * Without special flags the function sleeps until some file event
- * fires, or when the next time event occurs (if any).
- *
- * If flags is 0, the function does nothing and returns.
- * if flags has AE_ALL_EVENTS set, all the kind of events are processed.
- * if flags has AE_FILE_EVENTS set, file events are processed.
- * if flags has AE_TIME_EVENTS set, time events are processed.
- * if flags has AE_DONT_WAIT set the function returns ASAP until all
- * the events that's possible to process without to wait are processed.
- *
- * The function returns the number of events processed. */
-int aeProcessEvents(aeEventLoop *eventLoop, int flags)
-{
- int processed = 0, numevents;
-
- /* Nothing to do? return ASAP */
- if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0;
-
- /* Note that we want call select() even if there are no
- * file events to process as long as we want to process time
- * events, in order to sleep until the next time event is ready
- * to fire. */
- if (eventLoop->maxfd != -1 ||
- ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) {
- int j;
- aeTimeEvent *shortest = NULL;
- struct timeval tv, *tvp;
-
- if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT))
- shortest = aeSearchNearestTimer(eventLoop);
- if (shortest) {
- long now_sec, now_ms;
-
- aeGetTime(&now_sec, &now_ms);
- tvp = &tv;
-
- /* How many milliseconds we need to wait for the next
- * time event to fire? */
- long long ms =
- (shortest->when_sec - now_sec)*1000 +
- shortest->when_ms - now_ms;
-
- if (ms > 0) {
- tvp->tv_sec = ms/1000;
- tvp->tv_usec = (ms % 1000)*1000;
- } else {
- tvp->tv_sec = 0;
- tvp->tv_usec = 0;
- }
- } else {
- /* If we have to check for events but need to return
- * ASAP because of AE_DONT_WAIT we need to set the timeout
- * to zero */
- if (flags & AE_DONT_WAIT) {
- tv.tv_sec = tv.tv_usec = 0;
- tvp = &tv;
- } else {
- /* Otherwise we can block */
- tvp = NULL; /* wait forever */
- }
- }
-
- numevents = aeApiPoll(eventLoop, tvp);
- for (j = 0; j < numevents; j++) {
- aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd];
- int mask = eventLoop->fired[j].mask;
- int fd = eventLoop->fired[j].fd;
- int rfired = 0;
-
- /* note the fe->mask & mask & ... code: maybe an already processed
- * event removed an element that fired and we still didn't
- * processed, so we check if the event is still valid. */
- if (fe->mask & mask & AE_READABLE) {
- rfired = 1;
- fe->rfileProc(eventLoop,fd,fe->clientData,mask);
- }
- if (fe->mask & mask & AE_WRITABLE) {
- if (!rfired || fe->wfileProc != fe->rfileProc)
- fe->wfileProc(eventLoop,fd,fe->clientData,mask);
- }
- processed++;
- }
- }
- /* Check time events */
- if (flags & AE_TIME_EVENTS)
- processed += processTimeEvents(eventLoop);
-
- return processed; /* return the number of processed file/time events */
-}
-
-/* Wait for milliseconds until the given file descriptor becomes
- * writable/readable/exception */
-int aeWait(int fd, int mask, long long milliseconds) {
- struct pollfd pfd;
- int retmask = 0, retval;
-
- memset(&pfd, 0, sizeof(pfd));
- pfd.fd = fd;
- if (mask & AE_READABLE) pfd.events |= POLLIN;
- if (mask & AE_WRITABLE) pfd.events |= POLLOUT;
-
- if ((retval = poll(&pfd, 1, milliseconds))== 1) {
- if (pfd.revents & POLLIN) retmask |= AE_READABLE;
- if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE;
- if (pfd.revents & POLLERR) retmask |= AE_WRITABLE;
- if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE;
- return retmask;
- } else {
- return retval;
- }
-}
-
-void aeMain(aeEventLoop *eventLoop) {
- eventLoop->stop = 0;
- while (!eventLoop->stop) {
- if (eventLoop->beforesleep != NULL)
- eventLoop->beforesleep(eventLoop);
- aeProcessEvents(eventLoop, AE_ALL_EVENTS);
- }
-}
-
-char *aeGetApiName(void) {
- return aeApiName();
-}
-
-void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) {
- eventLoop->beforesleep = beforesleep;
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/ae/ae.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae.h b/cpp/src/plasma/thirdparty/ae/ae.h
deleted file mode 100644
index 827c4c9..0000000
--- a/cpp/src/plasma/thirdparty/ae/ae.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/* A simple event-driven programming library. Originally I wrote this code
- * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
- * it in form of a library for easy reuse.
- *
- * Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Redis nor the names of its contributors may be used
- * to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __AE_H__
-#define __AE_H__
-
-#include <time.h>
-
-#define AE_OK 0
-#define AE_ERR -1
-
-#define AE_NONE 0
-#define AE_READABLE 1
-#define AE_WRITABLE 2
-
-#define AE_FILE_EVENTS 1
-#define AE_TIME_EVENTS 2
-#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
-#define AE_DONT_WAIT 4
-
-#define AE_NOMORE -1
-#define AE_DELETED_EVENT_ID -1
-
-/* Macros */
-#define AE_NOTUSED(V) ((void) V)
-
-struct aeEventLoop;
-
-/* Types and data structures */
-typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
-typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData);
-typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData);
-typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop);
-
-/* File event structure */
-typedef struct aeFileEvent {
- int mask; /* one of AE_(READABLE|WRITABLE) */
- aeFileProc *rfileProc;
- aeFileProc *wfileProc;
- void *clientData;
-} aeFileEvent;
-
-/* Time event structure */
-typedef struct aeTimeEvent {
- long long id; /* time event identifier. */
- long when_sec; /* seconds */
- long when_ms; /* milliseconds */
- aeTimeProc *timeProc;
- aeEventFinalizerProc *finalizerProc;
- void *clientData;
- struct aeTimeEvent *next;
-} aeTimeEvent;
-
-/* A fired event */
-typedef struct aeFiredEvent {
- int fd;
- int mask;
-} aeFiredEvent;
-
-/* State of an event based program */
-typedef struct aeEventLoop {
- int maxfd; /* highest file descriptor currently registered */
- int setsize; /* max number of file descriptors tracked */
- long long timeEventNextId;
- time_t lastTime; /* Used to detect system clock skew */
- aeFileEvent *events; /* Registered events */
- aeFiredEvent *fired; /* Fired events */
- aeTimeEvent *timeEventHead;
- int stop;
- void *apidata; /* This is used for polling API specific data */
- aeBeforeSleepProc *beforesleep;
-} aeEventLoop;
-
-/* Prototypes */
-aeEventLoop *aeCreateEventLoop(int setsize);
-void aeDeleteEventLoop(aeEventLoop *eventLoop);
-void aeStop(aeEventLoop *eventLoop);
-int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
- aeFileProc *proc, void *clientData);
-void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask);
-int aeGetFileEvents(aeEventLoop *eventLoop, int fd);
-long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
- aeTimeProc *proc, void *clientData,
- aeEventFinalizerProc *finalizerProc);
-int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id);
-int aeProcessEvents(aeEventLoop *eventLoop, int flags);
-int aeWait(int fd, int mask, long long milliseconds);
-void aeMain(aeEventLoop *eventLoop);
-char *aeGetApiName(void);
-void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
-int aeGetSetSize(aeEventLoop *eventLoop);
-int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
-
-#endif
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/ae/ae_epoll.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae_epoll.c b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
deleted file mode 100644
index 410aac7..0000000
--- a/cpp/src/plasma/thirdparty/ae/ae_epoll.c
+++ /dev/null
@@ -1,135 +0,0 @@
-/* Linux epoll(2) based ae.c module
- *
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Redis nor the names of its contributors may be used
- * to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include <sys/epoll.h>
-
-typedef struct aeApiState {
- int epfd;
- struct epoll_event *events;
-} aeApiState;
-
-static int aeApiCreate(aeEventLoop *eventLoop) {
- aeApiState *state = zmalloc(sizeof(aeApiState));
-
- if (!state) return -1;
- state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize);
- if (!state->events) {
- zfree(state);
- return -1;
- }
- state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */
- if (state->epfd == -1) {
- zfree(state->events);
- zfree(state);
- return -1;
- }
- eventLoop->apidata = state;
- return 0;
-}
-
-static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
- aeApiState *state = eventLoop->apidata;
-
- state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize);
- return 0;
-}
-
-static void aeApiFree(aeEventLoop *eventLoop) {
- aeApiState *state = eventLoop->apidata;
-
- close(state->epfd);
- zfree(state->events);
- zfree(state);
-}
-
-static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
- aeApiState *state = eventLoop->apidata;
- struct epoll_event ee = {0}; /* avoid valgrind warning */
- /* If the fd was already monitored for some event, we need a MOD
- * operation. Otherwise we need an ADD operation. */
- int op = eventLoop->events[fd].mask == AE_NONE ?
- EPOLL_CTL_ADD : EPOLL_CTL_MOD;
-
- ee.events = 0;
- mask |= eventLoop->events[fd].mask; /* Merge old events */
- if (mask & AE_READABLE) ee.events |= EPOLLIN;
- if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
- ee.data.fd = fd;
- if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1;
- return 0;
-}
-
-static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
- aeApiState *state = eventLoop->apidata;
- struct epoll_event ee = {0}; /* avoid valgrind warning */
- int mask = eventLoop->events[fd].mask & (~delmask);
-
- ee.events = 0;
- if (mask & AE_READABLE) ee.events |= EPOLLIN;
- if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
- ee.data.fd = fd;
- if (mask != AE_NONE) {
- epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee);
- } else {
- /* Note, Kernel < 2.6.9 requires a non null event pointer even for
- * EPOLL_CTL_DEL. */
- epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee);
- }
-}
-
-static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
- aeApiState *state = eventLoop->apidata;
- int retval, numevents = 0;
-
- retval = epoll_wait(state->epfd,state->events,eventLoop->setsize,
- tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
- if (retval > 0) {
- int j;
-
- numevents = retval;
- for (j = 0; j < numevents; j++) {
- int mask = 0;
- struct epoll_event *e = state->events+j;
-
- if (e->events & EPOLLIN) mask |= AE_READABLE;
- if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
- if (e->events & EPOLLERR) mask |= AE_WRITABLE;
- if (e->events & EPOLLHUP) mask |= AE_WRITABLE;
- eventLoop->fired[j].fd = e->data.fd;
- eventLoop->fired[j].mask = mask;
- }
- }
- return numevents;
-}
-
-static char *aeApiName(void) {
- return "epoll";
-}
[03/14] arrow git commit: [C++] Remove Plasma source tree for 0.5.0
release pending IP Clearance
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/ae/ae_evport.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae_evport.c b/cpp/src/plasma/thirdparty/ae/ae_evport.c
deleted file mode 100644
index 5c317be..0000000
--- a/cpp/src/plasma/thirdparty/ae/ae_evport.c
+++ /dev/null
@@ -1,320 +0,0 @@
-/* ae.c module for illumos event ports.
- *
- * Copyright (c) 2012, Joyent, Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Redis nor the names of its contributors may be used
- * to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include <assert.h>
-#include <errno.h>
-#include <port.h>
-#include <poll.h>
-
-#include <sys/types.h>
-#include <sys/time.h>
-
-#include <stdio.h>
-
-static int evport_debug = 0;
-
-/*
- * This file implements the ae API using event ports, present on Solaris-based
- * systems since Solaris 10. Using the event port interface, we associate file
- * descriptors with the port. Each association also includes the set of poll(2)
- * events that the consumer is interested in (e.g., POLLIN and POLLOUT).
- *
- * There's one tricky piece to this implementation: when we return events via
- * aeApiPoll, the corresponding file descriptors become dissociated from the
- * port. This is necessary because poll events are level-triggered, so if the
- * fd didn't become dissociated, it would immediately fire another event since
- * the underlying state hasn't changed yet. We must re-associate the file
- * descriptor, but only after we know that our caller has actually read from it.
- * The ae API does not tell us exactly when that happens, but we do know that
- * it must happen by the time aeApiPoll is called again. Our solution is to
- * keep track of the last fds returned by aeApiPoll and re-associate them next
- * time aeApiPoll is invoked.
- *
- * To summarize, in this module, each fd association is EITHER (a) represented
- * only via the in-kernel association OR (b) represented by pending_fds and
- * pending_masks. (b) is only true for the last fds we returned from aeApiPoll,
- * and only until we enter aeApiPoll again (at which point we restore the
- * in-kernel association).
- */
-#define MAX_EVENT_BATCHSZ 512
-
-typedef struct aeApiState {
- int portfd; /* event port */
- int npending; /* # of pending fds */
- int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */
- int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */
-} aeApiState;
-
-static int aeApiCreate(aeEventLoop *eventLoop) {
- int i;
- aeApiState *state = zmalloc(sizeof(aeApiState));
- if (!state) return -1;
-
- state->portfd = port_create();
- if (state->portfd == -1) {
- zfree(state);
- return -1;
- }
-
- state->npending = 0;
-
- for (i = 0; i < MAX_EVENT_BATCHSZ; i++) {
- state->pending_fds[i] = -1;
- state->pending_masks[i] = AE_NONE;
- }
-
- eventLoop->apidata = state;
- return 0;
-}
-
-static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
- /* Nothing to resize here. */
- return 0;
-}
-
-static void aeApiFree(aeEventLoop *eventLoop) {
- aeApiState *state = eventLoop->apidata;
-
- close(state->portfd);
- zfree(state);
-}
-
-static int aeApiLookupPending(aeApiState *state, int fd) {
- int i;
-
- for (i = 0; i < state->npending; i++) {
- if (state->pending_fds[i] == fd)
- return (i);
- }
-
- return (-1);
-}
-
-/*
- * Helper function to invoke port_associate for the given fd and mask.
- */
-static int aeApiAssociate(const char *where, int portfd, int fd, int mask) {
- int events = 0;
- int rv, err;
-
- if (mask & AE_READABLE)
- events |= POLLIN;
- if (mask & AE_WRITABLE)
- events |= POLLOUT;
-
- if (evport_debug)
- fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events);
-
- rv = port_associate(portfd, PORT_SOURCE_FD, fd, events,
- (void *)(uintptr_t)mask);
- err = errno;
-
- if (evport_debug)
- fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err));
-
- if (rv == -1) {
- fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err));
-
- if (err == EAGAIN)
- fprintf(stderr, "aeApiAssociate: event port limit exceeded.");
- }
-
- return rv;
-}
-
-static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
- aeApiState *state = eventLoop->apidata;
- int fullmask, pfd;
-
- if (evport_debug)
- fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask);
-
- /*
- * Since port_associate's "events" argument replaces any existing events, we
- * must be sure to include whatever events are already associated when
- * we call port_associate() again.
- */
- fullmask = mask | eventLoop->events[fd].mask;
- pfd = aeApiLookupPending(state, fd);
-
- if (pfd != -1) {
- /*
- * This fd was recently returned from aeApiPoll. It should be safe to
- * assume that the consumer has processed that poll event, but we play
- * it safer by simply updating pending_mask. The fd will be
- * re-associated as usual when aeApiPoll is called again.
- */
- if (evport_debug)
- fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd);
- state->pending_masks[pfd] |= fullmask;
- return 0;
- }
-
- return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask));
-}
-
-static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
- aeApiState *state = eventLoop->apidata;
- int fullmask, pfd;
-
- if (evport_debug)
- fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask);
-
- pfd = aeApiLookupPending(state, fd);
-
- if (pfd != -1) {
- if (evport_debug)
- fprintf(stderr, "deleting event from pending fd %d\n", fd);
-
- /*
- * This fd was just returned from aeApiPoll, so it's not currently
- * associated with the port. All we need to do is update
- * pending_mask appropriately.
- */
- state->pending_masks[pfd] &= ~mask;
-
- if (state->pending_masks[pfd] == AE_NONE)
- state->pending_fds[pfd] = -1;
-
- return;
- }
-
- /*
- * The fd is currently associated with the port. Like with the add case
- * above, we must look at the full mask for the file descriptor before
- * updating that association. We don't have a good way of knowing what the
- * events are without looking into the eventLoop state directly. We rely on
- * the fact that our caller has already updated the mask in the eventLoop.
- */
-
- fullmask = eventLoop->events[fd].mask;
- if (fullmask == AE_NONE) {
- /*
- * We're removing *all* events, so use port_dissociate to remove the
- * association completely. Failure here indicates a bug.
- */
- if (evport_debug)
- fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd);
-
- if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) {
- perror("aeApiDelEvent: port_dissociate");
- abort(); /* will not return */
- }
- } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd,
- fullmask) != 0) {
- /*
- * ENOMEM is a potentially transient condition, but the kernel won't
- * generally return it unless things are really bad. EAGAIN indicates
- * we've reached an resource limit, for which it doesn't make sense to
- * retry (counter-intuitively). All other errors indicate a bug. In any
- * of these cases, the best we can do is to abort.
- */
- abort(); /* will not return */
- }
-}
-
-static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
- aeApiState *state = eventLoop->apidata;
- struct timespec timeout, *tsp;
- int mask, i;
- uint_t nevents;
- port_event_t event[MAX_EVENT_BATCHSZ];
-
- /*
- * If we've returned fd events before, we must re-associate them with the
- * port now, before calling port_get(). See the block comment at the top of
- * this file for an explanation of why.
- */
- for (i = 0; i < state->npending; i++) {
- if (state->pending_fds[i] == -1)
- /* This fd has since been deleted. */
- continue;
-
- if (aeApiAssociate("aeApiPoll", state->portfd,
- state->pending_fds[i], state->pending_masks[i]) != 0) {
- /* See aeApiDelEvent for why this case is fatal. */
- abort();
- }
-
- state->pending_masks[i] = AE_NONE;
- state->pending_fds[i] = -1;
- }
-
- state->npending = 0;
-
- if (tvp != NULL) {
- timeout.tv_sec = tvp->tv_sec;
- timeout.tv_nsec = tvp->tv_usec * 1000;
- tsp = &timeout;
- } else {
- tsp = NULL;
- }
-
- /*
- * port_getn can return with errno == ETIME having returned some events (!).
- * So if we get ETIME, we check nevents, too.
- */
- nevents = 1;
- if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents,
- tsp) == -1 && (errno != ETIME || nevents == 0)) {
- if (errno == ETIME || errno == EINTR)
- return 0;
-
- /* Any other error indicates a bug. */
- perror("aeApiPoll: port_get");
- abort();
- }
-
- state->npending = nevents;
-
- for (i = 0; i < nevents; i++) {
- mask = 0;
- if (event[i].portev_events & POLLIN)
- mask |= AE_READABLE;
- if (event[i].portev_events & POLLOUT)
- mask |= AE_WRITABLE;
-
- eventLoop->fired[i].fd = event[i].portev_object;
- eventLoop->fired[i].mask = mask;
-
- if (evport_debug)
- fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n",
- (int)event[i].portev_object, mask);
-
- state->pending_fds[i] = event[i].portev_object;
- state->pending_masks[i] = (uintptr_t)event[i].portev_user;
- }
-
- return nevents;
-}
-
-static char *aeApiName(void) {
- return "evport";
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae_kqueue.c b/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
deleted file mode 100644
index 6796f4c..0000000
--- a/cpp/src/plasma/thirdparty/ae/ae_kqueue.c
+++ /dev/null
@@ -1,138 +0,0 @@
-/* Kqueue(2)-based ae.c module
- *
- * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Redis nor the names of its contributors may be used
- * to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include <sys/types.h>
-#include <sys/event.h>
-#include <sys/time.h>
-
-typedef struct aeApiState {
- int kqfd;
- struct kevent *events;
-} aeApiState;
-
-static int aeApiCreate(aeEventLoop *eventLoop) {
- aeApiState *state = zmalloc(sizeof(aeApiState));
-
- if (!state) return -1;
- state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize);
- if (!state->events) {
- zfree(state);
- return -1;
- }
- state->kqfd = kqueue();
- if (state->kqfd == -1) {
- zfree(state->events);
- zfree(state);
- return -1;
- }
- eventLoop->apidata = state;
- return 0;
-}
-
-static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
- aeApiState *state = eventLoop->apidata;
-
- state->events = zrealloc(state->events, sizeof(struct kevent)*setsize);
- return 0;
-}
-
-static void aeApiFree(aeEventLoop *eventLoop) {
- aeApiState *state = eventLoop->apidata;
-
- close(state->kqfd);
- zfree(state->events);
- zfree(state);
-}
-
-static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
- aeApiState *state = eventLoop->apidata;
- struct kevent ke;
-
- if (mask & AE_READABLE) {
- EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
- if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
- }
- if (mask & AE_WRITABLE) {
- EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL);
- if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
- }
- return 0;
-}
-
-static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
- aeApiState *state = eventLoop->apidata;
- struct kevent ke;
-
- if (mask & AE_READABLE) {
- EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
- kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
- }
- if (mask & AE_WRITABLE) {
- EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
- kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
- }
-}
-
-static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
- aeApiState *state = eventLoop->apidata;
- int retval, numevents = 0;
-
- if (tvp != NULL) {
- struct timespec timeout;
- timeout.tv_sec = tvp->tv_sec;
- timeout.tv_nsec = tvp->tv_usec * 1000;
- retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
- &timeout);
- } else {
- retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
- NULL);
- }
-
- if (retval > 0) {
- int j;
-
- numevents = retval;
- for(j = 0; j < numevents; j++) {
- int mask = 0;
- struct kevent *e = state->events+j;
-
- if (e->filter == EVFILT_READ) mask |= AE_READABLE;
- if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
- eventLoop->fired[j].fd = e->ident;
- eventLoop->fired[j].mask = mask;
- }
- }
- return numevents;
-}
-
-static char *aeApiName(void) {
- return "kqueue";
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/ae/ae_select.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae_select.c b/cpp/src/plasma/thirdparty/ae/ae_select.c
deleted file mode 100644
index c039a8e..0000000
--- a/cpp/src/plasma/thirdparty/ae/ae_select.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/* Select()-based ae.c module.
- *
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Redis nor the names of its contributors may be used
- * to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include <sys/select.h>
-#include <string.h>
-
-typedef struct aeApiState {
- fd_set rfds, wfds;
- /* We need to have a copy of the fd sets as it's not safe to reuse
- * FD sets after select(). */
- fd_set _rfds, _wfds;
-} aeApiState;
-
-static int aeApiCreate(aeEventLoop *eventLoop) {
- aeApiState *state = zmalloc(sizeof(aeApiState));
-
- if (!state) return -1;
- FD_ZERO(&state->rfds);
- FD_ZERO(&state->wfds);
- eventLoop->apidata = state;
- return 0;
-}
-
-static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
- /* Just ensure we have enough room in the fd_set type. */
- if (setsize >= FD_SETSIZE) return -1;
- return 0;
-}
-
-static void aeApiFree(aeEventLoop *eventLoop) {
- zfree(eventLoop->apidata);
-}
-
-static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
- aeApiState *state = eventLoop->apidata;
-
- if (mask & AE_READABLE) FD_SET(fd,&state->rfds);
- if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds);
- return 0;
-}
-
-static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
- aeApiState *state = eventLoop->apidata;
-
- if (mask & AE_READABLE) FD_CLR(fd,&state->rfds);
- if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds);
-}
-
-static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
- aeApiState *state = eventLoop->apidata;
- int retval, j, numevents = 0;
-
- memcpy(&state->_rfds,&state->rfds,sizeof(fd_set));
- memcpy(&state->_wfds,&state->wfds,sizeof(fd_set));
-
- retval = select(eventLoop->maxfd+1,
- &state->_rfds,&state->_wfds,NULL,tvp);
- if (retval > 0) {
- for (j = 0; j <= eventLoop->maxfd; j++) {
- int mask = 0;
- aeFileEvent *fe = &eventLoop->events[j];
-
- if (fe->mask == AE_NONE) continue;
- if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds))
- mask |= AE_READABLE;
- if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds))
- mask |= AE_WRITABLE;
- eventLoop->fired[numevents].fd = j;
- eventLoop->fired[numevents].mask = mask;
- numevents++;
- }
- }
- return numevents;
-}
-
-static char *aeApiName(void) {
- return "select";
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/ae/config.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/config.h b/cpp/src/plasma/thirdparty/ae/config.h
deleted file mode 100644
index 4f8e1ea..0000000
--- a/cpp/src/plasma/thirdparty/ae/config.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Redis nor the names of its contributors may be used
- * to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __CONFIG_H
-#define __CONFIG_H
-
-#ifdef __APPLE__
-#include <AvailabilityMacros.h>
-#endif
-
-/* Test for polling API */
-#ifdef __linux__
-#define HAVE_EPOLL 1
-#endif
-
-#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__)
-#define HAVE_KQUEUE 1
-#endif
-
-#ifdef __sun
-#include <sys/feature_tests.h>
-#ifdef _DTRACE_VERSION
-#define HAVE_EVPORT 1
-#endif
-#endif
-
-
-#endif
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/thirdparty/ae/zmalloc.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/zmalloc.h b/cpp/src/plasma/thirdparty/ae/zmalloc.h
deleted file mode 100644
index 6c27dd4..0000000
--- a/cpp/src/plasma/thirdparty/ae/zmalloc.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Redis nor the names of its contributors may be used
- * to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _ZMALLOC_H
-#define _ZMALLOC_H
-
-#ifndef zmalloc
-#define zmalloc malloc
-#endif
-
-#ifndef zfree
-#define zfree free
-#endif
-
-#ifndef zrealloc
-#define zrealloc realloc
-#endif
-
-#endif /* _ZMALLOC_H */
[06/14] arrow git commit: [C++] Remove Plasma source tree for 0.5.0
release pending IP Clearance
Posted by we...@apache.org.
[C++] Remove Plasma source tree for 0.5.0 release pending IP Clearance
Change-Id: I1244ada7297d2f6f567d679452d21a26b91c0360
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/62ef2cd8
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/62ef2cd8
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/62ef2cd8
Branch: refs/heads/master
Commit: 62ef2cd8a39fc93e7fa4bb790d7cd92adb77571f
Parents: db181d1
Author: Wes McKinney <we...@twosigma.com>
Authored: Thu Jul 20 12:58:35 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Thu Jul 20 12:58:35 2017 -0400
----------------------------------------------------------------------
cpp/src/plasma/CMakeLists.txt | 113 -
cpp/src/plasma/client.cc | 557 --
cpp/src/plasma/client.h | 343 --
cpp/src/plasma/common.cc | 83 -
cpp/src/plasma/common.h | 63 -
cpp/src/plasma/events.cc | 81 -
cpp/src/plasma/events.h | 99 -
cpp/src/plasma/eviction_policy.cc | 107 -
cpp/src/plasma/eviction_policy.h | 134 -
cpp/src/plasma/extension.cc | 456 --
cpp/src/plasma/extension.h | 50 -
cpp/src/plasma/fling.cc | 90 -
cpp/src/plasma/fling.h | 52 -
cpp/src/plasma/format/.gitignore | 1 -
cpp/src/plasma/format/common.fbs | 34 -
cpp/src/plasma/format/plasma.fbs | 291 --
cpp/src/plasma/io.cc | 212 -
cpp/src/plasma/io.h | 55 -
cpp/src/plasma/malloc.cc | 178 -
cpp/src/plasma/malloc.h | 26 -
cpp/src/plasma/plasma.cc | 64 -
cpp/src/plasma/plasma.h | 191 -
cpp/src/plasma/protocol.cc | 502 --
cpp/src/plasma/protocol.h | 170 -
cpp/src/plasma/store.cc | 683 ---
cpp/src/plasma/store.h | 169 -
cpp/src/plasma/test/client_tests.cc | 132 -
cpp/src/plasma/test/run_tests.sh | 61 -
cpp/src/plasma/test/run_valgrind.sh | 27 -
cpp/src/plasma/test/serialization_tests.cc | 388 --
cpp/src/plasma/thirdparty/ae/ae.c | 465 --
cpp/src/plasma/thirdparty/ae/ae.h | 123 -
cpp/src/plasma/thirdparty/ae/ae_epoll.c | 135 -
cpp/src/plasma/thirdparty/ae/ae_evport.c | 320 --
cpp/src/plasma/thirdparty/ae/ae_kqueue.c | 138 -
cpp/src/plasma/thirdparty/ae/ae_select.c | 106 -
cpp/src/plasma/thirdparty/ae/config.h | 54 -
cpp/src/plasma/thirdparty/ae/zmalloc.h | 45 -
cpp/src/plasma/thirdparty/dlmalloc.c | 6281 -----------------------
cpp/src/plasma/thirdparty/xxhash.cc | 889 ----
cpp/src/plasma/thirdparty/xxhash.h | 293 --
41 files changed, 14261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
deleted file mode 100644
index 4ff3beb..0000000
--- a/cpp/src/plasma/CMakeLists.txt
+++ /dev/null
@@ -1,113 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-cmake_minimum_required(VERSION 2.8)
-
-project(plasma)
-
-find_package(PythonLibsNew REQUIRED)
-find_package(Threads)
-
-option(PLASMA_PYTHON
- "Build the Plasma Python extensions"
- OFF)
-
-if(APPLE)
- SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
-endif(APPLE)
-
-include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
-include_directories("${FLATBUFFERS_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/" "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" "${CMAKE_CURRENT_LIST_DIR}/../")
-
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L")
-
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-conversion")
-
-# Compile flatbuffers
-
-set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
-set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
-
-set(PLASMA_FBS_OUTPUT_FILES
- "${OUTPUT_DIR}/common_generated.h"
- "${OUTPUT_DIR}/plasma_generated.h")
-
-add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
-
-if(FLATBUFFERS_VENDORED)
- add_dependencies(gen_plasma_fbs flatbuffers_ep)
-endif()
-
-add_custom_command(
- OUTPUT ${PLASMA_FBS_OUTPUT_FILES}
- # The --gen-object-api flag generates a C++ class MessageT for each
- # flatbuffers message Message, which can be used to store deserialized
- # messages in data structures. This is currently used for ObjectInfo for
- # example.
- COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} --gen-object-api
- DEPENDS ${PLASMA_FBS_SRC}
- COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}"
- VERBATIM)
-
-if(UNIX AND NOT APPLE)
- link_libraries(rt)
-endif()
-
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
-
-set_source_files_properties(extension.cc PROPERTIES COMPILE_FLAGS -Wno-strict-aliasing)
-
-set(PLASMA_SRCS
- client.cc
- common.cc
- eviction_policy.cc
- events.cc
- fling.cc
- io.cc
- malloc.cc
- plasma.cc
- protocol.cc
- thirdparty/ae/ae.c
- thirdparty/xxhash.cc)
-
-ADD_ARROW_LIB(plasma
- SOURCES ${PLASMA_SRCS}
- DEPENDENCIES gen_plasma_fbs
- SHARED_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static
- STATIC_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static)
-
-# The optimization flag -O3 is suggested by dlmalloc.c, which is #included in
-# malloc.cc; we set it here regardless of whether we do a debug or release build.
-set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error -O3")
-
-add_executable(plasma_store store.cc)
-target_link_libraries(plasma_store plasma_static)
-
-ADD_ARROW_TEST(test/serialization_tests)
-ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static)
-ADD_ARROW_TEST(test/client_tests)
-ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static)
-
-if(PLASMA_PYTHON)
- add_library(plasma_extension SHARED extension.cc)
-
- if(APPLE)
- target_link_libraries(plasma_extension plasma_static "-undefined dynamic_lookup")
- else(APPLE)
- target_link_libraries(plasma_extension plasma_static -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive)
- endif(APPLE)
-endif()
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/client.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
deleted file mode 100644
index dcb78e7..0000000
--- a/cpp/src/plasma/client.cc
+++ /dev/null
@@ -1,557 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// PLASMA CLIENT: Client library for using the plasma store and manager
-
-#include "plasma/client.h"
-
-#ifdef _WIN32
-#include <Win32_Interop/win32_types.h>
-#endif
-
-#include <assert.h>
-#include <fcntl.h>
-#include <netinet/in.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include <algorithm>
-#include <thread>
-#include <vector>
-
-#include "plasma/common.h"
-#include "plasma/fling.h"
-#include "plasma/io.h"
-#include "plasma/plasma.h"
-#include "plasma/protocol.h"
-
-#define XXH_STATIC_LINKING_ONLY
-#include "thirdparty/xxhash.h"
-
-#define XXH64_DEFAULT_SEED 0
-
-// Number of threads used for memcopy and hash computations.
-constexpr int64_t kThreadPoolSize = 8;
-constexpr int64_t kBytesInMB = 1 << 20;
-static std::vector<std::thread> threadpool_(kThreadPoolSize);
-
-// If the file descriptor fd has been mmapped in this client process before,
-// return the pointer that was returned by mmap, otherwise mmap it and store the
-// pointer in a hash table.
-uint8_t* PlasmaClient::lookup_or_mmap(int fd, int store_fd_val, int64_t map_size) {
- auto entry = mmap_table_.find(store_fd_val);
- if (entry != mmap_table_.end()) {
- close(fd);
- return entry->second.pointer;
- } else {
- uint8_t* result = reinterpret_cast<uint8_t*>(
- mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
- // TODO(pcm): Don't fail here, instead return a Status.
- if (result == MAP_FAILED) { ARROW_LOG(FATAL) << "mmap failed"; }
- close(fd);
- ClientMmapTableEntry& entry = mmap_table_[store_fd_val];
- entry.pointer = result;
- entry.length = map_size;
- entry.count = 0;
- return result;
- }
-}
-
-// Get a pointer to a file that we know has been memory mapped in this client
-// process before.
-uint8_t* PlasmaClient::lookup_mmapped_file(int store_fd_val) {
- auto entry = mmap_table_.find(store_fd_val);
- ARROW_CHECK(entry != mmap_table_.end());
- return entry->second.pointer;
-}
-
-void PlasmaClient::increment_object_count(
- const ObjectID& object_id, PlasmaObject* object, bool is_sealed) {
- // Increment the count of the object to track the fact that it is being used.
- // The corresponding decrement should happen in PlasmaClient::Release.
- auto elem = objects_in_use_.find(object_id);
- ObjectInUseEntry* object_entry;
- if (elem == objects_in_use_.end()) {
- // Add this object ID to the hash table of object IDs in use. The
- // corresponding call to free happens in PlasmaClient::Release.
- objects_in_use_[object_id] =
- std::unique_ptr<ObjectInUseEntry>(new ObjectInUseEntry());
- objects_in_use_[object_id]->object = *object;
- objects_in_use_[object_id]->count = 0;
- objects_in_use_[object_id]->is_sealed = is_sealed;
- object_entry = objects_in_use_[object_id].get();
- // Increment the count of the number of objects in the memory-mapped file
- // that are being used. The corresponding decrement should happen in
- // PlasmaClient::Release.
- auto entry = mmap_table_.find(object->handle.store_fd);
- ARROW_CHECK(entry != mmap_table_.end());
- ARROW_CHECK(entry->second.count >= 0);
- // Update the in_use_object_bytes_.
- in_use_object_bytes_ +=
- (object_entry->object.data_size + object_entry->object.metadata_size);
- entry->second.count += 1;
- } else {
- object_entry = elem->second.get();
- ARROW_CHECK(object_entry->count > 0);
- }
- // Increment the count of the number of instances of this object that are
- // being used by this client. The corresponding decrement should happen in
- // PlasmaClient::Release.
- object_entry->count += 1;
-}
-
-Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
- uint8_t* metadata, int64_t metadata_size, uint8_t** data) {
- ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
- << data_size << " and metadata size " << metadata_size;
- RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
- std::vector<uint8_t> buffer;
- RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaCreateReply, &buffer));
- ObjectID id;
- PlasmaObject object;
- RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object));
- // If the CreateReply included an error, then the store will not send a file
- // descriptor.
- int fd = recv_fd(store_conn_);
- ARROW_CHECK(fd >= 0) << "recv not successful";
- ARROW_CHECK(object.data_size == data_size);
- ARROW_CHECK(object.metadata_size == metadata_size);
- // The metadata should come right after the data.
- ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
- *data = lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) +
- object.data_offset;
- // If plasma_create is being called from a transfer, then we will not copy the
- // metadata here. The metadata will be written along with the data streamed
- // from the transfer.
- if (metadata != NULL) {
- // Copy the metadata to the buffer.
- memcpy(*data + object.data_size, metadata, metadata_size);
- }
- // Increment the count of the number of instances of this object that this
- // client is using. A call to PlasmaClient::Release is required to decrement
- // this
- // count. Cache the reference to the object.
- increment_object_count(object_id, &object, false);
- // We increment the count a second time (and the corresponding decrement will
- // happen in a PlasmaClient::Release call in plasma_seal) so even if the
- // buffer
- // returned by PlasmaClient::Dreate goes out of scope, the object does not get
- // released before the call to PlasmaClient::Seal happens.
- increment_object_count(object_id, &object, false);
- return Status::OK();
-}
-
-Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
- int64_t timeout_ms, ObjectBuffer* object_buffers) {
- // Fill out the info for the objects that are already in use locally.
- bool all_present = true;
- for (int i = 0; i < num_objects; ++i) {
- auto object_entry = objects_in_use_.find(object_ids[i]);
- if (object_entry == objects_in_use_.end()) {
- // This object is not currently in use by this client, so we need to send
- // a request to the store.
- all_present = false;
- // Make a note to ourselves that the object is not present.
- object_buffers[i].data_size = -1;
- } else {
- // NOTE: If the object is still unsealed, we will deadlock, since we must
- // have been the one who created it.
- ARROW_CHECK(object_entry->second->is_sealed)
- << "Plasma client called get on an unsealed object that it created";
- PlasmaObject* object = &object_entry->second->object;
- object_buffers[i].data = lookup_mmapped_file(object->handle.store_fd);
- object_buffers[i].data = object_buffers[i].data + object->data_offset;
- object_buffers[i].data_size = object->data_size;
- object_buffers[i].metadata = object_buffers[i].data + object->data_size;
- object_buffers[i].metadata_size = object->metadata_size;
- // Increment the count of the number of instances of this object that this
- // client is using. A call to PlasmaClient::Release is required to
- // decrement this
- // count. Cache the reference to the object.
- increment_object_count(object_ids[i], object, true);
- }
- }
-
- if (all_present) { return Status::OK(); }
-
- // If we get here, then the objects aren't all currently in use by this
- // client, so we need to send a request to the plasma store.
- RETURN_NOT_OK(SendGetRequest(store_conn_, object_ids, num_objects, timeout_ms));
- std::vector<uint8_t> buffer;
- RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaGetReply, &buffer));
- std::vector<ObjectID> received_object_ids(num_objects);
- std::vector<PlasmaObject> object_data(num_objects);
- PlasmaObject* object;
- RETURN_NOT_OK(ReadGetReply(
- buffer.data(), received_object_ids.data(), object_data.data(), num_objects));
-
- for (int i = 0; i < num_objects; ++i) {
- DCHECK(received_object_ids[i] == object_ids[i]);
- object = &object_data[i];
- if (object_buffers[i].data_size != -1) {
- // If the object was already in use by the client, then the store should
- // have returned it.
- DCHECK_NE(object->data_size, -1);
- // We won't use this file descriptor, but the store sent us one, so we
- // need to receive it and then close it right away so we don't leak file
- // descriptors.
- int fd = recv_fd(store_conn_);
- close(fd);
- ARROW_CHECK(fd >= 0);
- // We've already filled out the information for this object, so we can
- // just continue.
- continue;
- }
- // If we are here, the object was not currently in use, so we need to
- // process the reply from the object store.
- if (object->data_size != -1) {
- // The object was retrieved. The user will be responsible for releasing
- // this object.
- int fd = recv_fd(store_conn_);
- ARROW_CHECK(fd >= 0);
- object_buffers[i].data =
- lookup_or_mmap(fd, object->handle.store_fd, object->handle.mmap_size);
- // Finish filling out the return values.
- object_buffers[i].data = object_buffers[i].data + object->data_offset;
- object_buffers[i].data_size = object->data_size;
- object_buffers[i].metadata = object_buffers[i].data + object->data_size;
- object_buffers[i].metadata_size = object->metadata_size;
- // Increment the count of the number of instances of this object that this
- // client is using. A call to PlasmaClient::Release is required to
- // decrement this
- // count. Cache the reference to the object.
- increment_object_count(received_object_ids[i], object, true);
- } else {
- // The object was not retrieved. Make sure we already put a -1 here to
- // indicate that the object was not retrieved. The caller is not
- // responsible for releasing this object.
- DCHECK_EQ(object_buffers[i].data_size, -1);
- object_buffers[i].data_size = -1;
- }
- }
- return Status::OK();
-}
-
-/// This is a helper method for implementing plasma_release. We maintain a
-/// buffer
-/// of release calls and only perform them once the buffer becomes full (as
-/// judged by the aggregate sizes of the objects). There may be multiple release
-/// calls for the same object ID in the buffer. In this case, the first release
-/// calls will not do anything. The client will only send a message to the store
-/// releasing the object when the client is truly done with the object.
-///
-/// @param conn The plasma connection.
-/// @param object_id The object ID to attempt to release.
-Status PlasmaClient::PerformRelease(const ObjectID& object_id) {
- // Decrement the count of the number of instances of this object that are
- // being used by this client. The corresponding increment should have happened
- // in PlasmaClient::Get.
- auto object_entry = objects_in_use_.find(object_id);
- ARROW_CHECK(object_entry != objects_in_use_.end());
- object_entry->second->count -= 1;
- ARROW_CHECK(object_entry->second->count >= 0);
- // Check if the client is no longer using this object.
- if (object_entry->second->count == 0) {
- // Decrement the count of the number of objects in this memory-mapped file
- // that the client is using. The corresponding increment should have
- // happened in plasma_get.
- int fd = object_entry->second->object.handle.store_fd;
- auto entry = mmap_table_.find(fd);
- ARROW_CHECK(entry != mmap_table_.end());
- entry->second.count -= 1;
- ARROW_CHECK(entry->second.count >= 0);
- // If none are being used then unmap the file.
- if (entry->second.count == 0) {
- munmap(entry->second.pointer, entry->second.length);
- // Remove the corresponding entry from the hash table.
- mmap_table_.erase(fd);
- }
- // Tell the store that the client no longer needs the object.
- RETURN_NOT_OK(SendReleaseRequest(store_conn_, object_id));
- // Update the in_use_object_bytes_.
- in_use_object_bytes_ -= (object_entry->second->object.data_size +
- object_entry->second->object.metadata_size);
- DCHECK_GE(in_use_object_bytes_, 0);
- // Remove the entry from the hash table of objects currently in use.
- objects_in_use_.erase(object_id);
- }
- return Status::OK();
-}
-
-Status PlasmaClient::Release(const ObjectID& object_id) {
- // Add the new object to the release history.
- release_history_.push_front(object_id);
- // If there are too many bytes in use by the client or if there are too many
- // pending release calls, and there are at least some pending release calls in
- // the release_history list, then release some objects.
- while ((in_use_object_bytes_ > std::min(kL3CacheSizeBytes, store_capacity_ / 100) ||
- release_history_.size() > config_.release_delay) &&
- release_history_.size() > 0) {
- // Perform a release for the object ID for the first pending release.
- RETURN_NOT_OK(PerformRelease(release_history_.back()));
- // Remove the last entry from the release history.
- release_history_.pop_back();
- }
- return Status::OK();
-}
-
-// This method is used to query whether the plasma store contains an object.
-Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) {
- // Check if we already have a reference to the object.
- if (objects_in_use_.count(object_id) > 0) {
- *has_object = 1;
- } else {
- // If we don't already have a reference to the object, check with the store
- // to see if we have the object.
- RETURN_NOT_OK(SendContainsRequest(store_conn_, object_id));
- std::vector<uint8_t> buffer;
- RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaContainsReply, &buffer));
- ObjectID object_id2;
- RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object));
- }
- return Status::OK();
-}
-
-static void ComputeBlockHash(const unsigned char* data, int64_t nbytes, uint64_t* hash) {
- XXH64_state_t hash_state;
- XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
- XXH64_update(&hash_state, data, nbytes);
- *hash = XXH64_digest(&hash_state);
-}
-
-static inline bool compute_object_hash_parallel(
- XXH64_state_t* hash_state, const unsigned char* data, int64_t nbytes) {
- // Note that this function will likely be faster if the address of data is
- // aligned on a 64-byte boundary.
- const int num_threads = kThreadPoolSize;
- uint64_t threadhash[num_threads + 1];
- const uint64_t data_address = reinterpret_cast<uint64_t>(data);
- const uint64_t num_blocks = nbytes / BLOCK_SIZE;
- const uint64_t chunk_size = (num_blocks / num_threads) * BLOCK_SIZE;
- const uint64_t right_address = data_address + chunk_size * num_threads;
- const uint64_t suffix = (data_address + nbytes) - right_address;
- // Now the data layout is | k * num_threads * block_size | suffix | ==
- // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size.
- // Each thread gets a "chunk" of k blocks, except the suffix thread.
-
- for (int i = 0; i < num_threads; i++) {
- threadpool_[i] = std::thread(ComputeBlockHash,
- reinterpret_cast<uint8_t*>(data_address) + i * chunk_size, chunk_size,
- &threadhash[i]);
- }
- ComputeBlockHash(
- reinterpret_cast<uint8_t*>(right_address), suffix, &threadhash[num_threads]);
-
- // Join the threads.
- for (auto& t : threadpool_) {
- if (t.joinable()) { t.join(); }
- }
-
- XXH64_update(hash_state, (unsigned char*)threadhash, sizeof(threadhash));
- return true;
-}
-
-static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
- XXH64_state_t hash_state;
- XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
- if (obj_buffer.data_size >= kBytesInMB) {
- compute_object_hash_parallel(
- &hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
- } else {
- XXH64_update(&hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
- }
- XXH64_update(
- &hash_state, (unsigned char*)obj_buffer.metadata, obj_buffer.metadata_size);
- return XXH64_digest(&hash_state);
-}
-
-bool plasma_compute_object_hash(
- PlasmaClient* conn, ObjectID object_id, unsigned char* digest) {
- // Get the plasma object data. We pass in a timeout of 0 to indicate that
- // the operation should timeout immediately.
- ObjectBuffer object_buffer;
- ARROW_CHECK_OK(conn->Get(&object_id, 1, 0, &object_buffer));
- // If the object was not retrieved, return false.
- if (object_buffer.data_size == -1) { return false; }
- // Compute the hash.
- uint64_t hash = compute_object_hash(object_buffer);
- memcpy(digest, &hash, sizeof(hash));
- // Release the plasma object.
- ARROW_CHECK_OK(conn->Release(object_id));
- return true;
-}
-
-Status PlasmaClient::Seal(const ObjectID& object_id) {
- // Make sure this client has a reference to the object before sending the
- // request to Plasma.
- auto object_entry = objects_in_use_.find(object_id);
- ARROW_CHECK(object_entry != objects_in_use_.end())
- << "Plasma client called seal an object without a reference to it";
- ARROW_CHECK(!object_entry->second->is_sealed)
- << "Plasma client called seal an already sealed object";
- object_entry->second->is_sealed = true;
- /// Send the seal request to Plasma.
- static unsigned char digest[kDigestSize];
- ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0]));
- RETURN_NOT_OK(SendSealRequest(store_conn_, object_id, &digest[0]));
- // We call PlasmaClient::Release to decrement the number of instances of this
- // object
- // that are currently being used by this client. The corresponding increment
- // happened in plasma_create and was used to ensure that the object was not
- // released before the call to PlasmaClient::Seal.
- return Release(object_id);
-}
-
-Status PlasmaClient::Delete(const ObjectID& object_id) {
- // TODO(rkn): In the future, we can use this method to give hints to the
- // eviction policy about when an object will no longer be needed.
- return Status::NotImplemented("PlasmaClient::Delete is not implemented.");
-}
-
-Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) {
- // Send a request to the store to evict objects.
- RETURN_NOT_OK(SendEvictRequest(store_conn_, num_bytes));
- // Wait for a response with the number of bytes actually evicted.
- std::vector<uint8_t> buffer;
- int64_t type;
- RETURN_NOT_OK(ReadMessage(store_conn_, &type, &buffer));
- return ReadEvictReply(buffer.data(), num_bytes_evicted);
-}
-
-Status PlasmaClient::Subscribe(int* fd) {
- int sock[2];
- // Create a non-blocking socket pair. This will only be used to send
- // notifications from the Plasma store to the client.
- socketpair(AF_UNIX, SOCK_STREAM, 0, sock);
- // Make the socket non-blocking.
- int flags = fcntl(sock[1], F_GETFL, 0);
- ARROW_CHECK(fcntl(sock[1], F_SETFL, flags | O_NONBLOCK) == 0);
- // Tell the Plasma store about the subscription.
- RETURN_NOT_OK(SendSubscribeRequest(store_conn_));
- // Send the file descriptor that the Plasma store should use to push
- // notifications about sealed objects to this client.
- ARROW_CHECK(send_fd(store_conn_, sock[1]) >= 0);
- close(sock[1]);
- // Return the file descriptor that the client should use to read notifications
- // about sealed objects.
- *fd = sock[0];
- return Status::OK();
-}
-
-Status PlasmaClient::Connect(const std::string& store_socket_name,
- const std::string& manager_socket_name, int release_delay) {
- store_conn_ = connect_ipc_sock_retry(store_socket_name, -1, -1);
- if (manager_socket_name != "") {
- manager_conn_ = connect_ipc_sock_retry(manager_socket_name, -1, -1);
- } else {
- manager_conn_ = -1;
- }
- config_.release_delay = release_delay;
- in_use_object_bytes_ = 0;
- // Send a ConnectRequest to the store to get its memory capacity.
- RETURN_NOT_OK(SendConnectRequest(store_conn_));
- std::vector<uint8_t> buffer;
- RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaConnectReply, &buffer));
- RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity_));
- return Status::OK();
-}
-
-Status PlasmaClient::Disconnect() {
- // NOTE: We purposefully do not finish sending release calls for objects in
- // use, so that we don't duplicate PlasmaClient::Release calls (when handling
- // a SIGTERM, for example).
-
- // Close the connections to Plasma. The Plasma store will release the objects
- // that were in use by us when handling the SIGPIPE.
- close(store_conn_);
- if (manager_conn_ >= 0) { close(manager_conn_); }
- return Status::OK();
-}
-
-#define h_addr h_addr_list[0]
-
-Status PlasmaClient::Transfer(const char* address, int port, const ObjectID& object_id) {
- return SendDataRequest(manager_conn_, object_id, address, port);
-}
-
-Status PlasmaClient::Fetch(int num_object_ids, const ObjectID* object_ids) {
- ARROW_CHECK(manager_conn_ >= 0);
- return SendFetchRequest(manager_conn_, object_ids, num_object_ids);
-}
-
-int PlasmaClient::get_manager_fd() {
- return manager_conn_;
-}
-
-Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
- ARROW_CHECK(manager_conn_ >= 0);
-
- RETURN_NOT_OK(SendStatusRequest(manager_conn_, &object_id, 1));
- std::vector<uint8_t> buffer;
- RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaStatusReply, &buffer));
- ObjectID id;
- RETURN_NOT_OK(ReadStatusReply(buffer.data(), &id, object_status, 1));
- ARROW_CHECK(object_id == id);
- return Status::OK();
-}
-
-Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_requests,
- int num_ready_objects, int64_t timeout_ms, int* num_objects_ready) {
- ARROW_CHECK(manager_conn_ >= 0);
- ARROW_CHECK(num_object_requests > 0);
- ARROW_CHECK(num_ready_objects > 0);
- ARROW_CHECK(num_ready_objects <= num_object_requests);
-
- for (int i = 0; i < num_object_requests; ++i) {
- ARROW_CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL ||
- object_requests[i].type == PLASMA_QUERY_ANYWHERE);
- }
-
- RETURN_NOT_OK(SendWaitRequest(manager_conn_, object_requests, num_object_requests,
- num_ready_objects, timeout_ms));
- std::vector<uint8_t> buffer;
- RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaWaitReply, &buffer));
- RETURN_NOT_OK(ReadWaitReply(buffer.data(), object_requests, &num_ready_objects));
-
- *num_objects_ready = 0;
- for (int i = 0; i < num_object_requests; ++i) {
- int type = object_requests[i].type;
- int status = object_requests[i].status;
- switch (type) {
- case PLASMA_QUERY_LOCAL:
- if (status == ObjectStatus_Local) { *num_objects_ready += 1; }
- break;
- case PLASMA_QUERY_ANYWHERE:
- if (status == ObjectStatus_Local || status == ObjectStatus_Remote) {
- *num_objects_ready += 1;
- } else {
- ARROW_CHECK(status == ObjectStatus_Nonexistent);
- }
- break;
- default:
- ARROW_LOG(FATAL) << "This code should be unreachable.";
- }
- }
- return Status::OK();
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/client.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
deleted file mode 100644
index fb3a161..0000000
--- a/cpp/src/plasma/client.h
+++ /dev/null
@@ -1,343 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_CLIENT_H
-#define PLASMA_CLIENT_H
-
-#include <stdbool.h>
-#include <time.h>
-
-#include <deque>
-#include <string>
-
-#include "plasma/plasma.h"
-
-using arrow::Status;
-
-#define PLASMA_DEFAULT_RELEASE_DELAY 64
-
-// Use 100MB as an overestimate of the L3 cache size.
-constexpr int64_t kL3CacheSizeBytes = 100000000;
-
-/// Object buffer data structure.
-struct ObjectBuffer {
- /// The size in bytes of the data object.
- int64_t data_size;
- /// The address of the data object.
- uint8_t* data;
- /// The metadata size in bytes.
- int64_t metadata_size;
- /// The address of the metadata.
- uint8_t* metadata;
-};
-
-/// Configuration options for the plasma client.
-struct PlasmaClientConfig {
- /// Number of release calls we wait until the object is actually released.
- /// This allows us to avoid invalidating the cpu cache on workers if objects
- /// are reused accross tasks.
- size_t release_delay;
-};
-
-struct ClientMmapTableEntry {
- /// The result of mmap for this file descriptor.
- uint8_t* pointer;
- /// The length of the memory-mapped file.
- size_t length;
- /// The number of objects in this memory-mapped file that are currently being
- /// used by the client. When this count reaches zeros, we unmap the file.
- int count;
-};
-
-struct ObjectInUseEntry {
- /// A count of the number of times this client has called PlasmaClient::Create
- /// or
- /// PlasmaClient::Get on this object ID minus the number of calls to
- /// PlasmaClient::Release.
- /// When this count reaches zero, we remove the entry from the ObjectsInUse
- /// and decrement a count in the relevant ClientMmapTableEntry.
- int count;
- /// Cached information to read the object.
- PlasmaObject object;
- /// A flag representing whether the object has been sealed.
- bool is_sealed;
-};
-
-class PlasmaClient {
- public:
- /// Connect to the local plasma store and plasma manager. Return
- /// the resulting connection.
- ///
- /// @param store_socket_name The name of the UNIX domain socket to use to
- /// connect to the Plasma store.
- /// @param manager_socket_name The name of the UNIX domain socket to use to
- /// connect to the local Plasma manager. If this is "", then this
- /// function will not connect to a manager.
- /// @param release_delay Number of released objects that are kept around
- /// and not evicted to avoid too many munmaps.
- /// @return The return status.
- Status Connect(const std::string& store_socket_name,
- const std::string& manager_socket_name, int release_delay);
-
- /// Create an object in the Plasma Store. Any metadata for this object must be
- /// be passed in when the object is created.
- ///
- /// @param object_id The ID to use for the newly created object.
- /// @param data_size The size in bytes of the space to be allocated for this
- /// object's
- /// data (this does not include space used for metadata).
- /// @param metadata The object's metadata. If there is no metadata, this
- /// pointer
- /// should be NULL.
- /// @param metadata_size The size in bytes of the metadata. If there is no
- /// metadata, this should be 0.
- /// @param data The address of the newly created object will be written here.
- /// @return The return status.
- Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
- int64_t metadata_size, uint8_t** data);
-
- /// Get some objects from the Plasma Store. This function will block until the
- /// objects have all been created and sealed in the Plasma Store or the
- /// timeout
- /// expires. The caller is responsible for releasing any retrieved objects,
- /// but
- /// the caller should not release objects that were not retrieved.
- ///
- /// @param object_ids The IDs of the objects to get.
- /// @param num_object_ids The number of object IDs to get.
- /// @param timeout_ms The amount of time in milliseconds to wait before this
- /// request times out. If this value is -1, then no timeout is set.
- /// @param object_buffers An array where the results will be stored. If the
- /// data
- /// size field is -1, then the object was not retrieved.
- /// @return The return status.
- Status Get(const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms,
- ObjectBuffer* object_buffers);
-
- /// Tell Plasma that the client no longer needs the object. This should be
- /// called
- /// after Get when the client is done with the object. After this call,
- /// the address returned by Get is no longer valid. This should be called
- /// once for each call to Get (with the same object ID).
- ///
- /// @param object_id The ID of the object that is no longer needed.
- /// @return The return status.
- Status Release(const ObjectID& object_id);
-
- /// Check if the object store contains a particular object and the object has
- /// been sealed. The result will be stored in has_object.
- ///
- /// @todo: We may want to indicate if the object has been created but not
- /// sealed.
- ///
- /// @param object_id The ID of the object whose presence we are checking.
- /// @param has_object The function will write true at this address if
- /// the object is present and false if it is not present.
- /// @return The return status.
- Status Contains(const ObjectID& object_id, bool* has_object);
-
- /// Seal an object in the object store. The object will be immutable after
- /// this
- /// call.
- ///
- /// @param object_id The ID of the object to seal.
- /// @return The return status.
- Status Seal(const ObjectID& object_id);
-
- /// Delete an object from the object store. This currently assumes that the
- /// object is present and has been sealed.
- ///
- /// @todo We may want to allow the deletion of objects that are not present or
- /// haven't been sealed.
- ///
- /// @param object_id The ID of the object to delete.
- /// @return The return status.
- Status Delete(const ObjectID& object_id);
-
- /// Delete objects until we have freed up num_bytes bytes or there are no more
- /// released objects that can be deleted.
- ///
- /// @param num_bytes The number of bytes to try to free up.
- /// @param num_bytes_evicted Out parameter for total number of bytes of space
- /// retrieved.
- /// @return The return status.
- Status Evict(int64_t num_bytes, int64_t& num_bytes_evicted);
-
- /// Subscribe to notifications when objects are sealed in the object store.
- /// Whenever an object is sealed, a message will be written to the client
- /// socket
- /// that is returned by this method.
- ///
- /// @param fd Out parameter for the file descriptor the client should use to
- /// read notifications
- /// from the object store about sealed objects.
- /// @return The return status.
- Status Subscribe(int* fd);
-
- /// Disconnect from the local plasma instance, including the local store and
- /// manager.
- ///
- /// @return The return status.
- Status Disconnect();
-
- /// Attempt to initiate the transfer of some objects from remote Plasma
- /// Stores.
- /// This method does not guarantee that the fetched objects will arrive
- /// locally.
- ///
- /// For an object that is available in the local Plasma Store, this method
- /// will
- /// not do anything. For an object that is not available locally, it will
- /// check
- /// if the object are already being fetched. If so, it will not do anything.
- /// If
- /// not, it will query the object table for a list of Plasma Managers that
- /// have
- /// the object. The object table will return a non-empty list, and this Plasma
- /// Manager will attempt to initiate transfers from one of those Plasma
- /// Managers.
- ///
- /// This function is non-blocking.
- ///
- /// This method is idempotent in the sense that it is ok to call it multiple
- /// times.
- ///
- /// @param num_object_ids The number of object IDs fetch is being called on.
- /// @param object_ids The IDs of the objects that fetch is being called on.
- /// @return The return status.
- Status Fetch(int num_object_ids, const ObjectID* object_ids);
-
- /// Wait for (1) a specified number of objects to be available (sealed) in the
- /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to
- /// expire. This is a blocking call.
- ///
- /// @param num_object_requests Size of the object_requests array.
- /// @param object_requests Object event array. Each element contains a request
- /// for a particular object_id. The type of request is specified in the
- /// "type" field.
- /// - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes
- /// available in the local Plasma Store. In this case, this function
- /// sets the "status" field to ObjectStatus_Local. Note, if the
- /// status
- /// is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent,
- /// but it may exist elsewhere in the system.
- /// - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id
- /// becomes
- /// available either at the local Plasma Store or on a remote Plasma
- /// Store. In this case, the functions sets the "status" field to
- /// ObjectStatus_Local or ObjectStatus_Remote.
- /// @param num_ready_objects The number of requests in object_requests array
- /// that
- /// must be satisfied before the function returns, unless it timeouts.
- /// The num_ready_objects should be no larger than num_object_requests.
- /// @param timeout_ms Timeout value in milliseconds. If this timeout expires
- /// before min_num_ready_objects of requests are satisfied, the
- /// function
- /// returns.
- /// @param num_objects_ready Out parameter for number of satisfied requests in
- /// the object_requests list. If the returned number is less than
- /// min_num_ready_objects this means that timeout expired.
- /// @return The return status.
- Status Wait(int64_t num_object_requests, ObjectRequest* object_requests,
- int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
-
- /// Transfer local object to a different plasma manager.
- ///
- /// @param conn The object containing the connection state.
- /// @param addr IP address of the plasma manager we are transfering to.
- /// @param port Port of the plasma manager we are transfering to.
- /// @object_id ObjectID of the object we are transfering.
- /// @return The return status.
- Status Transfer(const char* addr, int port, const ObjectID& object_id);
-
- /// Return the status of a given object. This method may query the object
- /// table.
- ///
- /// @param conn The object containing the connection state.
- /// @param object_id The ID of the object whose status we query.
- /// @param object_status Out parameter for object status. Can take the
- /// following values.
- /// - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma
- /// Store.
- /// has been already scheduled by the Plasma Manager.
- /// - PLASMA_CLIENT_TRANSFER, if the object is either currently being
- /// transferred or just scheduled.
- /// - PLASMA_CLIENT_REMOTE, if the object is stored at a remote
- /// Plasma Store.
- /// - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the
- /// system.
- /// @return The return status.
- Status Info(const ObjectID& object_id, int* object_status);
-
- /// Get the file descriptor for the socket connection to the plasma manager.
- ///
- /// @param conn The plasma connection.
- /// @return The file descriptor for the manager connection. If there is no
- /// connection to the manager, this is -1.
- int get_manager_fd();
-
- private:
- Status PerformRelease(const ObjectID& object_id);
-
- uint8_t* lookup_or_mmap(int fd, int store_fd_val, int64_t map_size);
-
- uint8_t* lookup_mmapped_file(int store_fd_val);
-
- void increment_object_count(
- const ObjectID& object_id, PlasmaObject* object, bool is_sealed);
-
- /// File descriptor of the Unix domain socket that connects to the store.
- int store_conn_;
- /// File descriptor of the Unix domain socket that connects to the manager.
- int manager_conn_;
- /// Table of dlmalloc buffer files that have been memory mapped so far. This
- /// is a hash table mapping a file descriptor to a struct containing the
- /// address of the corresponding memory-mapped file.
- std::unordered_map<int, ClientMmapTableEntry> mmap_table_;
- /// A hash table of the object IDs that are currently being used by this
- /// client.
- std::unordered_map<ObjectID, std::unique_ptr<ObjectInUseEntry>, UniqueIDHasher>
- objects_in_use_;
- /// Object IDs of the last few release calls. This is a deque and
- /// is used to delay releasing objects to see if they can be reused by
- /// subsequent tasks so we do not unneccessarily invalidate cpu caches.
- /// TODO(pcm): replace this with a proper lru cache using the size of the L3
- /// cache.
- std::deque<ObjectID> release_history_;
- /// The number of bytes in the combined objects that are held in the release
- /// history doubly-linked list. If this is too large then the client starts
- /// releasing objects.
- int64_t in_use_object_bytes_;
- /// Configuration options for the plasma client.
- PlasmaClientConfig config_;
- /// The amount of memory available to the Plasma store. The client needs this
- /// information to make sure that it does not delay in releasing so much
- /// memory that the store is unable to evict enough objects to free up space.
- int64_t store_capacity_;
-};
-
-/// Compute the hash of an object in the object store.
-///
-/// @param conn The object containing the connection state.
-/// @param object_id The ID of the object we want to hash.
-/// @param digest A pointer at which to return the hash digest of the object.
-/// The pointer must have at least DIGEST_SIZE bytes allocated.
-/// @return A boolean representing whether the hash operation succeeded.
-bool plasma_compute_object_hash(
- PlasmaClient* conn, ObjectID object_id, unsigned char* digest);
-
-#endif // PLASMA_CLIENT_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/common.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/common.cc b/cpp/src/plasma/common.cc
deleted file mode 100644
index a09a963..0000000
--- a/cpp/src/plasma/common.cc
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/common.h"
-
-#include <random>
-
-#include "format/plasma_generated.h"
-
-using arrow::Status;
-
-UniqueID UniqueID::from_random() {
- UniqueID id;
- uint8_t* data = id.mutable_data();
- std::random_device engine;
- for (int i = 0; i < kUniqueIDSize; i++) {
- data[i] = static_cast<uint8_t>(engine());
- }
- return id;
-}
-
-UniqueID UniqueID::from_binary(const std::string& binary) {
- UniqueID id;
- std::memcpy(&id, binary.data(), sizeof(id));
- return id;
-}
-
-const uint8_t* UniqueID::data() const {
- return id_;
-}
-
-uint8_t* UniqueID::mutable_data() {
- return id_;
-}
-
-std::string UniqueID::binary() const {
- return std::string(reinterpret_cast<const char*>(id_), kUniqueIDSize);
-}
-
-std::string UniqueID::hex() const {
- constexpr char hex[] = "0123456789abcdef";
- std::string result;
- for (int i = 0; i < kUniqueIDSize; i++) {
- unsigned int val = id_[i];
- result.push_back(hex[val >> 4]);
- result.push_back(hex[val & 0xf]);
- }
- return result;
-}
-
-bool UniqueID::operator==(const UniqueID& rhs) const {
- return std::memcmp(data(), rhs.data(), kUniqueIDSize) == 0;
-}
-
-Status plasma_error_status(int plasma_error) {
- switch (plasma_error) {
- case PlasmaError_OK:
- return Status::OK();
- case PlasmaError_ObjectExists:
- return Status::PlasmaObjectExists("object already exists in the plasma store");
- case PlasmaError_ObjectNonexistent:
- return Status::PlasmaObjectNonexistent("object does not exist in the plasma store");
- case PlasmaError_OutOfMemory:
- return Status::PlasmaStoreFull("object does not fit in the plasma store");
- default:
- ARROW_LOG(FATAL) << "unknown plasma error code " << plasma_error;
- }
- return Status::OK();
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/common.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
deleted file mode 100644
index 85dc74b..0000000
--- a/cpp/src/plasma/common.h
+++ /dev/null
@@ -1,63 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_COMMON_H
-#define PLASMA_COMMON_H
-
-#include <cstring>
-#include <string>
-// TODO(pcm): Convert getopt and sscanf in the store to use more idiomatic C++
-// and get rid of the next three lines:
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include "arrow/status.h"
-#include "arrow/util/logging.h"
-
-constexpr int64_t kUniqueIDSize = 20;
-
-class UniqueID {
- public:
- static UniqueID from_random();
- static UniqueID from_binary(const std::string& binary);
- bool operator==(const UniqueID& rhs) const;
- const uint8_t* data() const;
- uint8_t* mutable_data();
- std::string binary() const;
- std::string hex() const;
-
- private:
- uint8_t id_[kUniqueIDSize];
-};
-
-static_assert(std::is_pod<UniqueID>::value, "UniqueID must be plain old data");
-
-struct UniqueIDHasher {
- // ObjectID hashing function.
- size_t operator()(const UniqueID& id) const {
- size_t result;
- std::memcpy(&result, id.data(), sizeof(size_t));
- return result;
- }
-};
-
-typedef UniqueID ObjectID;
-
-arrow::Status plasma_error_status(int plasma_error);
-
-#endif // PLASMA_COMMON_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/events.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
deleted file mode 100644
index a9f7356..0000000
--- a/cpp/src/plasma/events.cc
+++ /dev/null
@@ -1,81 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/events.h"
-
-#include <errno.h>
-
-void EventLoop::file_event_callback(
- aeEventLoop* loop, int fd, void* context, int events) {
- FileCallback* callback = reinterpret_cast<FileCallback*>(context);
- (*callback)(events);
-}
-
-int EventLoop::timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context) {
- TimerCallback* callback = reinterpret_cast<TimerCallback*>(context);
- return (*callback)(timer_id);
-}
-
-constexpr int kInitialEventLoopSize = 1024;
-
-EventLoop::EventLoop() {
- loop_ = aeCreateEventLoop(kInitialEventLoopSize);
-}
-
-bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback) {
- if (file_callbacks_.find(fd) != file_callbacks_.end()) { return false; }
- auto data = std::unique_ptr<FileCallback>(new FileCallback(callback));
- void* context = reinterpret_cast<void*>(data.get());
- // Try to add the file descriptor.
- int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
- // If it cannot be added, increase the size of the event loop.
- if (err == AE_ERR && errno == ERANGE) {
- err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2);
- if (err != AE_OK) { return false; }
- err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
- }
- // In any case, test if there were errors.
- if (err == AE_OK) {
- file_callbacks_.emplace(fd, std::move(data));
- return true;
- }
- return false;
-}
-
-void EventLoop::remove_file_event(int fd) {
- aeDeleteFileEvent(loop_, fd, AE_READABLE | AE_WRITABLE);
- file_callbacks_.erase(fd);
-}
-
-void EventLoop::run() {
- aeMain(loop_);
-}
-
-int64_t EventLoop::add_timer(int64_t timeout, const TimerCallback& callback) {
- auto data = std::unique_ptr<TimerCallback>(new TimerCallback(callback));
- void* context = reinterpret_cast<void*>(data.get());
- int64_t timer_id =
- aeCreateTimeEvent(loop_, timeout, EventLoop::timer_event_callback, context, NULL);
- timer_callbacks_.emplace(timer_id, std::move(data));
- return timer_id;
-}
-
-int EventLoop::remove_timer(int64_t timer_id) {
- int err = aeDeleteTimeEvent(loop_, timer_id);
- timer_callbacks_.erase(timer_id);
- return err;
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/events.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
deleted file mode 100644
index bd93d6b..0000000
--- a/cpp/src/plasma/events.h
+++ /dev/null
@@ -1,99 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_EVENTS
-#define PLASMA_EVENTS
-
-#include <functional>
-#include <memory>
-#include <unordered_map>
-
-extern "C" {
-#include "ae/ae.h"
-}
-
-/// Constant specifying that the timer is done and it will be removed.
-constexpr int kEventLoopTimerDone = AE_NOMORE;
-
-/// Read event on the file descriptor.
-constexpr int kEventLoopRead = AE_READABLE;
-
-/// Write event on the file descriptor.
-constexpr int kEventLoopWrite = AE_WRITABLE;
-
-typedef long long TimerID; // NOLINT
-
-class EventLoop {
- public:
- // Signature of the handler that will be called when there is a new event
- // on the file descriptor that this handler has been registered for.
- //
- // The arguments are the event flags (read or write).
- using FileCallback = std::function<void(int)>;
-
- // This handler will be called when a timer times out. The timer id is
- // passed as an argument. The return is the number of milliseconds the timer
- // shall be reset to or kEventLoopTimerDone if the timer shall not be
- // triggered again.
- using TimerCallback = std::function<int(int64_t)>;
-
- EventLoop();
-
- /// Add a new file event handler to the event loop.
- ///
- /// @param fd The file descriptor we are listening to.
- /// @param events The flags for events we are listening to (read or write).
- /// @param callback The callback that will be called when the event happens.
- /// @return Returns true if the event handler was added successfully.
- bool add_file_event(int fd, int events, const FileCallback& callback);
-
- /// Remove a file event handler from the event loop.
- ///
- /// @param fd The file descriptor of the event handler.
- /// @return Void.
- void remove_file_event(int fd);
-
- /// Register a handler that will be called after a time slice of
- /// "timeout" milliseconds.
- ///
- /// @param timeout The timeout in milliseconds.
- /// @param callback The callback for the timeout.
- /// @return The ID of the newly created timer.
- int64_t add_timer(int64_t timeout, const TimerCallback& callback);
-
- /// Remove a timer handler from the event loop.
- ///
- /// @param timer_id The ID of the timer that is to be removed.
- /// @return The ae.c error code. TODO(pcm): needs to be standardized
- int remove_timer(int64_t timer_id);
-
- /// Run the event loop.
- ///
- /// @return Void.
- void run();
-
- private:
- static void file_event_callback(aeEventLoop* loop, int fd, void* context, int events);
-
- static int timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context);
-
- aeEventLoop* loop_;
- std::unordered_map<int, std::unique_ptr<FileCallback>> file_callbacks_;
- std::unordered_map<int64_t, std::unique_ptr<TimerCallback>> timer_callbacks_;
-};
-
-#endif // PLASMA_EVENTS
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/eviction_policy.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
deleted file mode 100644
index 4ae6384..0000000
--- a/cpp/src/plasma/eviction_policy.cc
+++ /dev/null
@@ -1,107 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/eviction_policy.h"
-
-#include <algorithm>
-
-void LRUCache::add(const ObjectID& key, int64_t size) {
- auto it = item_map_.find(key);
- ARROW_CHECK(it == item_map_.end());
- /* Note that it is important to use a list so the iterators stay valid. */
- item_list_.emplace_front(key, size);
- item_map_.emplace(key, item_list_.begin());
-}
-
-void LRUCache::remove(const ObjectID& key) {
- auto it = item_map_.find(key);
- ARROW_CHECK(it != item_map_.end());
- item_list_.erase(it->second);
- item_map_.erase(it);
-}
-
-int64_t LRUCache::choose_objects_to_evict(
- int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
- int64_t bytes_evicted = 0;
- auto it = item_list_.end();
- while (bytes_evicted < num_bytes_required && it != item_list_.begin()) {
- it--;
- objects_to_evict->push_back(it->first);
- bytes_evicted += it->second;
- }
- return bytes_evicted;
-}
-
-EvictionPolicy::EvictionPolicy(PlasmaStoreInfo* store_info)
- : memory_used_(0), store_info_(store_info) {}
-
-int64_t EvictionPolicy::choose_objects_to_evict(
- int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
- int64_t bytes_evicted =
- cache_.choose_objects_to_evict(num_bytes_required, objects_to_evict);
- /* Update the LRU cache. */
- for (auto& object_id : *objects_to_evict) {
- cache_.remove(object_id);
- }
- /* Update the number of bytes used. */
- memory_used_ -= bytes_evicted;
- return bytes_evicted;
-}
-
-void EvictionPolicy::object_created(const ObjectID& object_id) {
- auto entry = store_info_->objects[object_id].get();
- cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
-}
-
-bool EvictionPolicy::require_space(
- int64_t size, std::vector<ObjectID>* objects_to_evict) {
- /* Check if there is enough space to create the object. */
- int64_t required_space = memory_used_ + size - store_info_->memory_capacity;
- int64_t num_bytes_evicted;
- if (required_space > 0) {
- /* Try to free up at least as much space as we need right now but ideally
- * up to 20% of the total capacity. */
- int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5);
- ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects";
- /* Choose some objects to evict, and update the return pointers. */
- num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict);
- ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting "
- << objects_to_evict->size() << " objects to free up "
- << num_bytes_evicted << " bytes.";
- } else {
- num_bytes_evicted = 0;
- }
- if (num_bytes_evicted >= required_space) {
- /* We only increment the space used if there is enough space to create the
- * object. */
- memory_used_ += size;
- }
- return num_bytes_evicted >= required_space;
-}
-
-void EvictionPolicy::begin_object_access(
- const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
- /* If the object is in the LRU cache, remove it. */
- cache_.remove(object_id);
-}
-
-void EvictionPolicy::end_object_access(
- const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
- auto entry = store_info_->objects[object_id].get();
- /* Add the object to the LRU cache.*/
- cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/eviction_policy.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
deleted file mode 100644
index 3815fc6..0000000
--- a/cpp/src/plasma/eviction_policy.h
+++ /dev/null
@@ -1,134 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_EVICTION_POLICY_H
-#define PLASMA_EVICTION_POLICY_H
-
-#include <list>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-#include "plasma/common.h"
-#include "plasma/plasma.h"
-
-// ==== The eviction policy ====
-//
-// This file contains declaration for all functions and data structures that
-// need to be provided if you want to implement a new eviction algorithm for the
-// Plasma store.
-
-class LRUCache {
- public:
- LRUCache() {}
-
- void add(const ObjectID& key, int64_t size);
-
- void remove(const ObjectID& key);
-
- int64_t choose_objects_to_evict(
- int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
-
- private:
- /// A doubly-linked list containing the items in the cache and
- /// their sizes in LRU order.
- typedef std::list<std::pair<ObjectID, int64_t>> ItemList;
- ItemList item_list_;
- /// A hash table mapping the object ID of an object in the cache to its
- /// location in the doubly linked list item_list_.
- std::unordered_map<ObjectID, ItemList::iterator, UniqueIDHasher> item_map_;
-};
-
-/// The eviction policy.
-class EvictionPolicy {
- public:
- /// Construct an eviction policy.
- ///
- /// @param store_info Information about the Plasma store that is exposed
- /// to the eviction policy.
- explicit EvictionPolicy(PlasmaStoreInfo* store_info);
-
- /// This method will be called whenever an object is first created in order to
- /// add it to the LRU cache. This is done so that the first time, the Plasma
- /// store calls begin_object_access, we can remove the object from the LRU
- /// cache.
- ///
- /// @param object_id The object ID of the object that was created.
- /// @return Void.
- void object_created(const ObjectID& object_id);
-
- /// This method will be called when the Plasma store needs more space, perhaps
- /// to create a new object. If the required amount of space cannot be freed up,
- /// then a fatal error will be thrown. When this method is called, the eviction
- /// policy will assume that the objects chosen to be evicted will in fact be
- /// evicted from the Plasma store by the caller.
- ///
- /// @param size The size in bytes of the new object, including both data and
- /// metadata.
- /// @param objects_to_evict The object IDs that were chosen for eviction will
- /// be stored into this vector.
- /// @return True if enough space can be freed and false otherwise.
- bool require_space(int64_t size, std::vector<ObjectID>* objects_to_evict);
-
- /// This method will be called whenever an unused object in the Plasma store
- /// starts to be used. When this method is called, the eviction policy will
- /// assume that the objects chosen to be evicted will in fact be evicted from
- /// the Plasma store by the caller.
- ///
- /// @param object_id The ID of the object that is now being used.
- /// @param objects_to_evict The object IDs that were chosen for eviction will
- /// be stored into this vector.
- /// @return Void.
- void begin_object_access(
- const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
-
- /// This method will be called whenever an object in the Plasma store that was
- /// being used is no longer being used. When this method is called, the
- /// eviction policy will assume that the objects chosen to be evicted will in
- /// fact be evicted from the Plasma store by the caller.
- ///
- /// @param object_id The ID of the object that is no longer being used.
- /// @param objects_to_evict The object IDs that were chosen for eviction will
- /// be stored into this vector.
- /// @return Void.
- void end_object_access(
- const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
-
- /// Choose some objects to evict from the Plasma store. When this method is
- /// called, the eviction policy will assume that the objects chosen to be
- /// evicted will in fact be evicted from the Plasma store by the caller.
- ///
- /// @note This method is not part of the API. It is exposed in the header file
- /// only for testing.
- ///
- /// @param num_bytes_required The number of bytes of space to try to free up.
- /// @param objects_to_evict The object IDs that were chosen for eviction will
- /// be stored into this vector.
- /// @return The total number of bytes of space chosen to be evicted.
- int64_t choose_objects_to_evict(
- int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
-
- private:
- /// The amount of memory (in bytes) currently being used.
- int64_t memory_used_;
- /// Pointer to the plasma store info.
- PlasmaStoreInfo* store_info_;
- /// Datastructure for the LRU cache.
- LRUCache cache_;
-};
-
-#endif // PLASMA_EVICTION_POLICY_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/extension.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/extension.cc b/cpp/src/plasma/extension.cc
deleted file mode 100644
index 5d61e33..0000000
--- a/cpp/src/plasma/extension.cc
+++ /dev/null
@@ -1,456 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "plasma/extension.h"
-
-#include <algorithm>
-#include <vector>
-
-#include "plasma/client.h"
-#include "plasma/common.h"
-#include "plasma/io.h"
-#include "plasma/protocol.h"
-
-PyObject* PlasmaOutOfMemoryError;
-PyObject* PlasmaObjectExistsError;
-
-PyObject* PyPlasma_connect(PyObject* self, PyObject* args) {
- const char* store_socket_name;
- const char* manager_socket_name;
- int release_delay;
- if (!PyArg_ParseTuple(
- args, "ssi", &store_socket_name, &manager_socket_name, &release_delay)) {
- return NULL;
- }
- PlasmaClient* client = new PlasmaClient();
- ARROW_CHECK_OK(client->Connect(store_socket_name, manager_socket_name, release_delay));
-
- return PyCapsule_New(client, "plasma", NULL);
-}
-
-PyObject* PyPlasma_disconnect(PyObject* self, PyObject* args) {
- PyObject* client_capsule;
- if (!PyArg_ParseTuple(args, "O", &client_capsule)) { return NULL; }
- PlasmaClient* client;
- ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client));
- ARROW_CHECK_OK(client->Disconnect());
- /* We use the context of the connection capsule to indicate if the connection
- * is still active (if the context is NULL) or if it is closed (if the context
- * is (void*) 0x1). This is neccessary because the primary pointer of the
- * capsule cannot be NULL. */
- PyCapsule_SetContext(client_capsule, reinterpret_cast<void*>(0x1));
- Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_create(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- ObjectID object_id;
- Py_ssize_t size;
- PyObject* metadata;
- if (!PyArg_ParseTuple(args, "O&O&nO", PyObjectToPlasmaClient, &client,
- PyStringToUniqueID, &object_id, &size, &metadata)) {
- return NULL;
- }
- if (!PyByteArray_Check(metadata)) {
- PyErr_SetString(PyExc_TypeError, "metadata must be a bytearray");
- return NULL;
- }
- uint8_t* data;
- Status s = client->Create(object_id, size,
- reinterpret_cast<uint8_t*>(PyByteArray_AsString(metadata)),
- PyByteArray_Size(metadata), &data);
- if (s.IsPlasmaObjectExists()) {
- PyErr_SetString(PlasmaObjectExistsError,
- "An object with this ID already exists in the plasma "
- "store.");
- return NULL;
- }
- if (s.IsPlasmaStoreFull()) {
- PyErr_SetString(PlasmaOutOfMemoryError,
- "The plasma store ran out of memory and could not create "
- "this object.");
- return NULL;
- }
- ARROW_CHECK(s.ok());
-
-#if PY_MAJOR_VERSION >= 3
- return PyMemoryView_FromMemory(reinterpret_cast<char*>(data), size, PyBUF_WRITE);
-#else
- return PyBuffer_FromReadWriteMemory(reinterpret_cast<void*>(data), size);
-#endif
-}
-
-PyObject* PyPlasma_hash(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- ObjectID object_id;
- if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
- &object_id)) {
- return NULL;
- }
- unsigned char digest[kDigestSize];
- bool success = plasma_compute_object_hash(client, object_id, digest);
- if (success) {
- PyObject* digest_string =
- PyBytes_FromStringAndSize(reinterpret_cast<char*>(digest), kDigestSize);
- return digest_string;
- } else {
- Py_RETURN_NONE;
- }
-}
-
-PyObject* PyPlasma_seal(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- ObjectID object_id;
- if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
- &object_id)) {
- return NULL;
- }
- ARROW_CHECK_OK(client->Seal(object_id));
- Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_release(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- ObjectID object_id;
- if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
- &object_id)) {
- return NULL;
- }
- ARROW_CHECK_OK(client->Release(object_id));
- Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_get(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- PyObject* object_id_list;
- Py_ssize_t timeout_ms;
- if (!PyArg_ParseTuple(
- args, "O&On", PyObjectToPlasmaClient, &client, &object_id_list, &timeout_ms)) {
- return NULL;
- }
-
- Py_ssize_t num_object_ids = PyList_Size(object_id_list);
- std::vector<ObjectID> object_ids(num_object_ids);
- std::vector<ObjectBuffer> object_buffers(num_object_ids);
-
- for (int i = 0; i < num_object_ids; ++i) {
- PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
- }
-
- Py_BEGIN_ALLOW_THREADS;
- ARROW_CHECK_OK(
- client->Get(object_ids.data(), num_object_ids, timeout_ms, object_buffers.data()));
- Py_END_ALLOW_THREADS;
-
- PyObject* returns = PyList_New(num_object_ids);
- for (int i = 0; i < num_object_ids; ++i) {
- if (object_buffers[i].data_size != -1) {
- /* The object was retrieved, so return the object. */
- PyObject* t = PyTuple_New(2);
- Py_ssize_t data_size = static_cast<Py_ssize_t>(object_buffers[i].data_size);
- Py_ssize_t metadata_size = static_cast<Py_ssize_t>(object_buffers[i].metadata_size);
-#if PY_MAJOR_VERSION >= 3
- char* data = reinterpret_cast<char*>(object_buffers[i].data);
- char* metadata = reinterpret_cast<char*>(object_buffers[i].metadata);
- PyTuple_SET_ITEM(t, 0, PyMemoryView_FromMemory(data, data_size, PyBUF_READ));
- PyTuple_SET_ITEM(
- t, 1, PyMemoryView_FromMemory(metadata, metadata_size, PyBUF_READ));
-#else
- void* data = reinterpret_cast<void*>(object_buffers[i].data);
- void* metadata = reinterpret_cast<void*>(object_buffers[i].metadata);
- PyTuple_SET_ITEM(t, 0, PyBuffer_FromMemory(data, data_size));
- PyTuple_SET_ITEM(t, 1, PyBuffer_FromMemory(metadata, metadata_size));
-#endif
- ARROW_CHECK(PyList_SetItem(returns, i, t) == 0);
- } else {
- /* The object was not retrieved, so just add None to the list of return
- * values. */
- Py_INCREF(Py_None);
- ARROW_CHECK(PyList_SetItem(returns, i, Py_None) == 0);
- }
- }
- return returns;
-}
-
-PyObject* PyPlasma_contains(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- ObjectID object_id;
- if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
- &object_id)) {
- return NULL;
- }
- bool has_object;
- ARROW_CHECK_OK(client->Contains(object_id, &has_object));
-
- if (has_object) {
- Py_RETURN_TRUE;
- } else {
- Py_RETURN_FALSE;
- }
-}
-
-PyObject* PyPlasma_fetch(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- PyObject* object_id_list;
- if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client, &object_id_list)) {
- return NULL;
- }
- if (client->get_manager_fd() == -1) {
- PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
- return NULL;
- }
- Py_ssize_t n = PyList_Size(object_id_list);
- ObjectID* object_ids = new ObjectID[n];
- for (int i = 0; i < n; ++i) {
- PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
- }
- ARROW_CHECK_OK(client->Fetch(static_cast<int>(n), object_ids));
- delete[] object_ids;
- Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_wait(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- PyObject* object_id_list;
- Py_ssize_t timeout;
- int num_returns;
- if (!PyArg_ParseTuple(args, "O&Oni", PyObjectToPlasmaClient, &client, &object_id_list,
- &timeout, &num_returns)) {
- return NULL;
- }
- Py_ssize_t n = PyList_Size(object_id_list);
-
- if (client->get_manager_fd() == -1) {
- PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
- return NULL;
- }
- if (num_returns < 0) {
- PyErr_SetString(
- PyExc_RuntimeError, "The argument num_returns cannot be less than zero.");
- return NULL;
- }
- if (num_returns > n) {
- PyErr_SetString(PyExc_RuntimeError,
- "The argument num_returns cannot be greater than len(object_ids)");
- return NULL;
- }
- int64_t threshold = 1 << 30;
- if (timeout > threshold) {
- PyErr_SetString(
- PyExc_RuntimeError, "The argument timeout cannot be greater than 2 ** 30.");
- return NULL;
- }
-
- std::vector<ObjectRequest> object_requests(n);
- for (int i = 0; i < n; ++i) {
- ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i),
- &object_requests[i].object_id) == 1);
- object_requests[i].type = PLASMA_QUERY_ANYWHERE;
- }
- /* Drop the global interpreter lock while we are waiting, so other threads can
- * run. */
- int num_return_objects;
- Py_BEGIN_ALLOW_THREADS;
- ARROW_CHECK_OK(
- client->Wait(n, object_requests.data(), num_returns, timeout, &num_return_objects));
- Py_END_ALLOW_THREADS;
-
- int num_to_return = std::min(num_return_objects, num_returns);
- PyObject* ready_ids = PyList_New(num_to_return);
- PyObject* waiting_ids = PySet_New(object_id_list);
- int num_returned = 0;
- for (int i = 0; i < n; ++i) {
- if (num_returned == num_to_return) { break; }
- if (object_requests[i].status == ObjectStatus_Local ||
- object_requests[i].status == ObjectStatus_Remote) {
- PyObject* ready = PyBytes_FromStringAndSize(
- reinterpret_cast<char*>(&object_requests[i].object_id),
- sizeof(object_requests[i].object_id));
- PyList_SetItem(ready_ids, num_returned, ready);
- PySet_Discard(waiting_ids, ready);
- num_returned += 1;
- } else {
- ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent);
- }
- }
- ARROW_CHECK(num_returned == num_to_return);
- /* Return both the ready IDs and the remaining IDs. */
- PyObject* t = PyTuple_New(2);
- PyTuple_SetItem(t, 0, ready_ids);
- PyTuple_SetItem(t, 1, waiting_ids);
- return t;
-}
-
-PyObject* PyPlasma_evict(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- Py_ssize_t num_bytes;
- if (!PyArg_ParseTuple(args, "O&n", PyObjectToPlasmaClient, &client, &num_bytes)) {
- return NULL;
- }
- int64_t evicted_bytes;
- ARROW_CHECK_OK(client->Evict(static_cast<int64_t>(num_bytes), evicted_bytes));
- return PyLong_FromSsize_t(static_cast<Py_ssize_t>(evicted_bytes));
-}
-
-PyObject* PyPlasma_delete(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- ObjectID object_id;
- if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
- &object_id)) {
- return NULL;
- }
- ARROW_CHECK_OK(client->Delete(object_id));
- Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_transfer(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- ObjectID object_id;
- const char* addr;
- int port;
- if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client,
- PyStringToUniqueID, &object_id, &addr, &port)) {
- return NULL;
- }
-
- if (client->get_manager_fd() == -1) {
- PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
- return NULL;
- }
-
- ARROW_CHECK_OK(client->Transfer(addr, port, object_id));
- Py_RETURN_NONE;
-}
-
-PyObject* PyPlasma_subscribe(PyObject* self, PyObject* args) {
- PlasmaClient* client;
- if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) { return NULL; }
-
- int sock;
- ARROW_CHECK_OK(client->Subscribe(&sock));
- return PyLong_FromLong(sock);
-}
-
-PyObject* PyPlasma_receive_notification(PyObject* self, PyObject* args) {
- int plasma_sock;
-
- if (!PyArg_ParseTuple(args, "i", &plasma_sock)) { return NULL; }
- /* Receive object notification from the plasma connection socket. If the
- * object was added, return a tuple of its fields: ObjectID, data_size,
- * metadata_size. If the object was deleted, data_size and metadata_size will
- * be set to -1. */
- uint8_t* notification = read_message_async(plasma_sock);
- if (notification == NULL) {
- PyErr_SetString(
- PyExc_RuntimeError, "Failed to read object notification from Plasma socket");
- return NULL;
- }
- auto object_info = flatbuffers::GetRoot<ObjectInfo>(notification);
- /* Construct a tuple from object_info and return. */
- PyObject* t = PyTuple_New(3);
- PyTuple_SetItem(t, 0, PyBytes_FromStringAndSize(object_info->object_id()->data(),
- object_info->object_id()->size()));
- if (object_info->is_deletion()) {
- PyTuple_SetItem(t, 1, PyLong_FromLong(-1));
- PyTuple_SetItem(t, 2, PyLong_FromLong(-1));
- } else {
- PyTuple_SetItem(t, 1, PyLong_FromLong(object_info->data_size()));
- PyTuple_SetItem(t, 2, PyLong_FromLong(object_info->metadata_size()));
- }
-
- delete[] notification;
- return t;
-}
-
-static PyMethodDef plasma_methods[] = {
- {"connect", PyPlasma_connect, METH_VARARGS, "Connect to plasma."},
- {"disconnect", PyPlasma_disconnect, METH_VARARGS, "Disconnect from plasma."},
- {"create", PyPlasma_create, METH_VARARGS, "Create a new plasma object."},
- {"hash", PyPlasma_hash, METH_VARARGS, "Compute the hash of a plasma object."},
- {"seal", PyPlasma_seal, METH_VARARGS, "Seal a plasma object."},
- {"get", PyPlasma_get, METH_VARARGS, "Get a plasma object."},
- {"contains", PyPlasma_contains, METH_VARARGS,
- "Does the plasma store contain this plasma object?"},
- {"fetch", PyPlasma_fetch, METH_VARARGS,
- "Fetch the object from another plasma manager instance."},
- {"wait", PyPlasma_wait, METH_VARARGS,
- "Wait until num_returns objects in object_ids are ready."},
- {"evict", PyPlasma_evict, METH_VARARGS,
- "Evict some objects until we recover some number of bytes."},
- {"release", PyPlasma_release, METH_VARARGS, "Release the plasma object."},
- {"delete", PyPlasma_delete, METH_VARARGS, "Delete a plasma object."},
- {"transfer", PyPlasma_transfer, METH_VARARGS,
- "Transfer object to another plasma manager."},
- {"subscribe", PyPlasma_subscribe, METH_VARARGS,
- "Subscribe to the plasma notification socket."},
- {"receive_notification", PyPlasma_receive_notification, METH_VARARGS,
- "Receive next notification from plasma notification socket."},
- {NULL} /* Sentinel */
-};
-
-#if PY_MAJOR_VERSION >= 3
-static struct PyModuleDef moduledef = {
- PyModuleDef_HEAD_INIT, "libplasma", /* m_name */
- "A Python client library for plasma.", /* m_doc */
- 0, /* m_size */
- plasma_methods, /* m_methods */
- NULL, /* m_reload */
- NULL, /* m_traverse */
- NULL, /* m_clear */
- NULL, /* m_free */
-};
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define INITERROR return NULL
-#else
-#define INITERROR return
-#endif
-
-#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
-#define PyMODINIT_FUNC void
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
-#else
-#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
-#endif
-
-MOD_INIT(libplasma) {
-#if PY_MAJOR_VERSION >= 3
- PyObject* m = PyModule_Create(&moduledef);
-#else
- PyObject* m =
- Py_InitModule3("libplasma", plasma_methods, "A Python client library for plasma.");
-#endif
-
- /* Create a custom exception for when an object ID is reused. */
- char plasma_object_exists_error[] = "plasma_object_exists.error";
- PlasmaObjectExistsError = PyErr_NewException(plasma_object_exists_error, NULL, NULL);
- Py_INCREF(PlasmaObjectExistsError);
- PyModule_AddObject(m, "plasma_object_exists_error", PlasmaObjectExistsError);
- /* Create a custom exception for when the plasma store is out of memory. */
- char plasma_out_of_memory_error[] = "plasma_out_of_memory.error";
- PlasmaOutOfMemoryError = PyErr_NewException(plasma_out_of_memory_error, NULL, NULL);
- Py_INCREF(PlasmaOutOfMemoryError);
- PyModule_AddObject(m, "plasma_out_of_memory_error", PlasmaOutOfMemoryError);
-
-#if PY_MAJOR_VERSION >= 3
- return m;
-#endif
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/62ef2cd8/cpp/src/plasma/extension.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/extension.h b/cpp/src/plasma/extension.h
deleted file mode 100644
index cee30ab..0000000
--- a/cpp/src/plasma/extension.h
+++ /dev/null
@@ -1,50 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PLASMA_EXTENSION_H
-#define PLASMA_EXTENSION_H
-
-#undef _XOPEN_SOURCE
-#undef _POSIX_C_SOURCE
-#include <Python.h>
-
-#include "bytesobject.h" // NOLINT
-
-#include "plasma/client.h"
-#include "plasma/common.h"
-
-static int PyObjectToPlasmaClient(PyObject* object, PlasmaClient** client) {
- if (PyCapsule_IsValid(object, "plasma")) {
- *client = reinterpret_cast<PlasmaClient*>(PyCapsule_GetPointer(object, "plasma"));
- return 1;
- } else {
- PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule");
- return 0;
- }
-}
-
-int PyStringToUniqueID(PyObject* object, ObjectID* object_id) {
- if (PyBytes_Check(object)) {
- memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID));
- return 1;
- } else {
- PyErr_SetString(PyExc_TypeError, "must be a 20 character string");
- return 0;
- }
-}
-
-#endif // PLASMA_EXTENSION_H
[14/14] arrow git commit: [C++] Restore Plasma source tree after
0.5.0 release
Posted by we...@apache.org.
[C++] Restore Plasma source tree after 0.5.0 release
This reverts commit 62ef2cd8a39fc93e7fa4bb790d7cd92adb77571f.
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/2c810151
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/2c810151
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/2c810151
Branch: refs/heads/master
Commit: 2c8101515e2a0ab515c03f82dc84b02ca6c466da
Parents: 9b26ed8
Author: Wes McKinney <we...@twosigma.com>
Authored: Sun Jul 23 14:37:54 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jul 23 14:37:54 2017 -0400
----------------------------------------------------------------------
cpp/src/plasma/CMakeLists.txt | 113 +
cpp/src/plasma/client.cc | 557 ++
cpp/src/plasma/client.h | 343 ++
cpp/src/plasma/common.cc | 83 +
cpp/src/plasma/common.h | 63 +
cpp/src/plasma/events.cc | 81 +
cpp/src/plasma/events.h | 99 +
cpp/src/plasma/eviction_policy.cc | 107 +
cpp/src/plasma/eviction_policy.h | 134 +
cpp/src/plasma/extension.cc | 456 ++
cpp/src/plasma/extension.h | 50 +
cpp/src/plasma/fling.cc | 90 +
cpp/src/plasma/fling.h | 52 +
cpp/src/plasma/format/.gitignore | 1 +
cpp/src/plasma/format/common.fbs | 34 +
cpp/src/plasma/format/plasma.fbs | 291 ++
cpp/src/plasma/io.cc | 212 +
cpp/src/plasma/io.h | 55 +
cpp/src/plasma/malloc.cc | 178 +
cpp/src/plasma/malloc.h | 26 +
cpp/src/plasma/plasma.cc | 64 +
cpp/src/plasma/plasma.h | 191 +
cpp/src/plasma/protocol.cc | 502 ++
cpp/src/plasma/protocol.h | 170 +
cpp/src/plasma/store.cc | 683 +++
cpp/src/plasma/store.h | 169 +
cpp/src/plasma/test/client_tests.cc | 132 +
cpp/src/plasma/test/run_tests.sh | 61 +
cpp/src/plasma/test/run_valgrind.sh | 27 +
cpp/src/plasma/test/serialization_tests.cc | 388 ++
cpp/src/plasma/thirdparty/ae/ae.c | 465 ++
cpp/src/plasma/thirdparty/ae/ae.h | 123 +
cpp/src/plasma/thirdparty/ae/ae_epoll.c | 135 +
cpp/src/plasma/thirdparty/ae/ae_evport.c | 320 ++
cpp/src/plasma/thirdparty/ae/ae_kqueue.c | 138 +
cpp/src/plasma/thirdparty/ae/ae_select.c | 106 +
cpp/src/plasma/thirdparty/ae/config.h | 54 +
cpp/src/plasma/thirdparty/ae/zmalloc.h | 45 +
cpp/src/plasma/thirdparty/dlmalloc.c | 6281 +++++++++++++++++++++++
cpp/src/plasma/thirdparty/xxhash.cc | 889 ++++
cpp/src/plasma/thirdparty/xxhash.h | 293 ++
41 files changed, 14261 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
new file mode 100644
index 0000000..4ff3beb
--- /dev/null
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 2.8)
+
+project(plasma)
+
+find_package(PythonLibsNew REQUIRED)
+find_package(Threads)
+
+option(PLASMA_PYTHON
+ "Build the Plasma Python extensions"
+ OFF)
+
+if(APPLE)
+ SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
+endif(APPLE)
+
+include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
+include_directories("${FLATBUFFERS_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/" "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" "${CMAKE_CURRENT_LIST_DIR}/../")
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L")
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-conversion")
+
+# Compile flatbuffers
+
+set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
+set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
+
+set(PLASMA_FBS_OUTPUT_FILES
+ "${OUTPUT_DIR}/common_generated.h"
+ "${OUTPUT_DIR}/plasma_generated.h")
+
+add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
+
+if(FLATBUFFERS_VENDORED)
+ add_dependencies(gen_plasma_fbs flatbuffers_ep)
+endif()
+
+add_custom_command(
+ OUTPUT ${PLASMA_FBS_OUTPUT_FILES}
+ # The --gen-object-api flag generates a C++ class MessageT for each
+ # flatbuffers message Message, which can be used to store deserialized
+ # messages in data structures. This is currently used for ObjectInfo for
+ # example.
+ COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} --gen-object-api
+ DEPENDS ${PLASMA_FBS_SRC}
+ COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}"
+ VERBATIM)
+
+if(UNIX AND NOT APPLE)
+ link_libraries(rt)
+endif()
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
+set_source_files_properties(extension.cc PROPERTIES COMPILE_FLAGS -Wno-strict-aliasing)
+
+set(PLASMA_SRCS
+ client.cc
+ common.cc
+ eviction_policy.cc
+ events.cc
+ fling.cc
+ io.cc
+ malloc.cc
+ plasma.cc
+ protocol.cc
+ thirdparty/ae/ae.c
+ thirdparty/xxhash.cc)
+
+ADD_ARROW_LIB(plasma
+ SOURCES ${PLASMA_SRCS}
+ DEPENDENCIES gen_plasma_fbs
+ SHARED_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static
+ STATIC_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} arrow_static)
+
+# The optimization flag -O3 is suggested by dlmalloc.c, which is #included in
+# malloc.cc; we set it here regardless of whether we do a debug or release build.
+set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error -O3")
+
+add_executable(plasma_store store.cc)
+target_link_libraries(plasma_store plasma_static)
+
+ADD_ARROW_TEST(test/serialization_tests)
+ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static)
+ADD_ARROW_TEST(test/client_tests)
+ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static)
+
+if(PLASMA_PYTHON)
+ add_library(plasma_extension SHARED extension.cc)
+
+ if(APPLE)
+ target_link_libraries(plasma_extension plasma_static "-undefined dynamic_lookup")
+ else(APPLE)
+ target_link_libraries(plasma_extension plasma_static -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive)
+ endif(APPLE)
+endif()
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/client.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc
new file mode 100644
index 0000000..dcb78e7
--- /dev/null
+++ b/cpp/src/plasma/client.cc
@@ -0,0 +1,557 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// PLASMA CLIENT: Client library for using the plasma store and manager
+
+#include "plasma/client.h"
+
+#ifdef _WIN32
+#include <Win32_Interop/win32_types.h>
+#endif
+
+#include <assert.h>
+#include <fcntl.h>
+#include <netinet/in.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <thread>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/fling.h"
+#include "plasma/io.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+#define XXH_STATIC_LINKING_ONLY
+#include "thirdparty/xxhash.h"
+
+#define XXH64_DEFAULT_SEED 0
+
+// Number of threads used for memcopy and hash computations.
+constexpr int64_t kThreadPoolSize = 8;
+constexpr int64_t kBytesInMB = 1 << 20;
+static std::vector<std::thread> threadpool_(kThreadPoolSize);
+
+// If the file descriptor fd has been mmapped in this client process before,
+// return the pointer that was returned by mmap, otherwise mmap it and store the
+// pointer in a hash table.
+uint8_t* PlasmaClient::lookup_or_mmap(int fd, int store_fd_val, int64_t map_size) {
+ auto entry = mmap_table_.find(store_fd_val);
+ if (entry != mmap_table_.end()) {
+ close(fd);
+ return entry->second.pointer;
+ } else {
+ uint8_t* result = reinterpret_cast<uint8_t*>(
+ mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
+ // TODO(pcm): Don't fail here, instead return a Status.
+ if (result == MAP_FAILED) { ARROW_LOG(FATAL) << "mmap failed"; }
+ close(fd);
+ ClientMmapTableEntry& entry = mmap_table_[store_fd_val];
+ entry.pointer = result;
+ entry.length = map_size;
+ entry.count = 0;
+ return result;
+ }
+}
+
+// Get a pointer to a file that we know has been memory mapped in this client
+// process before.
+uint8_t* PlasmaClient::lookup_mmapped_file(int store_fd_val) {
+ auto entry = mmap_table_.find(store_fd_val);
+ ARROW_CHECK(entry != mmap_table_.end());
+ return entry->second.pointer;
+}
+
+void PlasmaClient::increment_object_count(
+ const ObjectID& object_id, PlasmaObject* object, bool is_sealed) {
+ // Increment the count of the object to track the fact that it is being used.
+ // The corresponding decrement should happen in PlasmaClient::Release.
+ auto elem = objects_in_use_.find(object_id);
+ ObjectInUseEntry* object_entry;
+ if (elem == objects_in_use_.end()) {
+ // Add this object ID to the hash table of object IDs in use. The
+ // corresponding call to free happens in PlasmaClient::Release.
+ objects_in_use_[object_id] =
+ std::unique_ptr<ObjectInUseEntry>(new ObjectInUseEntry());
+ objects_in_use_[object_id]->object = *object;
+ objects_in_use_[object_id]->count = 0;
+ objects_in_use_[object_id]->is_sealed = is_sealed;
+ object_entry = objects_in_use_[object_id].get();
+ // Increment the count of the number of objects in the memory-mapped file
+ // that are being used. The corresponding decrement should happen in
+ // PlasmaClient::Release.
+ auto entry = mmap_table_.find(object->handle.store_fd);
+ ARROW_CHECK(entry != mmap_table_.end());
+ ARROW_CHECK(entry->second.count >= 0);
+ // Update the in_use_object_bytes_.
+ in_use_object_bytes_ +=
+ (object_entry->object.data_size + object_entry->object.metadata_size);
+ entry->second.count += 1;
+ } else {
+ object_entry = elem->second.get();
+ ARROW_CHECK(object_entry->count > 0);
+ }
+ // Increment the count of the number of instances of this object that are
+ // being used by this client. The corresponding decrement should happen in
+ // PlasmaClient::Release.
+ object_entry->count += 1;
+}
+
+Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size,
+ uint8_t* metadata, int64_t metadata_size, uint8_t** data) {
+ ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size "
+ << data_size << " and metadata size " << metadata_size;
+ RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size));
+ std::vector<uint8_t> buffer;
+ RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaCreateReply, &buffer));
+ ObjectID id;
+ PlasmaObject object;
+ RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object));
+ // If the CreateReply included an error, then the store will not send a file
+ // descriptor.
+ int fd = recv_fd(store_conn_);
+ ARROW_CHECK(fd >= 0) << "recv not successful";
+ ARROW_CHECK(object.data_size == data_size);
+ ARROW_CHECK(object.metadata_size == metadata_size);
+ // The metadata should come right after the data.
+ ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
+ *data = lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) +
+ object.data_offset;
+ // If plasma_create is being called from a transfer, then we will not copy the
+ // metadata here. The metadata will be written along with the data streamed
+ // from the transfer.
+ if (metadata != NULL) {
+ // Copy the metadata to the buffer.
+ memcpy(*data + object.data_size, metadata, metadata_size);
+ }
+ // Increment the count of the number of instances of this object that this
+ // client is using. A call to PlasmaClient::Release is required to decrement
+ // this
+ // count. Cache the reference to the object.
+ increment_object_count(object_id, &object, false);
+ // We increment the count a second time (and the corresponding decrement will
+ // happen in a PlasmaClient::Release call in plasma_seal) so even if the
+ // buffer
+ // returned by PlasmaClient::Dreate goes out of scope, the object does not get
+ // released before the call to PlasmaClient::Seal happens.
+ increment_object_count(object_id, &object, false);
+ return Status::OK();
+}
+
+Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects,
+ int64_t timeout_ms, ObjectBuffer* object_buffers) {
+ // Fill out the info for the objects that are already in use locally.
+ bool all_present = true;
+ for (int i = 0; i < num_objects; ++i) {
+ auto object_entry = objects_in_use_.find(object_ids[i]);
+ if (object_entry == objects_in_use_.end()) {
+ // This object is not currently in use by this client, so we need to send
+ // a request to the store.
+ all_present = false;
+ // Make a note to ourselves that the object is not present.
+ object_buffers[i].data_size = -1;
+ } else {
+ // NOTE: If the object is still unsealed, we will deadlock, since we must
+ // have been the one who created it.
+ ARROW_CHECK(object_entry->second->is_sealed)
+ << "Plasma client called get on an unsealed object that it created";
+ PlasmaObject* object = &object_entry->second->object;
+ object_buffers[i].data = lookup_mmapped_file(object->handle.store_fd);
+ object_buffers[i].data = object_buffers[i].data + object->data_offset;
+ object_buffers[i].data_size = object->data_size;
+ object_buffers[i].metadata = object_buffers[i].data + object->data_size;
+ object_buffers[i].metadata_size = object->metadata_size;
+ // Increment the count of the number of instances of this object that this
+ // client is using. A call to PlasmaClient::Release is required to
+ // decrement this
+ // count. Cache the reference to the object.
+ increment_object_count(object_ids[i], object, true);
+ }
+ }
+
+ if (all_present) { return Status::OK(); }
+
+ // If we get here, then the objects aren't all currently in use by this
+ // client, so we need to send a request to the plasma store.
+ RETURN_NOT_OK(SendGetRequest(store_conn_, object_ids, num_objects, timeout_ms));
+ std::vector<uint8_t> buffer;
+ RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaGetReply, &buffer));
+ std::vector<ObjectID> received_object_ids(num_objects);
+ std::vector<PlasmaObject> object_data(num_objects);
+ PlasmaObject* object;
+ RETURN_NOT_OK(ReadGetReply(
+ buffer.data(), received_object_ids.data(), object_data.data(), num_objects));
+
+ for (int i = 0; i < num_objects; ++i) {
+ DCHECK(received_object_ids[i] == object_ids[i]);
+ object = &object_data[i];
+ if (object_buffers[i].data_size != -1) {
+ // If the object was already in use by the client, then the store should
+ // have returned it.
+ DCHECK_NE(object->data_size, -1);
+ // We won't use this file descriptor, but the store sent us one, so we
+ // need to receive it and then close it right away so we don't leak file
+ // descriptors.
+ int fd = recv_fd(store_conn_);
+ close(fd);
+ ARROW_CHECK(fd >= 0);
+ // We've already filled out the information for this object, so we can
+ // just continue.
+ continue;
+ }
+ // If we are here, the object was not currently in use, so we need to
+ // process the reply from the object store.
+ if (object->data_size != -1) {
+ // The object was retrieved. The user will be responsible for releasing
+ // this object.
+ int fd = recv_fd(store_conn_);
+ ARROW_CHECK(fd >= 0);
+ object_buffers[i].data =
+ lookup_or_mmap(fd, object->handle.store_fd, object->handle.mmap_size);
+ // Finish filling out the return values.
+ object_buffers[i].data = object_buffers[i].data + object->data_offset;
+ object_buffers[i].data_size = object->data_size;
+ object_buffers[i].metadata = object_buffers[i].data + object->data_size;
+ object_buffers[i].metadata_size = object->metadata_size;
+ // Increment the count of the number of instances of this object that this
+ // client is using. A call to PlasmaClient::Release is required to
+ // decrement this
+ // count. Cache the reference to the object.
+ increment_object_count(received_object_ids[i], object, true);
+ } else {
+ // The object was not retrieved. Make sure we already put a -1 here to
+ // indicate that the object was not retrieved. The caller is not
+ // responsible for releasing this object.
+ DCHECK_EQ(object_buffers[i].data_size, -1);
+ object_buffers[i].data_size = -1;
+ }
+ }
+ return Status::OK();
+}
+
+/// This is a helper method for implementing plasma_release. We maintain a
+/// buffer
+/// of release calls and only perform them once the buffer becomes full (as
+/// judged by the aggregate sizes of the objects). There may be multiple release
+/// calls for the same object ID in the buffer. In this case, the first release
+/// calls will not do anything. The client will only send a message to the store
+/// releasing the object when the client is truly done with the object.
+///
+/// @param conn The plasma connection.
+/// @param object_id The object ID to attempt to release.
+Status PlasmaClient::PerformRelease(const ObjectID& object_id) {
+ // Decrement the count of the number of instances of this object that are
+ // being used by this client. The corresponding increment should have happened
+ // in PlasmaClient::Get.
+ auto object_entry = objects_in_use_.find(object_id);
+ ARROW_CHECK(object_entry != objects_in_use_.end());
+ object_entry->second->count -= 1;
+ ARROW_CHECK(object_entry->second->count >= 0);
+ // Check if the client is no longer using this object.
+ if (object_entry->second->count == 0) {
+ // Decrement the count of the number of objects in this memory-mapped file
+ // that the client is using. The corresponding increment should have
+ // happened in plasma_get.
+ int fd = object_entry->second->object.handle.store_fd;
+ auto entry = mmap_table_.find(fd);
+ ARROW_CHECK(entry != mmap_table_.end());
+ entry->second.count -= 1;
+ ARROW_CHECK(entry->second.count >= 0);
+ // If none are being used then unmap the file.
+ if (entry->second.count == 0) {
+ munmap(entry->second.pointer, entry->second.length);
+ // Remove the corresponding entry from the hash table.
+ mmap_table_.erase(fd);
+ }
+ // Tell the store that the client no longer needs the object.
+ RETURN_NOT_OK(SendReleaseRequest(store_conn_, object_id));
+ // Update the in_use_object_bytes_.
+ in_use_object_bytes_ -= (object_entry->second->object.data_size +
+ object_entry->second->object.metadata_size);
+ DCHECK_GE(in_use_object_bytes_, 0);
+ // Remove the entry from the hash table of objects currently in use.
+ objects_in_use_.erase(object_id);
+ }
+ return Status::OK();
+}
+
+Status PlasmaClient::Release(const ObjectID& object_id) {
+ // Add the new object to the release history.
+ release_history_.push_front(object_id);
+ // If there are too many bytes in use by the client or if there are too many
+ // pending release calls, and there are at least some pending release calls in
+ // the release_history list, then release some objects.
+ while ((in_use_object_bytes_ > std::min(kL3CacheSizeBytes, store_capacity_ / 100) ||
+ release_history_.size() > config_.release_delay) &&
+ release_history_.size() > 0) {
+ // Perform a release for the object ID for the first pending release.
+ RETURN_NOT_OK(PerformRelease(release_history_.back()));
+ // Remove the last entry from the release history.
+ release_history_.pop_back();
+ }
+ return Status::OK();
+}
+
+// This method is used to query whether the plasma store contains an object.
+Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) {
+ // Check if we already have a reference to the object.
+ if (objects_in_use_.count(object_id) > 0) {
+ *has_object = 1;
+ } else {
+ // If we don't already have a reference to the object, check with the store
+ // to see if we have the object.
+ RETURN_NOT_OK(SendContainsRequest(store_conn_, object_id));
+ std::vector<uint8_t> buffer;
+ RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaContainsReply, &buffer));
+ ObjectID object_id2;
+ RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object));
+ }
+ return Status::OK();
+}
+
+static void ComputeBlockHash(const unsigned char* data, int64_t nbytes, uint64_t* hash) {
+ XXH64_state_t hash_state;
+ XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
+ XXH64_update(&hash_state, data, nbytes);
+ *hash = XXH64_digest(&hash_state);
+}
+
+static inline bool compute_object_hash_parallel(
+ XXH64_state_t* hash_state, const unsigned char* data, int64_t nbytes) {
+ // Note that this function will likely be faster if the address of data is
+ // aligned on a 64-byte boundary.
+ const int num_threads = kThreadPoolSize;
+ uint64_t threadhash[num_threads + 1];
+ const uint64_t data_address = reinterpret_cast<uint64_t>(data);
+ const uint64_t num_blocks = nbytes / BLOCK_SIZE;
+ const uint64_t chunk_size = (num_blocks / num_threads) * BLOCK_SIZE;
+ const uint64_t right_address = data_address + chunk_size * num_threads;
+ const uint64_t suffix = (data_address + nbytes) - right_address;
+ // Now the data layout is | k * num_threads * block_size | suffix | ==
+ // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size.
+ // Each thread gets a "chunk" of k blocks, except the suffix thread.
+
+ for (int i = 0; i < num_threads; i++) {
+ threadpool_[i] = std::thread(ComputeBlockHash,
+ reinterpret_cast<uint8_t*>(data_address) + i * chunk_size, chunk_size,
+ &threadhash[i]);
+ }
+ ComputeBlockHash(
+ reinterpret_cast<uint8_t*>(right_address), suffix, &threadhash[num_threads]);
+
+ // Join the threads.
+ for (auto& t : threadpool_) {
+ if (t.joinable()) { t.join(); }
+ }
+
+ XXH64_update(hash_state, (unsigned char*)threadhash, sizeof(threadhash));
+ return true;
+}
+
+static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) {
+ XXH64_state_t hash_state;
+ XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
+ if (obj_buffer.data_size >= kBytesInMB) {
+ compute_object_hash_parallel(
+ &hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
+ } else {
+ XXH64_update(&hash_state, (unsigned char*)obj_buffer.data, obj_buffer.data_size);
+ }
+ XXH64_update(
+ &hash_state, (unsigned char*)obj_buffer.metadata, obj_buffer.metadata_size);
+ return XXH64_digest(&hash_state);
+}
+
+bool plasma_compute_object_hash(
+ PlasmaClient* conn, ObjectID object_id, unsigned char* digest) {
+ // Get the plasma object data. We pass in a timeout of 0 to indicate that
+ // the operation should timeout immediately.
+ ObjectBuffer object_buffer;
+ ARROW_CHECK_OK(conn->Get(&object_id, 1, 0, &object_buffer));
+ // If the object was not retrieved, return false.
+ if (object_buffer.data_size == -1) { return false; }
+ // Compute the hash.
+ uint64_t hash = compute_object_hash(object_buffer);
+ memcpy(digest, &hash, sizeof(hash));
+ // Release the plasma object.
+ ARROW_CHECK_OK(conn->Release(object_id));
+ return true;
+}
+
+Status PlasmaClient::Seal(const ObjectID& object_id) {
+ // Make sure this client has a reference to the object before sending the
+ // request to Plasma.
+ auto object_entry = objects_in_use_.find(object_id);
+ ARROW_CHECK(object_entry != objects_in_use_.end())
+ << "Plasma client called seal an object without a reference to it";
+ ARROW_CHECK(!object_entry->second->is_sealed)
+ << "Plasma client called seal an already sealed object";
+ object_entry->second->is_sealed = true;
+ /// Send the seal request to Plasma.
+ static unsigned char digest[kDigestSize];
+ ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0]));
+ RETURN_NOT_OK(SendSealRequest(store_conn_, object_id, &digest[0]));
+ // We call PlasmaClient::Release to decrement the number of instances of this
+ // object
+ // that are currently being used by this client. The corresponding increment
+ // happened in plasma_create and was used to ensure that the object was not
+ // released before the call to PlasmaClient::Seal.
+ return Release(object_id);
+}
+
+Status PlasmaClient::Delete(const ObjectID& object_id) {
+ // TODO(rkn): In the future, we can use this method to give hints to the
+ // eviction policy about when an object will no longer be needed.
+ return Status::NotImplemented("PlasmaClient::Delete is not implemented.");
+}
+
+Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) {
+ // Send a request to the store to evict objects.
+ RETURN_NOT_OK(SendEvictRequest(store_conn_, num_bytes));
+ // Wait for a response with the number of bytes actually evicted.
+ std::vector<uint8_t> buffer;
+ int64_t type;
+ RETURN_NOT_OK(ReadMessage(store_conn_, &type, &buffer));
+ return ReadEvictReply(buffer.data(), num_bytes_evicted);
+}
+
+Status PlasmaClient::Subscribe(int* fd) {
+ int sock[2];
+ // Create a non-blocking socket pair. This will only be used to send
+ // notifications from the Plasma store to the client.
+ socketpair(AF_UNIX, SOCK_STREAM, 0, sock);
+ // Make the socket non-blocking.
+ int flags = fcntl(sock[1], F_GETFL, 0);
+ ARROW_CHECK(fcntl(sock[1], F_SETFL, flags | O_NONBLOCK) == 0);
+ // Tell the Plasma store about the subscription.
+ RETURN_NOT_OK(SendSubscribeRequest(store_conn_));
+ // Send the file descriptor that the Plasma store should use to push
+ // notifications about sealed objects to this client.
+ ARROW_CHECK(send_fd(store_conn_, sock[1]) >= 0);
+ close(sock[1]);
+ // Return the file descriptor that the client should use to read notifications
+ // about sealed objects.
+ *fd = sock[0];
+ return Status::OK();
+}
+
+Status PlasmaClient::Connect(const std::string& store_socket_name,
+ const std::string& manager_socket_name, int release_delay) {
+ store_conn_ = connect_ipc_sock_retry(store_socket_name, -1, -1);
+ if (manager_socket_name != "") {
+ manager_conn_ = connect_ipc_sock_retry(manager_socket_name, -1, -1);
+ } else {
+ manager_conn_ = -1;
+ }
+ config_.release_delay = release_delay;
+ in_use_object_bytes_ = 0;
+ // Send a ConnectRequest to the store to get its memory capacity.
+ RETURN_NOT_OK(SendConnectRequest(store_conn_));
+ std::vector<uint8_t> buffer;
+ RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaConnectReply, &buffer));
+ RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity_));
+ return Status::OK();
+}
+
+Status PlasmaClient::Disconnect() {
+ // NOTE: We purposefully do not finish sending release calls for objects in
+ // use, so that we don't duplicate PlasmaClient::Release calls (when handling
+ // a SIGTERM, for example).
+
+ // Close the connections to Plasma. The Plasma store will release the objects
+ // that were in use by us when handling the SIGPIPE.
+ close(store_conn_);
+ if (manager_conn_ >= 0) { close(manager_conn_); }
+ return Status::OK();
+}
+
+#define h_addr h_addr_list[0]
+
+Status PlasmaClient::Transfer(const char* address, int port, const ObjectID& object_id) {
+ return SendDataRequest(manager_conn_, object_id, address, port);
+}
+
+Status PlasmaClient::Fetch(int num_object_ids, const ObjectID* object_ids) {
+ ARROW_CHECK(manager_conn_ >= 0);
+ return SendFetchRequest(manager_conn_, object_ids, num_object_ids);
+}
+
+int PlasmaClient::get_manager_fd() {
+ return manager_conn_;
+}
+
+Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) {
+ ARROW_CHECK(manager_conn_ >= 0);
+
+ RETURN_NOT_OK(SendStatusRequest(manager_conn_, &object_id, 1));
+ std::vector<uint8_t> buffer;
+ RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaStatusReply, &buffer));
+ ObjectID id;
+ RETURN_NOT_OK(ReadStatusReply(buffer.data(), &id, object_status, 1));
+ ARROW_CHECK(object_id == id);
+ return Status::OK();
+}
+
+Status PlasmaClient::Wait(int64_t num_object_requests, ObjectRequest* object_requests,
+ int num_ready_objects, int64_t timeout_ms, int* num_objects_ready) {
+ ARROW_CHECK(manager_conn_ >= 0);
+ ARROW_CHECK(num_object_requests > 0);
+ ARROW_CHECK(num_ready_objects > 0);
+ ARROW_CHECK(num_ready_objects <= num_object_requests);
+
+ for (int i = 0; i < num_object_requests; ++i) {
+ ARROW_CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL ||
+ object_requests[i].type == PLASMA_QUERY_ANYWHERE);
+ }
+
+ RETURN_NOT_OK(SendWaitRequest(manager_conn_, object_requests, num_object_requests,
+ num_ready_objects, timeout_ms));
+ std::vector<uint8_t> buffer;
+ RETURN_NOT_OK(PlasmaReceive(manager_conn_, MessageType_PlasmaWaitReply, &buffer));
+ RETURN_NOT_OK(ReadWaitReply(buffer.data(), object_requests, &num_ready_objects));
+
+ *num_objects_ready = 0;
+ for (int i = 0; i < num_object_requests; ++i) {
+ int type = object_requests[i].type;
+ int status = object_requests[i].status;
+ switch (type) {
+ case PLASMA_QUERY_LOCAL:
+ if (status == ObjectStatus_Local) { *num_objects_ready += 1; }
+ break;
+ case PLASMA_QUERY_ANYWHERE:
+ if (status == ObjectStatus_Local || status == ObjectStatus_Remote) {
+ *num_objects_ready += 1;
+ } else {
+ ARROW_CHECK(status == ObjectStatus_Nonexistent);
+ }
+ break;
+ default:
+ ARROW_LOG(FATAL) << "This code should be unreachable.";
+ }
+ }
+ return Status::OK();
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/client.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h
new file mode 100644
index 0000000..fb3a161
--- /dev/null
+++ b/cpp/src/plasma/client.h
@@ -0,0 +1,343 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_CLIENT_H
+#define PLASMA_CLIENT_H
+
+#include <stdbool.h>
+#include <time.h>
+
+#include <deque>
+#include <string>
+
+#include "plasma/plasma.h"
+
+using arrow::Status;
+
+#define PLASMA_DEFAULT_RELEASE_DELAY 64
+
+// Use 100MB as an overestimate of the L3 cache size.
+constexpr int64_t kL3CacheSizeBytes = 100000000;
+
+/// Object buffer data structure.
+struct ObjectBuffer {
+ /// The size in bytes of the data object.
+ int64_t data_size;
+ /// The address of the data object.
+ uint8_t* data;
+ /// The metadata size in bytes.
+ int64_t metadata_size;
+ /// The address of the metadata.
+ uint8_t* metadata;
+};
+
+/// Configuration options for the plasma client.
+struct PlasmaClientConfig {
+ /// Number of release calls we wait until the object is actually released.
+ /// This allows us to avoid invalidating the cpu cache on workers if objects
+ /// are reused accross tasks.
+ size_t release_delay;
+};
+
+struct ClientMmapTableEntry {
+ /// The result of mmap for this file descriptor.
+ uint8_t* pointer;
+ /// The length of the memory-mapped file.
+ size_t length;
+ /// The number of objects in this memory-mapped file that are currently being
+ /// used by the client. When this count reaches zeros, we unmap the file.
+ int count;
+};
+
+struct ObjectInUseEntry {
+ /// A count of the number of times this client has called PlasmaClient::Create
+ /// or
+ /// PlasmaClient::Get on this object ID minus the number of calls to
+ /// PlasmaClient::Release.
+ /// When this count reaches zero, we remove the entry from the ObjectsInUse
+ /// and decrement a count in the relevant ClientMmapTableEntry.
+ int count;
+ /// Cached information to read the object.
+ PlasmaObject object;
+ /// A flag representing whether the object has been sealed.
+ bool is_sealed;
+};
+
+class PlasmaClient {
+ public:
+ /// Connect to the local plasma store and plasma manager. Return
+ /// the resulting connection.
+ ///
+ /// @param store_socket_name The name of the UNIX domain socket to use to
+ /// connect to the Plasma store.
+ /// @param manager_socket_name The name of the UNIX domain socket to use to
+ /// connect to the local Plasma manager. If this is "", then this
+ /// function will not connect to a manager.
+ /// @param release_delay Number of released objects that are kept around
+ /// and not evicted to avoid too many munmaps.
+ /// @return The return status.
+ Status Connect(const std::string& store_socket_name,
+ const std::string& manager_socket_name, int release_delay);
+
+ /// Create an object in the Plasma Store. Any metadata for this object must be
+ /// be passed in when the object is created.
+ ///
+ /// @param object_id The ID to use for the newly created object.
+ /// @param data_size The size in bytes of the space to be allocated for this
+ /// object's
+ /// data (this does not include space used for metadata).
+ /// @param metadata The object's metadata. If there is no metadata, this
+ /// pointer
+ /// should be NULL.
+ /// @param metadata_size The size in bytes of the metadata. If there is no
+ /// metadata, this should be 0.
+ /// @param data The address of the newly created object will be written here.
+ /// @return The return status.
+ Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata,
+ int64_t metadata_size, uint8_t** data);
+
+ /// Get some objects from the Plasma Store. This function will block until the
+ /// objects have all been created and sealed in the Plasma Store or the
+ /// timeout
+ /// expires. The caller is responsible for releasing any retrieved objects,
+ /// but
+ /// the caller should not release objects that were not retrieved.
+ ///
+ /// @param object_ids The IDs of the objects to get.
+ /// @param num_object_ids The number of object IDs to get.
+ /// @param timeout_ms The amount of time in milliseconds to wait before this
+ /// request times out. If this value is -1, then no timeout is set.
+ /// @param object_buffers An array where the results will be stored. If the
+ /// data
+ /// size field is -1, then the object was not retrieved.
+ /// @return The return status.
+ Status Get(const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms,
+ ObjectBuffer* object_buffers);
+
+ /// Tell Plasma that the client no longer needs the object. This should be
+ /// called
+ /// after Get when the client is done with the object. After this call,
+ /// the address returned by Get is no longer valid. This should be called
+ /// once for each call to Get (with the same object ID).
+ ///
+ /// @param object_id The ID of the object that is no longer needed.
+ /// @return The return status.
+ Status Release(const ObjectID& object_id);
+
+ /// Check if the object store contains a particular object and the object has
+ /// been sealed. The result will be stored in has_object.
+ ///
+ /// @todo: We may want to indicate if the object has been created but not
+ /// sealed.
+ ///
+ /// @param object_id The ID of the object whose presence we are checking.
+ /// @param has_object The function will write true at this address if
+ /// the object is present and false if it is not present.
+ /// @return The return status.
+ Status Contains(const ObjectID& object_id, bool* has_object);
+
+ /// Seal an object in the object store. The object will be immutable after
+ /// this
+ /// call.
+ ///
+ /// @param object_id The ID of the object to seal.
+ /// @return The return status.
+ Status Seal(const ObjectID& object_id);
+
+ /// Delete an object from the object store. This currently assumes that the
+ /// object is present and has been sealed.
+ ///
+ /// @todo We may want to allow the deletion of objects that are not present or
+ /// haven't been sealed.
+ ///
+ /// @param object_id The ID of the object to delete.
+ /// @return The return status.
+ Status Delete(const ObjectID& object_id);
+
+ /// Delete objects until we have freed up num_bytes bytes or there are no more
+ /// released objects that can be deleted.
+ ///
+ /// @param num_bytes The number of bytes to try to free up.
+ /// @param num_bytes_evicted Out parameter for total number of bytes of space
+ /// retrieved.
+ /// @return The return status.
+ Status Evict(int64_t num_bytes, int64_t& num_bytes_evicted);
+
+ /// Subscribe to notifications when objects are sealed in the object store.
+ /// Whenever an object is sealed, a message will be written to the client
+ /// socket
+ /// that is returned by this method.
+ ///
+ /// @param fd Out parameter for the file descriptor the client should use to
+ /// read notifications
+ /// from the object store about sealed objects.
+ /// @return The return status.
+ Status Subscribe(int* fd);
+
+ /// Disconnect from the local plasma instance, including the local store and
+ /// manager.
+ ///
+ /// @return The return status.
+ Status Disconnect();
+
+ /// Attempt to initiate the transfer of some objects from remote Plasma
+ /// Stores.
+ /// This method does not guarantee that the fetched objects will arrive
+ /// locally.
+ ///
+ /// For an object that is available in the local Plasma Store, this method
+ /// will
+ /// not do anything. For an object that is not available locally, it will
+ /// check
+ /// if the object are already being fetched. If so, it will not do anything.
+ /// If
+ /// not, it will query the object table for a list of Plasma Managers that
+ /// have
+ /// the object. The object table will return a non-empty list, and this Plasma
+ /// Manager will attempt to initiate transfers from one of those Plasma
+ /// Managers.
+ ///
+ /// This function is non-blocking.
+ ///
+ /// This method is idempotent in the sense that it is ok to call it multiple
+ /// times.
+ ///
+ /// @param num_object_ids The number of object IDs fetch is being called on.
+ /// @param object_ids The IDs of the objects that fetch is being called on.
+ /// @return The return status.
+ Status Fetch(int num_object_ids, const ObjectID* object_ids);
+
+ /// Wait for (1) a specified number of objects to be available (sealed) in the
+ /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to
+ /// expire. This is a blocking call.
+ ///
+ /// @param num_object_requests Size of the object_requests array.
+ /// @param object_requests Object event array. Each element contains a request
+ /// for a particular object_id. The type of request is specified in the
+ /// "type" field.
+ /// - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes
+ /// available in the local Plasma Store. In this case, this function
+ /// sets the "status" field to ObjectStatus_Local. Note, if the
+ /// status
+ /// is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent,
+ /// but it may exist elsewhere in the system.
+ /// - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id
+ /// becomes
+ /// available either at the local Plasma Store or on a remote Plasma
+ /// Store. In this case, the functions sets the "status" field to
+ /// ObjectStatus_Local or ObjectStatus_Remote.
+ /// @param num_ready_objects The number of requests in object_requests array
+ /// that
+ /// must be satisfied before the function returns, unless it timeouts.
+ /// The num_ready_objects should be no larger than num_object_requests.
+ /// @param timeout_ms Timeout value in milliseconds. If this timeout expires
+ /// before min_num_ready_objects of requests are satisfied, the
+ /// function
+ /// returns.
+ /// @param num_objects_ready Out parameter for number of satisfied requests in
+ /// the object_requests list. If the returned number is less than
+ /// min_num_ready_objects this means that timeout expired.
+ /// @return The return status.
+ Status Wait(int64_t num_object_requests, ObjectRequest* object_requests,
+ int num_ready_objects, int64_t timeout_ms, int* num_objects_ready);
+
+ /// Transfer local object to a different plasma manager.
+ ///
+ /// @param conn The object containing the connection state.
+ /// @param addr IP address of the plasma manager we are transfering to.
+ /// @param port Port of the plasma manager we are transfering to.
+ /// @object_id ObjectID of the object we are transfering.
+ /// @return The return status.
+ Status Transfer(const char* addr, int port, const ObjectID& object_id);
+
+ /// Return the status of a given object. This method may query the object
+ /// table.
+ ///
+ /// @param conn The object containing the connection state.
+ /// @param object_id The ID of the object whose status we query.
+ /// @param object_status Out parameter for object status. Can take the
+ /// following values.
+ /// - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma
+ /// Store.
+ /// has been already scheduled by the Plasma Manager.
+ /// - PLASMA_CLIENT_TRANSFER, if the object is either currently being
+ /// transferred or just scheduled.
+ /// - PLASMA_CLIENT_REMOTE, if the object is stored at a remote
+ /// Plasma Store.
+ /// - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the
+ /// system.
+ /// @return The return status.
+ Status Info(const ObjectID& object_id, int* object_status);
+
+ /// Get the file descriptor for the socket connection to the plasma manager.
+ ///
+ /// @param conn The plasma connection.
+ /// @return The file descriptor for the manager connection. If there is no
+ /// connection to the manager, this is -1.
+ int get_manager_fd();
+
+ private:
+ Status PerformRelease(const ObjectID& object_id);
+
+ uint8_t* lookup_or_mmap(int fd, int store_fd_val, int64_t map_size);
+
+ uint8_t* lookup_mmapped_file(int store_fd_val);
+
+ void increment_object_count(
+ const ObjectID& object_id, PlasmaObject* object, bool is_sealed);
+
+ /// File descriptor of the Unix domain socket that connects to the store.
+ int store_conn_;
+ /// File descriptor of the Unix domain socket that connects to the manager.
+ int manager_conn_;
+ /// Table of dlmalloc buffer files that have been memory mapped so far. This
+ /// is a hash table mapping a file descriptor to a struct containing the
+ /// address of the corresponding memory-mapped file.
+ std::unordered_map<int, ClientMmapTableEntry> mmap_table_;
+ /// A hash table of the object IDs that are currently being used by this
+ /// client.
+ std::unordered_map<ObjectID, std::unique_ptr<ObjectInUseEntry>, UniqueIDHasher>
+ objects_in_use_;
+ /// Object IDs of the last few release calls. This is a deque and
+ /// is used to delay releasing objects to see if they can be reused by
+ /// subsequent tasks so we do not unneccessarily invalidate cpu caches.
+ /// TODO(pcm): replace this with a proper lru cache using the size of the L3
+ /// cache.
+ std::deque<ObjectID> release_history_;
+ /// The number of bytes in the combined objects that are held in the release
+ /// history doubly-linked list. If this is too large then the client starts
+ /// releasing objects.
+ int64_t in_use_object_bytes_;
+ /// Configuration options for the plasma client.
+ PlasmaClientConfig config_;
+ /// The amount of memory available to the Plasma store. The client needs this
+ /// information to make sure that it does not delay in releasing so much
+ /// memory that the store is unable to evict enough objects to free up space.
+ int64_t store_capacity_;
+};
+
+/// Compute the hash of an object in the object store.
+///
+/// @param conn The object containing the connection state.
+/// @param object_id The ID of the object we want to hash.
+/// @param digest A pointer at which to return the hash digest of the object.
+/// The pointer must have at least DIGEST_SIZE bytes allocated.
+/// @return A boolean representing whether the hash operation succeeded.
+bool plasma_compute_object_hash(
+ PlasmaClient* conn, ObjectID object_id, unsigned char* digest);
+
+#endif // PLASMA_CLIENT_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/common.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/common.cc b/cpp/src/plasma/common.cc
new file mode 100644
index 0000000..a09a963
--- /dev/null
+++ b/cpp/src/plasma/common.cc
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/common.h"
+
+#include <random>
+
+#include "format/plasma_generated.h"
+
+using arrow::Status;
+
+UniqueID UniqueID::from_random() {
+ UniqueID id;
+ uint8_t* data = id.mutable_data();
+ std::random_device engine;
+ for (int i = 0; i < kUniqueIDSize; i++) {
+ data[i] = static_cast<uint8_t>(engine());
+ }
+ return id;
+}
+
+UniqueID UniqueID::from_binary(const std::string& binary) {
+ UniqueID id;
+ std::memcpy(&id, binary.data(), sizeof(id));
+ return id;
+}
+
+const uint8_t* UniqueID::data() const {
+ return id_;
+}
+
+uint8_t* UniqueID::mutable_data() {
+ return id_;
+}
+
+std::string UniqueID::binary() const {
+ return std::string(reinterpret_cast<const char*>(id_), kUniqueIDSize);
+}
+
+std::string UniqueID::hex() const {
+ constexpr char hex[] = "0123456789abcdef";
+ std::string result;
+ for (int i = 0; i < kUniqueIDSize; i++) {
+ unsigned int val = id_[i];
+ result.push_back(hex[val >> 4]);
+ result.push_back(hex[val & 0xf]);
+ }
+ return result;
+}
+
+bool UniqueID::operator==(const UniqueID& rhs) const {
+ return std::memcmp(data(), rhs.data(), kUniqueIDSize) == 0;
+}
+
+Status plasma_error_status(int plasma_error) {
+ switch (plasma_error) {
+ case PlasmaError_OK:
+ return Status::OK();
+ case PlasmaError_ObjectExists:
+ return Status::PlasmaObjectExists("object already exists in the plasma store");
+ case PlasmaError_ObjectNonexistent:
+ return Status::PlasmaObjectNonexistent("object does not exist in the plasma store");
+ case PlasmaError_OutOfMemory:
+ return Status::PlasmaStoreFull("object does not fit in the plasma store");
+ default:
+ ARROW_LOG(FATAL) << "unknown plasma error code " << plasma_error;
+ }
+ return Status::OK();
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/common.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/common.h b/cpp/src/plasma/common.h
new file mode 100644
index 0000000..85dc74b
--- /dev/null
+++ b/cpp/src/plasma/common.h
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_COMMON_H
+#define PLASMA_COMMON_H
+
+#include <cstring>
+#include <string>
+// TODO(pcm): Convert getopt and sscanf in the store to use more idiomatic C++
+// and get rid of the next three lines:
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+constexpr int64_t kUniqueIDSize = 20;
+
+class UniqueID {
+ public:
+ static UniqueID from_random();
+ static UniqueID from_binary(const std::string& binary);
+ bool operator==(const UniqueID& rhs) const;
+ const uint8_t* data() const;
+ uint8_t* mutable_data();
+ std::string binary() const;
+ std::string hex() const;
+
+ private:
+ uint8_t id_[kUniqueIDSize];
+};
+
+static_assert(std::is_pod<UniqueID>::value, "UniqueID must be plain old data");
+
+struct UniqueIDHasher {
+ // ObjectID hashing function.
+ size_t operator()(const UniqueID& id) const {
+ size_t result;
+ std::memcpy(&result, id.data(), sizeof(size_t));
+ return result;
+ }
+};
+
+typedef UniqueID ObjectID;
+
+arrow::Status plasma_error_status(int plasma_error);
+
+#endif // PLASMA_COMMON_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/events.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/events.cc b/cpp/src/plasma/events.cc
new file mode 100644
index 0000000..a9f7356
--- /dev/null
+++ b/cpp/src/plasma/events.cc
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/events.h"
+
+#include <errno.h>
+
+void EventLoop::file_event_callback(
+ aeEventLoop* loop, int fd, void* context, int events) {
+ FileCallback* callback = reinterpret_cast<FileCallback*>(context);
+ (*callback)(events);
+}
+
+int EventLoop::timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context) {
+ TimerCallback* callback = reinterpret_cast<TimerCallback*>(context);
+ return (*callback)(timer_id);
+}
+
+constexpr int kInitialEventLoopSize = 1024;
+
+EventLoop::EventLoop() {
+ loop_ = aeCreateEventLoop(kInitialEventLoopSize);
+}
+
+bool EventLoop::add_file_event(int fd, int events, const FileCallback& callback) {
+ if (file_callbacks_.find(fd) != file_callbacks_.end()) { return false; }
+ auto data = std::unique_ptr<FileCallback>(new FileCallback(callback));
+ void* context = reinterpret_cast<void*>(data.get());
+ // Try to add the file descriptor.
+ int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
+ // If it cannot be added, increase the size of the event loop.
+ if (err == AE_ERR && errno == ERANGE) {
+ err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2);
+ if (err != AE_OK) { return false; }
+ err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context);
+ }
+ // In any case, test if there were errors.
+ if (err == AE_OK) {
+ file_callbacks_.emplace(fd, std::move(data));
+ return true;
+ }
+ return false;
+}
+
+void EventLoop::remove_file_event(int fd) {
+ aeDeleteFileEvent(loop_, fd, AE_READABLE | AE_WRITABLE);
+ file_callbacks_.erase(fd);
+}
+
+void EventLoop::run() {
+ aeMain(loop_);
+}
+
+int64_t EventLoop::add_timer(int64_t timeout, const TimerCallback& callback) {
+ auto data = std::unique_ptr<TimerCallback>(new TimerCallback(callback));
+ void* context = reinterpret_cast<void*>(data.get());
+ int64_t timer_id =
+ aeCreateTimeEvent(loop_, timeout, EventLoop::timer_event_callback, context, NULL);
+ timer_callbacks_.emplace(timer_id, std::move(data));
+ return timer_id;
+}
+
+int EventLoop::remove_timer(int64_t timer_id) {
+ int err = aeDeleteTimeEvent(loop_, timer_id);
+ timer_callbacks_.erase(timer_id);
+ return err;
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/events.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/events.h b/cpp/src/plasma/events.h
new file mode 100644
index 0000000..bd93d6b
--- /dev/null
+++ b/cpp/src/plasma/events.h
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EVENTS
+#define PLASMA_EVENTS
+
+#include <functional>
+#include <memory>
+#include <unordered_map>
+
+extern "C" {
+#include "ae/ae.h"
+}
+
+/// Constant specifying that the timer is done and it will be removed.
+constexpr int kEventLoopTimerDone = AE_NOMORE;
+
+/// Read event on the file descriptor.
+constexpr int kEventLoopRead = AE_READABLE;
+
+/// Write event on the file descriptor.
+constexpr int kEventLoopWrite = AE_WRITABLE;
+
+typedef long long TimerID; // NOLINT
+
+class EventLoop {
+ public:
+ // Signature of the handler that will be called when there is a new event
+ // on the file descriptor that this handler has been registered for.
+ //
+ // The arguments are the event flags (read or write).
+ using FileCallback = std::function<void(int)>;
+
+ // This handler will be called when a timer times out. The timer id is
+ // passed as an argument. The return is the number of milliseconds the timer
+ // shall be reset to or kEventLoopTimerDone if the timer shall not be
+ // triggered again.
+ using TimerCallback = std::function<int(int64_t)>;
+
+ EventLoop();
+
+ /// Add a new file event handler to the event loop.
+ ///
+ /// @param fd The file descriptor we are listening to.
+ /// @param events The flags for events we are listening to (read or write).
+ /// @param callback The callback that will be called when the event happens.
+ /// @return Returns true if the event handler was added successfully.
+ bool add_file_event(int fd, int events, const FileCallback& callback);
+
+ /// Remove a file event handler from the event loop.
+ ///
+ /// @param fd The file descriptor of the event handler.
+ /// @return Void.
+ void remove_file_event(int fd);
+
+ /// Register a handler that will be called after a time slice of
+ /// "timeout" milliseconds.
+ ///
+ /// @param timeout The timeout in milliseconds.
+ /// @param callback The callback for the timeout.
+ /// @return The ID of the newly created timer.
+ int64_t add_timer(int64_t timeout, const TimerCallback& callback);
+
+ /// Remove a timer handler from the event loop.
+ ///
+ /// @param timer_id The ID of the timer that is to be removed.
+ /// @return The ae.c error code. TODO(pcm): needs to be standardized
+ int remove_timer(int64_t timer_id);
+
+ /// Run the event loop.
+ ///
+ /// @return Void.
+ void run();
+
+ private:
+ static void file_event_callback(aeEventLoop* loop, int fd, void* context, int events);
+
+ static int timer_event_callback(aeEventLoop* loop, TimerID timer_id, void* context);
+
+ aeEventLoop* loop_;
+ std::unordered_map<int, std::unique_ptr<FileCallback>> file_callbacks_;
+ std::unordered_map<int64_t, std::unique_ptr<TimerCallback>> timer_callbacks_;
+};
+
+#endif // PLASMA_EVENTS
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/eviction_policy.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc
new file mode 100644
index 0000000..4ae6384
--- /dev/null
+++ b/cpp/src/plasma/eviction_policy.cc
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/eviction_policy.h"
+
+#include <algorithm>
+
+void LRUCache::add(const ObjectID& key, int64_t size) {
+ auto it = item_map_.find(key);
+ ARROW_CHECK(it == item_map_.end());
+ /* Note that it is important to use a list so the iterators stay valid. */
+ item_list_.emplace_front(key, size);
+ item_map_.emplace(key, item_list_.begin());
+}
+
+void LRUCache::remove(const ObjectID& key) {
+ auto it = item_map_.find(key);
+ ARROW_CHECK(it != item_map_.end());
+ item_list_.erase(it->second);
+ item_map_.erase(it);
+}
+
+int64_t LRUCache::choose_objects_to_evict(
+ int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
+ int64_t bytes_evicted = 0;
+ auto it = item_list_.end();
+ while (bytes_evicted < num_bytes_required && it != item_list_.begin()) {
+ it--;
+ objects_to_evict->push_back(it->first);
+ bytes_evicted += it->second;
+ }
+ return bytes_evicted;
+}
+
+EvictionPolicy::EvictionPolicy(PlasmaStoreInfo* store_info)
+ : memory_used_(0), store_info_(store_info) {}
+
+int64_t EvictionPolicy::choose_objects_to_evict(
+ int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict) {
+ int64_t bytes_evicted =
+ cache_.choose_objects_to_evict(num_bytes_required, objects_to_evict);
+ /* Update the LRU cache. */
+ for (auto& object_id : *objects_to_evict) {
+ cache_.remove(object_id);
+ }
+ /* Update the number of bytes used. */
+ memory_used_ -= bytes_evicted;
+ return bytes_evicted;
+}
+
+void EvictionPolicy::object_created(const ObjectID& object_id) {
+ auto entry = store_info_->objects[object_id].get();
+ cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
+}
+
+bool EvictionPolicy::require_space(
+ int64_t size, std::vector<ObjectID>* objects_to_evict) {
+ /* Check if there is enough space to create the object. */
+ int64_t required_space = memory_used_ + size - store_info_->memory_capacity;
+ int64_t num_bytes_evicted;
+ if (required_space > 0) {
+ /* Try to free up at least as much space as we need right now but ideally
+ * up to 20% of the total capacity. */
+ int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5);
+ ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects";
+ /* Choose some objects to evict, and update the return pointers. */
+ num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict);
+ ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting "
+ << objects_to_evict->size() << " objects to free up "
+ << num_bytes_evicted << " bytes.";
+ } else {
+ num_bytes_evicted = 0;
+ }
+ if (num_bytes_evicted >= required_space) {
+ /* We only increment the space used if there is enough space to create the
+ * object. */
+ memory_used_ += size;
+ }
+ return num_bytes_evicted >= required_space;
+}
+
+void EvictionPolicy::begin_object_access(
+ const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
+ /* If the object is in the LRU cache, remove it. */
+ cache_.remove(object_id);
+}
+
+void EvictionPolicy::end_object_access(
+ const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict) {
+ auto entry = store_info_->objects[object_id].get();
+ /* Add the object to the LRU cache.*/
+ cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/eviction_policy.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h
new file mode 100644
index 0000000..3815fc6
--- /dev/null
+++ b/cpp/src/plasma/eviction_policy.h
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EVICTION_POLICY_H
+#define PLASMA_EVICTION_POLICY_H
+
+#include <list>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/plasma.h"
+
+// ==== The eviction policy ====
+//
+// This file contains declaration for all functions and data structures that
+// need to be provided if you want to implement a new eviction algorithm for the
+// Plasma store.
+
+class LRUCache {
+ public:
+ LRUCache() {}
+
+ void add(const ObjectID& key, int64_t size);
+
+ void remove(const ObjectID& key);
+
+ int64_t choose_objects_to_evict(
+ int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
+
+ private:
+ /// A doubly-linked list containing the items in the cache and
+ /// their sizes in LRU order.
+ typedef std::list<std::pair<ObjectID, int64_t>> ItemList;
+ ItemList item_list_;
+ /// A hash table mapping the object ID of an object in the cache to its
+ /// location in the doubly linked list item_list_.
+ std::unordered_map<ObjectID, ItemList::iterator, UniqueIDHasher> item_map_;
+};
+
+/// The eviction policy.
+class EvictionPolicy {
+ public:
+ /// Construct an eviction policy.
+ ///
+ /// @param store_info Information about the Plasma store that is exposed
+ /// to the eviction policy.
+ explicit EvictionPolicy(PlasmaStoreInfo* store_info);
+
+ /// This method will be called whenever an object is first created in order to
+ /// add it to the LRU cache. This is done so that the first time, the Plasma
+ /// store calls begin_object_access, we can remove the object from the LRU
+ /// cache.
+ ///
+ /// @param object_id The object ID of the object that was created.
+ /// @return Void.
+ void object_created(const ObjectID& object_id);
+
+ /// This method will be called when the Plasma store needs more space, perhaps
+ /// to create a new object. If the required amount of space cannot be freed up,
+ /// then a fatal error will be thrown. When this method is called, the eviction
+ /// policy will assume that the objects chosen to be evicted will in fact be
+ /// evicted from the Plasma store by the caller.
+ ///
+ /// @param size The size in bytes of the new object, including both data and
+ /// metadata.
+ /// @param objects_to_evict The object IDs that were chosen for eviction will
+ /// be stored into this vector.
+ /// @return True if enough space can be freed and false otherwise.
+ bool require_space(int64_t size, std::vector<ObjectID>* objects_to_evict);
+
+ /// This method will be called whenever an unused object in the Plasma store
+ /// starts to be used. When this method is called, the eviction policy will
+ /// assume that the objects chosen to be evicted will in fact be evicted from
+ /// the Plasma store by the caller.
+ ///
+ /// @param object_id The ID of the object that is now being used.
+ /// @param objects_to_evict The object IDs that were chosen for eviction will
+ /// be stored into this vector.
+ /// @return Void.
+ void begin_object_access(
+ const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
+
+ /// This method will be called whenever an object in the Plasma store that was
+ /// being used is no longer being used. When this method is called, the
+ /// eviction policy will assume that the objects chosen to be evicted will in
+ /// fact be evicted from the Plasma store by the caller.
+ ///
+ /// @param object_id The ID of the object that is no longer being used.
+ /// @param objects_to_evict The object IDs that were chosen for eviction will
+ /// be stored into this vector.
+ /// @return Void.
+ void end_object_access(
+ const ObjectID& object_id, std::vector<ObjectID>* objects_to_evict);
+
+ /// Choose some objects to evict from the Plasma store. When this method is
+ /// called, the eviction policy will assume that the objects chosen to be
+ /// evicted will in fact be evicted from the Plasma store by the caller.
+ ///
+ /// @note This method is not part of the API. It is exposed in the header file
+ /// only for testing.
+ ///
+ /// @param num_bytes_required The number of bytes of space to try to free up.
+ /// @param objects_to_evict The object IDs that were chosen for eviction will
+ /// be stored into this vector.
+ /// @return The total number of bytes of space chosen to be evicted.
+ int64_t choose_objects_to_evict(
+ int64_t num_bytes_required, std::vector<ObjectID>* objects_to_evict);
+
+ private:
+ /// The amount of memory (in bytes) currently being used.
+ int64_t memory_used_;
+ /// Pointer to the plasma store info.
+ PlasmaStoreInfo* store_info_;
+ /// Datastructure for the LRU cache.
+ LRUCache cache_;
+};
+
+#endif // PLASMA_EVICTION_POLICY_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/extension.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/extension.cc b/cpp/src/plasma/extension.cc
new file mode 100644
index 0000000..5d61e33
--- /dev/null
+++ b/cpp/src/plasma/extension.cc
@@ -0,0 +1,456 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "plasma/extension.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+#include "plasma/io.h"
+#include "plasma/protocol.h"
+
+PyObject* PlasmaOutOfMemoryError;
+PyObject* PlasmaObjectExistsError;
+
+PyObject* PyPlasma_connect(PyObject* self, PyObject* args) {
+ const char* store_socket_name;
+ const char* manager_socket_name;
+ int release_delay;
+ if (!PyArg_ParseTuple(
+ args, "ssi", &store_socket_name, &manager_socket_name, &release_delay)) {
+ return NULL;
+ }
+ PlasmaClient* client = new PlasmaClient();
+ ARROW_CHECK_OK(client->Connect(store_socket_name, manager_socket_name, release_delay));
+
+ return PyCapsule_New(client, "plasma", NULL);
+}
+
+PyObject* PyPlasma_disconnect(PyObject* self, PyObject* args) {
+ PyObject* client_capsule;
+ if (!PyArg_ParseTuple(args, "O", &client_capsule)) { return NULL; }
+ PlasmaClient* client;
+ ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client));
+ ARROW_CHECK_OK(client->Disconnect());
+ /* We use the context of the connection capsule to indicate if the connection
+ * is still active (if the context is NULL) or if it is closed (if the context
+ * is (void*) 0x1). This is neccessary because the primary pointer of the
+ * capsule cannot be NULL. */
+ PyCapsule_SetContext(client_capsule, reinterpret_cast<void*>(0x1));
+ Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_create(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ ObjectID object_id;
+ Py_ssize_t size;
+ PyObject* metadata;
+ if (!PyArg_ParseTuple(args, "O&O&nO", PyObjectToPlasmaClient, &client,
+ PyStringToUniqueID, &object_id, &size, &metadata)) {
+ return NULL;
+ }
+ if (!PyByteArray_Check(metadata)) {
+ PyErr_SetString(PyExc_TypeError, "metadata must be a bytearray");
+ return NULL;
+ }
+ uint8_t* data;
+ Status s = client->Create(object_id, size,
+ reinterpret_cast<uint8_t*>(PyByteArray_AsString(metadata)),
+ PyByteArray_Size(metadata), &data);
+ if (s.IsPlasmaObjectExists()) {
+ PyErr_SetString(PlasmaObjectExistsError,
+ "An object with this ID already exists in the plasma "
+ "store.");
+ return NULL;
+ }
+ if (s.IsPlasmaStoreFull()) {
+ PyErr_SetString(PlasmaOutOfMemoryError,
+ "The plasma store ran out of memory and could not create "
+ "this object.");
+ return NULL;
+ }
+ ARROW_CHECK(s.ok());
+
+#if PY_MAJOR_VERSION >= 3
+ return PyMemoryView_FromMemory(reinterpret_cast<char*>(data), size, PyBUF_WRITE);
+#else
+ return PyBuffer_FromReadWriteMemory(reinterpret_cast<void*>(data), size);
+#endif
+}
+
+PyObject* PyPlasma_hash(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ ObjectID object_id;
+ if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+ &object_id)) {
+ return NULL;
+ }
+ unsigned char digest[kDigestSize];
+ bool success = plasma_compute_object_hash(client, object_id, digest);
+ if (success) {
+ PyObject* digest_string =
+ PyBytes_FromStringAndSize(reinterpret_cast<char*>(digest), kDigestSize);
+ return digest_string;
+ } else {
+ Py_RETURN_NONE;
+ }
+}
+
+PyObject* PyPlasma_seal(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ ObjectID object_id;
+ if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+ &object_id)) {
+ return NULL;
+ }
+ ARROW_CHECK_OK(client->Seal(object_id));
+ Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_release(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ ObjectID object_id;
+ if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+ &object_id)) {
+ return NULL;
+ }
+ ARROW_CHECK_OK(client->Release(object_id));
+ Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_get(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ PyObject* object_id_list;
+ Py_ssize_t timeout_ms;
+ if (!PyArg_ParseTuple(
+ args, "O&On", PyObjectToPlasmaClient, &client, &object_id_list, &timeout_ms)) {
+ return NULL;
+ }
+
+ Py_ssize_t num_object_ids = PyList_Size(object_id_list);
+ std::vector<ObjectID> object_ids(num_object_ids);
+ std::vector<ObjectBuffer> object_buffers(num_object_ids);
+
+ for (int i = 0; i < num_object_ids; ++i) {
+ PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
+ }
+
+ Py_BEGIN_ALLOW_THREADS;
+ ARROW_CHECK_OK(
+ client->Get(object_ids.data(), num_object_ids, timeout_ms, object_buffers.data()));
+ Py_END_ALLOW_THREADS;
+
+ PyObject* returns = PyList_New(num_object_ids);
+ for (int i = 0; i < num_object_ids; ++i) {
+ if (object_buffers[i].data_size != -1) {
+ /* The object was retrieved, so return the object. */
+ PyObject* t = PyTuple_New(2);
+ Py_ssize_t data_size = static_cast<Py_ssize_t>(object_buffers[i].data_size);
+ Py_ssize_t metadata_size = static_cast<Py_ssize_t>(object_buffers[i].metadata_size);
+#if PY_MAJOR_VERSION >= 3
+ char* data = reinterpret_cast<char*>(object_buffers[i].data);
+ char* metadata = reinterpret_cast<char*>(object_buffers[i].metadata);
+ PyTuple_SET_ITEM(t, 0, PyMemoryView_FromMemory(data, data_size, PyBUF_READ));
+ PyTuple_SET_ITEM(
+ t, 1, PyMemoryView_FromMemory(metadata, metadata_size, PyBUF_READ));
+#else
+ void* data = reinterpret_cast<void*>(object_buffers[i].data);
+ void* metadata = reinterpret_cast<void*>(object_buffers[i].metadata);
+ PyTuple_SET_ITEM(t, 0, PyBuffer_FromMemory(data, data_size));
+ PyTuple_SET_ITEM(t, 1, PyBuffer_FromMemory(metadata, metadata_size));
+#endif
+ ARROW_CHECK(PyList_SetItem(returns, i, t) == 0);
+ } else {
+ /* The object was not retrieved, so just add None to the list of return
+ * values. */
+ Py_INCREF(Py_None);
+ ARROW_CHECK(PyList_SetItem(returns, i, Py_None) == 0);
+ }
+ }
+ return returns;
+}
+
+PyObject* PyPlasma_contains(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ ObjectID object_id;
+ if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+ &object_id)) {
+ return NULL;
+ }
+ bool has_object;
+ ARROW_CHECK_OK(client->Contains(object_id, &has_object));
+
+ if (has_object) {
+ Py_RETURN_TRUE;
+ } else {
+ Py_RETURN_FALSE;
+ }
+}
+
+PyObject* PyPlasma_fetch(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ PyObject* object_id_list;
+ if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client, &object_id_list)) {
+ return NULL;
+ }
+ if (client->get_manager_fd() == -1) {
+ PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+ return NULL;
+ }
+ Py_ssize_t n = PyList_Size(object_id_list);
+ ObjectID* object_ids = new ObjectID[n];
+ for (int i = 0; i < n; ++i) {
+ PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
+ }
+ ARROW_CHECK_OK(client->Fetch(static_cast<int>(n), object_ids));
+ delete[] object_ids;
+ Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_wait(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ PyObject* object_id_list;
+ Py_ssize_t timeout;
+ int num_returns;
+ if (!PyArg_ParseTuple(args, "O&Oni", PyObjectToPlasmaClient, &client, &object_id_list,
+ &timeout, &num_returns)) {
+ return NULL;
+ }
+ Py_ssize_t n = PyList_Size(object_id_list);
+
+ if (client->get_manager_fd() == -1) {
+ PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+ return NULL;
+ }
+ if (num_returns < 0) {
+ PyErr_SetString(
+ PyExc_RuntimeError, "The argument num_returns cannot be less than zero.");
+ return NULL;
+ }
+ if (num_returns > n) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "The argument num_returns cannot be greater than len(object_ids)");
+ return NULL;
+ }
+ int64_t threshold = 1 << 30;
+ if (timeout > threshold) {
+ PyErr_SetString(
+ PyExc_RuntimeError, "The argument timeout cannot be greater than 2 ** 30.");
+ return NULL;
+ }
+
+ std::vector<ObjectRequest> object_requests(n);
+ for (int i = 0; i < n; ++i) {
+ ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i),
+ &object_requests[i].object_id) == 1);
+ object_requests[i].type = PLASMA_QUERY_ANYWHERE;
+ }
+ /* Drop the global interpreter lock while we are waiting, so other threads can
+ * run. */
+ int num_return_objects;
+ Py_BEGIN_ALLOW_THREADS;
+ ARROW_CHECK_OK(
+ client->Wait(n, object_requests.data(), num_returns, timeout, &num_return_objects));
+ Py_END_ALLOW_THREADS;
+
+ int num_to_return = std::min(num_return_objects, num_returns);
+ PyObject* ready_ids = PyList_New(num_to_return);
+ PyObject* waiting_ids = PySet_New(object_id_list);
+ int num_returned = 0;
+ for (int i = 0; i < n; ++i) {
+ if (num_returned == num_to_return) { break; }
+ if (object_requests[i].status == ObjectStatus_Local ||
+ object_requests[i].status == ObjectStatus_Remote) {
+ PyObject* ready = PyBytes_FromStringAndSize(
+ reinterpret_cast<char*>(&object_requests[i].object_id),
+ sizeof(object_requests[i].object_id));
+ PyList_SetItem(ready_ids, num_returned, ready);
+ PySet_Discard(waiting_ids, ready);
+ num_returned += 1;
+ } else {
+ ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent);
+ }
+ }
+ ARROW_CHECK(num_returned == num_to_return);
+ /* Return both the ready IDs and the remaining IDs. */
+ PyObject* t = PyTuple_New(2);
+ PyTuple_SetItem(t, 0, ready_ids);
+ PyTuple_SetItem(t, 1, waiting_ids);
+ return t;
+}
+
+PyObject* PyPlasma_evict(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ Py_ssize_t num_bytes;
+ if (!PyArg_ParseTuple(args, "O&n", PyObjectToPlasmaClient, &client, &num_bytes)) {
+ return NULL;
+ }
+ int64_t evicted_bytes;
+ ARROW_CHECK_OK(client->Evict(static_cast<int64_t>(num_bytes), evicted_bytes));
+ return PyLong_FromSsize_t(static_cast<Py_ssize_t>(evicted_bytes));
+}
+
+PyObject* PyPlasma_delete(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ ObjectID object_id;
+ if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID,
+ &object_id)) {
+ return NULL;
+ }
+ ARROW_CHECK_OK(client->Delete(object_id));
+ Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_transfer(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ ObjectID object_id;
+ const char* addr;
+ int port;
+ if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client,
+ PyStringToUniqueID, &object_id, &addr, &port)) {
+ return NULL;
+ }
+
+ if (client->get_manager_fd() == -1) {
+ PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
+ return NULL;
+ }
+
+ ARROW_CHECK_OK(client->Transfer(addr, port, object_id));
+ Py_RETURN_NONE;
+}
+
+PyObject* PyPlasma_subscribe(PyObject* self, PyObject* args) {
+ PlasmaClient* client;
+ if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) { return NULL; }
+
+ int sock;
+ ARROW_CHECK_OK(client->Subscribe(&sock));
+ return PyLong_FromLong(sock);
+}
+
+PyObject* PyPlasma_receive_notification(PyObject* self, PyObject* args) {
+ int plasma_sock;
+
+ if (!PyArg_ParseTuple(args, "i", &plasma_sock)) { return NULL; }
+ /* Receive object notification from the plasma connection socket. If the
+ * object was added, return a tuple of its fields: ObjectID, data_size,
+ * metadata_size. If the object was deleted, data_size and metadata_size will
+ * be set to -1. */
+ uint8_t* notification = read_message_async(plasma_sock);
+ if (notification == NULL) {
+ PyErr_SetString(
+ PyExc_RuntimeError, "Failed to read object notification from Plasma socket");
+ return NULL;
+ }
+ auto object_info = flatbuffers::GetRoot<ObjectInfo>(notification);
+ /* Construct a tuple from object_info and return. */
+ PyObject* t = PyTuple_New(3);
+ PyTuple_SetItem(t, 0, PyBytes_FromStringAndSize(object_info->object_id()->data(),
+ object_info->object_id()->size()));
+ if (object_info->is_deletion()) {
+ PyTuple_SetItem(t, 1, PyLong_FromLong(-1));
+ PyTuple_SetItem(t, 2, PyLong_FromLong(-1));
+ } else {
+ PyTuple_SetItem(t, 1, PyLong_FromLong(object_info->data_size()));
+ PyTuple_SetItem(t, 2, PyLong_FromLong(object_info->metadata_size()));
+ }
+
+ delete[] notification;
+ return t;
+}
+
+static PyMethodDef plasma_methods[] = {
+ {"connect", PyPlasma_connect, METH_VARARGS, "Connect to plasma."},
+ {"disconnect", PyPlasma_disconnect, METH_VARARGS, "Disconnect from plasma."},
+ {"create", PyPlasma_create, METH_VARARGS, "Create a new plasma object."},
+ {"hash", PyPlasma_hash, METH_VARARGS, "Compute the hash of a plasma object."},
+ {"seal", PyPlasma_seal, METH_VARARGS, "Seal a plasma object."},
+ {"get", PyPlasma_get, METH_VARARGS, "Get a plasma object."},
+ {"contains", PyPlasma_contains, METH_VARARGS,
+ "Does the plasma store contain this plasma object?"},
+ {"fetch", PyPlasma_fetch, METH_VARARGS,
+ "Fetch the object from another plasma manager instance."},
+ {"wait", PyPlasma_wait, METH_VARARGS,
+ "Wait until num_returns objects in object_ids are ready."},
+ {"evict", PyPlasma_evict, METH_VARARGS,
+ "Evict some objects until we recover some number of bytes."},
+ {"release", PyPlasma_release, METH_VARARGS, "Release the plasma object."},
+ {"delete", PyPlasma_delete, METH_VARARGS, "Delete a plasma object."},
+ {"transfer", PyPlasma_transfer, METH_VARARGS,
+ "Transfer object to another plasma manager."},
+ {"subscribe", PyPlasma_subscribe, METH_VARARGS,
+ "Subscribe to the plasma notification socket."},
+ {"receive_notification", PyPlasma_receive_notification, METH_VARARGS,
+ "Receive next notification from plasma notification socket."},
+ {NULL} /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+ PyModuleDef_HEAD_INIT, "libplasma", /* m_name */
+ "A Python client library for plasma.", /* m_doc */
+ 0, /* m_size */
+ plasma_methods, /* m_methods */
+ NULL, /* m_reload */
+ NULL, /* m_traverse */
+ NULL, /* m_clear */
+ NULL, /* m_free */
+};
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define INITERROR return NULL
+#else
+#define INITERROR return
+#endif
+
+#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
+#define PyMODINIT_FUNC void
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
+#else
+#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
+#endif
+
+MOD_INIT(libplasma) {
+#if PY_MAJOR_VERSION >= 3
+ PyObject* m = PyModule_Create(&moduledef);
+#else
+ PyObject* m =
+ Py_InitModule3("libplasma", plasma_methods, "A Python client library for plasma.");
+#endif
+
+ /* Create a custom exception for when an object ID is reused. */
+ char plasma_object_exists_error[] = "plasma_object_exists.error";
+ PlasmaObjectExistsError = PyErr_NewException(plasma_object_exists_error, NULL, NULL);
+ Py_INCREF(PlasmaObjectExistsError);
+ PyModule_AddObject(m, "plasma_object_exists_error", PlasmaObjectExistsError);
+ /* Create a custom exception for when the plasma store is out of memory. */
+ char plasma_out_of_memory_error[] = "plasma_out_of_memory.error";
+ PlasmaOutOfMemoryError = PyErr_NewException(plasma_out_of_memory_error, NULL, NULL);
+ Py_INCREF(PlasmaOutOfMemoryError);
+ PyModule_AddObject(m, "plasma_out_of_memory_error", PlasmaOutOfMemoryError);
+
+#if PY_MAJOR_VERSION >= 3
+ return m;
+#endif
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/extension.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/extension.h b/cpp/src/plasma/extension.h
new file mode 100644
index 0000000..cee30ab
--- /dev/null
+++ b/cpp/src/plasma/extension.h
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_EXTENSION_H
+#define PLASMA_EXTENSION_H
+
+#undef _XOPEN_SOURCE
+#undef _POSIX_C_SOURCE
+#include <Python.h>
+
+#include "bytesobject.h" // NOLINT
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+
+static int PyObjectToPlasmaClient(PyObject* object, PlasmaClient** client) {
+ if (PyCapsule_IsValid(object, "plasma")) {
+ *client = reinterpret_cast<PlasmaClient*>(PyCapsule_GetPointer(object, "plasma"));
+ return 1;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule");
+ return 0;
+ }
+}
+
+int PyStringToUniqueID(PyObject* object, ObjectID* object_id) {
+ if (PyBytes_Check(object)) {
+ memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID));
+ return 1;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "must be a 20 character string");
+ return 0;
+ }
+}
+
+#endif // PLASMA_EXTENSION_H
[12/14] arrow git commit: [C++] Restore Plasma source tree after
0.5.0 release
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/store.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc
new file mode 100644
index 0000000..9394e3d
--- /dev/null
+++ b/cpp/src/plasma/store.cc
@@ -0,0 +1,683 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// PLASMA STORE: This is a simple object store server process
+//
+// It accepts incoming client connections on a unix domain socket
+// (name passed in via the -s option of the executable) and uses a
+// single thread to serve the clients. Each client establishes a
+// connection and can create objects, wait for objects and seal
+// objects through that connection.
+//
+// It keeps a hash table that maps object_ids (which are 20 byte long,
+// just enough to store and SHA1 hash) to memory mapped files.
+
+#include "plasma/store.h"
+
+#include <assert.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <limits.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/statvfs.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include <deque>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "format/common_generated.h"
+#include "plasma/common.h"
+#include "plasma/fling.h"
+#include "plasma/io.h"
+#include "plasma/malloc.h"
+
+extern "C" {
+void* dlmalloc(size_t bytes);
+void* dlmemalign(size_t alignment, size_t bytes);
+void dlfree(void* mem);
+size_t dlmalloc_set_footprint_limit(size_t bytes);
+}
+
+struct GetRequest {
+ GetRequest(Client* client, const std::vector<ObjectID>& object_ids);
+ /// The client that called get.
+ Client* client;
+ /// The ID of the timer that will time out and cause this wait to return to
+ /// the client if it hasn't already returned.
+ int64_t timer;
+ /// The object IDs involved in this request. This is used in the reply.
+ std::vector<ObjectID> object_ids;
+ /// The object information for the objects in this request. This is used in
+ /// the reply.
+ std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> objects;
+ /// The minimum number of objects to wait for in this request.
+ int64_t num_objects_to_wait_for;
+ /// The number of object requests in this wait request that are already
+ /// satisfied.
+ int64_t num_satisfied;
+};
+
+GetRequest::GetRequest(Client* client, const std::vector<ObjectID>& object_ids)
+ : client(client),
+ timer(-1),
+ object_ids(object_ids.begin(), object_ids.end()),
+ objects(object_ids.size()),
+ num_satisfied(0) {
+ std::unordered_set<ObjectID, UniqueIDHasher> unique_ids(
+ object_ids.begin(), object_ids.end());
+ num_objects_to_wait_for = unique_ids.size();
+}
+
+Client::Client(int fd) : fd(fd) {}
+
+PlasmaStore::PlasmaStore(EventLoop* loop, int64_t system_memory)
+ : loop_(loop), eviction_policy_(&store_info_) {
+ store_info_.memory_capacity = system_memory;
+}
+
+// TODO(pcm): Get rid of this destructor by using RAII to clean up data.
+PlasmaStore::~PlasmaStore() {
+ for (const auto& element : pending_notifications_) {
+ auto object_notifications = element.second.object_notifications;
+ for (size_t i = 0; i < object_notifications.size(); ++i) {
+ uint8_t* notification = reinterpret_cast<uint8_t*>(object_notifications.at(i));
+ uint8_t* data = notification;
+ // TODO(pcm): Get rid of this delete.
+ delete[] data;
+ }
+ }
+}
+
+// If this client is not already using the object, add the client to the
+// object's list of clients, otherwise do nothing.
+void PlasmaStore::add_client_to_object_clients(ObjectTableEntry* entry, Client* client) {
+ // Check if this client is already using the object.
+ if (entry->clients.find(client) != entry->clients.end()) { return; }
+ // If there are no other clients using this object, notify the eviction policy
+ // that the object is being used.
+ if (entry->clients.size() == 0) {
+ // Tell the eviction policy that this object is being used.
+ std::vector<ObjectID> objects_to_evict;
+ eviction_policy_.begin_object_access(entry->object_id, &objects_to_evict);
+ delete_objects(objects_to_evict);
+ }
+ // Add the client pointer to the list of clients using this object.
+ entry->clients.insert(client);
+}
+
+// Create a new object buffer in the hash table.
+int PlasmaStore::create_object(const ObjectID& object_id, int64_t data_size,
+ int64_t metadata_size, Client* client, PlasmaObject* result) {
+ ARROW_LOG(DEBUG) << "creating object " << object_id.hex();
+ if (store_info_.objects.count(object_id) != 0) {
+ // There is already an object with the same ID in the Plasma Store, so
+ // ignore this requst.
+ return PlasmaError_ObjectExists;
+ }
+ // Try to evict objects until there is enough space.
+ uint8_t* pointer;
+ do {
+ // Allocate space for the new object. We use dlmemalign instead of dlmalloc
+ // in order to align the allocated region to a 64-byte boundary. This is not
+ // strictly necessary, but it is an optimization that could speed up the
+ // computation of a hash of the data (see compute_object_hash_parallel in
+ // plasma_client.cc). Note that even though this pointer is 64-byte aligned,
+ // it is not guaranteed that the corresponding pointer in the client will be
+ // 64-byte aligned, but in practice it often will be.
+ pointer =
+ reinterpret_cast<uint8_t*>(dlmemalign(BLOCK_SIZE, data_size + metadata_size));
+ if (pointer == NULL) {
+ // Tell the eviction policy how much space we need to create this object.
+ std::vector<ObjectID> objects_to_evict;
+ bool success =
+ eviction_policy_.require_space(data_size + metadata_size, &objects_to_evict);
+ delete_objects(objects_to_evict);
+ // Return an error to the client if not enough space could be freed to
+ // create the object.
+ if (!success) { return PlasmaError_OutOfMemory; }
+ }
+ } while (pointer == NULL);
+ int fd;
+ int64_t map_size;
+ ptrdiff_t offset;
+ get_malloc_mapinfo(pointer, &fd, &map_size, &offset);
+ assert(fd != -1);
+
+ auto entry = std::unique_ptr<ObjectTableEntry>(new ObjectTableEntry());
+ entry->object_id = object_id;
+ entry->info.object_id = object_id.binary();
+ entry->info.data_size = data_size;
+ entry->info.metadata_size = metadata_size;
+ entry->pointer = pointer;
+ // TODO(pcm): Set the other fields.
+ entry->fd = fd;
+ entry->map_size = map_size;
+ entry->offset = offset;
+ entry->state = PLASMA_CREATED;
+
+ store_info_.objects[object_id] = std::move(entry);
+ result->handle.store_fd = fd;
+ result->handle.mmap_size = map_size;
+ result->data_offset = offset;
+ result->metadata_offset = offset + data_size;
+ result->data_size = data_size;
+ result->metadata_size = metadata_size;
+ // Notify the eviction policy that this object was created. This must be done
+ // immediately before the call to add_client_to_object_clients so that the
+ // eviction policy does not have an opportunity to evict the object.
+ eviction_policy_.object_created(object_id);
+ // Record that this client is using this object.
+ add_client_to_object_clients(store_info_.objects[object_id].get(), client);
+ return PlasmaError_OK;
+}
+
+void PlasmaObject_init(PlasmaObject* object, ObjectTableEntry* entry) {
+ DCHECK(object != NULL);
+ DCHECK(entry != NULL);
+ DCHECK(entry->state == PLASMA_SEALED);
+ object->handle.store_fd = entry->fd;
+ object->handle.mmap_size = entry->map_size;
+ object->data_offset = entry->offset;
+ object->metadata_offset = entry->offset + entry->info.data_size;
+ object->data_size = entry->info.data_size;
+ object->metadata_size = entry->info.metadata_size;
+}
+
+void PlasmaStore::return_from_get(GetRequest* get_req) {
+ // Send the get reply to the client.
+ Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], get_req->objects,
+ get_req->object_ids.size());
+ warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd);
+ // If we successfully sent the get reply message to the client, then also send
+ // the file descriptors.
+ if (s.ok()) {
+ // Send all of the file descriptors for the present objects.
+ for (const auto& object_id : get_req->object_ids) {
+ PlasmaObject& object = get_req->objects[object_id];
+ // We use the data size to indicate whether the object is present or not.
+ if (object.data_size != -1) {
+ int error_code = send_fd(get_req->client->fd, object.handle.store_fd);
+ // If we failed to send the file descriptor, loop until we have sent it
+ // successfully. TODO(rkn): This is problematic for two reasons. First
+ // of all, sending the file descriptor should just succeed without any
+ // errors, but sometimes I see a "Message too long" error number.
+ // Second, looping like this allows a client to potentially block the
+ // plasma store event loop which should never happen.
+ while (error_code < 0) {
+ if (errno == EMSGSIZE) {
+ ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying.";
+ error_code = send_fd(get_req->client->fd, object.handle.store_fd);
+ continue;
+ }
+ warn_if_sigpipe(error_code, get_req->client->fd);
+ break;
+ }
+ }
+ }
+ }
+
+ // Remove the get request from each of the relevant object_get_requests hash
+ // tables if it is present there. It should only be present there if the get
+ // request timed out.
+ for (ObjectID& object_id : get_req->object_ids) {
+ auto& get_requests = object_get_requests_[object_id];
+ // Erase get_req from the vector.
+ auto it = std::find(get_requests.begin(), get_requests.end(), get_req);
+ if (it != get_requests.end()) { get_requests.erase(it); }
+ }
+ // Remove the get request.
+ if (get_req->timer != -1) { ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK); }
+ delete get_req;
+}
+
+void PlasmaStore::update_object_get_requests(const ObjectID& object_id) {
+ std::vector<GetRequest*>& get_requests = object_get_requests_[object_id];
+ size_t index = 0;
+ size_t num_requests = get_requests.size();
+ for (size_t i = 0; i < num_requests; ++i) {
+ GetRequest* get_req = get_requests[index];
+ auto entry = get_object_table_entry(&store_info_, object_id);
+ ARROW_CHECK(entry != NULL);
+
+ PlasmaObject_init(&get_req->objects[object_id], entry);
+ get_req->num_satisfied += 1;
+ // Record the fact that this client will be using this object and will
+ // be responsible for releasing this object.
+ add_client_to_object_clients(entry, get_req->client);
+
+ // If this get request is done, reply to the client.
+ if (get_req->num_satisfied == get_req->num_objects_to_wait_for) {
+ return_from_get(get_req);
+ } else {
+ // The call to return_from_get will remove the current element in the
+ // array, so we only increment the counter in the else branch.
+ index += 1;
+ }
+ }
+
+ DCHECK(index == get_requests.size());
+ // Remove the array of get requests for this object, since no one should be
+ // waiting for this object anymore.
+ object_get_requests_.erase(object_id);
+}
+
+void PlasmaStore::process_get_request(
+ Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms) {
+ // Create a get request for this object.
+ GetRequest* get_req = new GetRequest(client, object_ids);
+
+ for (auto object_id : object_ids) {
+ // Check if this object is already present locally. If so, record that the
+ // object is being used and mark it as accounted for.
+ auto entry = get_object_table_entry(&store_info_, object_id);
+ if (entry && entry->state == PLASMA_SEALED) {
+ // Update the get request to take into account the present object.
+ PlasmaObject_init(&get_req->objects[object_id], entry);
+ get_req->num_satisfied += 1;
+ // If necessary, record that this client is using this object. In the case
+ // where entry == NULL, this will be called from seal_object.
+ add_client_to_object_clients(entry, client);
+ } else {
+ // Add a placeholder plasma object to the get request to indicate that the
+ // object is not present. This will be parsed by the client. We set the
+ // data size to -1 to indicate that the object is not present.
+ get_req->objects[object_id].data_size = -1;
+ // Add the get request to the relevant data structures.
+ object_get_requests_[object_id].push_back(get_req);
+ }
+ }
+
+ // If all of the objects are present already or if the timeout is 0, return to
+ // the client.
+ if (get_req->num_satisfied == get_req->num_objects_to_wait_for || timeout_ms == 0) {
+ return_from_get(get_req);
+ } else if (timeout_ms != -1) {
+ // Set a timer that will cause the get request to return to the client. Note
+ // that a timeout of -1 is used to indicate that no timer should be set.
+ get_req->timer = loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) {
+ return_from_get(get_req);
+ return kEventLoopTimerDone;
+ });
+ }
+}
+
+int PlasmaStore::remove_client_from_object_clients(
+ ObjectTableEntry* entry, Client* client) {
+ auto it = entry->clients.find(client);
+ if (it != entry->clients.end()) {
+ entry->clients.erase(it);
+ // If no more clients are using this object, notify the eviction policy
+ // that the object is no longer being used.
+ if (entry->clients.size() == 0) {
+ // Tell the eviction policy that this object is no longer being used.
+ std::vector<ObjectID> objects_to_evict;
+ eviction_policy_.end_object_access(entry->object_id, &objects_to_evict);
+ delete_objects(objects_to_evict);
+ }
+ // Return 1 to indicate that the client was removed.
+ return 1;
+ } else {
+ // Return 0 to indicate that the client was not removed.
+ return 0;
+ }
+}
+
+void PlasmaStore::release_object(const ObjectID& object_id, Client* client) {
+ auto entry = get_object_table_entry(&store_info_, object_id);
+ ARROW_CHECK(entry != NULL);
+ // Remove the client from the object's array of clients.
+ ARROW_CHECK(remove_client_from_object_clients(entry, client) == 1);
+}
+
+// Check if an object is present.
+int PlasmaStore::contains_object(const ObjectID& object_id) {
+ auto entry = get_object_table_entry(&store_info_, object_id);
+ return entry && (entry->state == PLASMA_SEALED) ? OBJECT_FOUND : OBJECT_NOT_FOUND;
+}
+
+// Seal an object that has been created in the hash table.
+void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[]) {
+ ARROW_LOG(DEBUG) << "sealing object " << object_id.hex();
+ auto entry = get_object_table_entry(&store_info_, object_id);
+ ARROW_CHECK(entry != NULL);
+ ARROW_CHECK(entry->state == PLASMA_CREATED);
+ // Set the state of object to SEALED.
+ entry->state = PLASMA_SEALED;
+ // Set the object digest.
+ entry->info.digest = std::string(reinterpret_cast<char*>(&digest[0]), kDigestSize);
+ // Inform all subscribers that a new object has been sealed.
+ push_notification(&entry->info);
+
+ // Update all get requests that involve this object.
+ update_object_get_requests(object_id);
+}
+
+void PlasmaStore::delete_objects(const std::vector<ObjectID>& object_ids) {
+ for (const auto& object_id : object_ids) {
+ ARROW_LOG(DEBUG) << "deleting object " << object_id.hex();
+ auto entry = get_object_table_entry(&store_info_, object_id);
+ // TODO(rkn): This should probably not fail, but should instead throw an
+ // error. Maybe we should also support deleting objects that have been
+ // created but not sealed.
+ ARROW_CHECK(entry != NULL) << "To delete an object it must be in the object table.";
+ ARROW_CHECK(entry->state == PLASMA_SEALED)
+ << "To delete an object it must have been sealed.";
+ ARROW_CHECK(entry->clients.size() == 0)
+ << "To delete an object, there must be no clients currently using it.";
+ dlfree(entry->pointer);
+ store_info_.objects.erase(object_id);
+ // Inform all subscribers that the object has been deleted.
+ ObjectInfoT notification;
+ notification.object_id = object_id.binary();
+ notification.is_deletion = true;
+ push_notification(¬ification);
+ }
+}
+
+void PlasmaStore::connect_client(int listener_sock) {
+ int client_fd = AcceptClient(listener_sock);
+ // This is freed in disconnect_client.
+ Client* client = new Client(client_fd);
+ // Add a callback to handle events on this socket.
+ // TODO(pcm): Check return value.
+ loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) {
+ Status s = process_message(client);
+ if (!s.ok()) { ARROW_LOG(FATAL) << "Failed to process file event: " << s; }
+ });
+ ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
+}
+
+void PlasmaStore::disconnect_client(Client* client) {
+ ARROW_CHECK(client != NULL);
+ ARROW_CHECK(client->fd > 0);
+ loop_->remove_file_event(client->fd);
+ // Close the socket.
+ close(client->fd);
+ ARROW_LOG(INFO) << "Disconnecting client on fd " << client->fd;
+ // If this client was using any objects, remove it from the appropriate
+ // lists.
+ for (const auto& entry : store_info_.objects) {
+ remove_client_from_object_clients(entry.second.get(), client);
+ }
+ // Note, the store may still attempt to send a message to the disconnected
+ // client (for example, when an object ID that the client was waiting for
+ // is ready). In these cases, the attempt to send the message will fail, but
+ // the store should just ignore the failure.
+ delete client;
+}
+
+/// Send notifications about sealed objects to the subscribers. This is called
+/// in seal_object. If the socket's send buffer is full, the notification will
+/// be
+/// buffered, and this will be called again when the send buffer has room.
+///
+/// @param client The client to send the notification to.
+/// @return Void.
+void PlasmaStore::send_notifications(int client_fd) {
+ auto it = pending_notifications_.find(client_fd);
+
+ int num_processed = 0;
+ bool closed = false;
+ // Loop over the array of pending notifications and send as many of them as
+ // possible.
+ for (size_t i = 0; i < it->second.object_notifications.size(); ++i) {
+ uint8_t* notification =
+ reinterpret_cast<uint8_t*>(it->second.object_notifications.at(i));
+ // Decode the length, which is the first bytes of the message.
+ int64_t size = *(reinterpret_cast<int64_t*>(notification));
+
+ // Attempt to send a notification about this object ID.
+ ssize_t nbytes = send(client_fd, notification, sizeof(int64_t) + size, 0);
+ if (nbytes >= 0) {
+ ARROW_CHECK(nbytes == static_cast<ssize_t>(sizeof(int64_t)) + size);
+ } else if (nbytes == -1 &&
+ (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
+ ARROW_LOG(DEBUG) << "The socket's send buffer is full, so we are caching this "
+ "notification and will send it later.";
+ // Add a callback to the event loop to send queued notifications whenever
+ // there is room in the socket's send buffer. Callbacks can be added
+ // more than once here and will be overwritten. The callback is removed
+ // at the end of the method.
+ // TODO(pcm): Introduce status codes and check in case the file descriptor
+ // is added twice.
+ loop_->add_file_event(client_fd, kEventLoopWrite,
+ [this, client_fd](int events) { send_notifications(client_fd); });
+ break;
+ } else {
+ ARROW_LOG(WARNING) << "Failed to send notification to client on fd " << client_fd;
+ if (errno == EPIPE) {
+ closed = true;
+ break;
+ }
+ }
+ num_processed += 1;
+ // The corresponding malloc happened in create_object_info_buffer
+ // within push_notification.
+ delete[] notification;
+ }
+ // Remove the sent notifications from the array.
+ it->second.object_notifications.erase(it->second.object_notifications.begin(),
+ it->second.object_notifications.begin() + num_processed);
+
+ // Stop sending notifications if the pipe was broken.
+ if (closed) {
+ close(client_fd);
+ pending_notifications_.erase(client_fd);
+ }
+
+ // If we have sent all notifications, remove the fd from the event loop.
+ if (it->second.object_notifications.empty()) { loop_->remove_file_event(client_fd); }
+}
+
+void PlasmaStore::push_notification(ObjectInfoT* object_info) {
+ for (auto& element : pending_notifications_) {
+ uint8_t* notification = create_object_info_buffer(object_info);
+ element.second.object_notifications.push_back(notification);
+ send_notifications(element.first);
+ // The notification gets freed in send_notifications when the notification
+ // is sent over the socket.
+ }
+}
+
+// Subscribe to notifications about sealed objects.
+void PlasmaStore::subscribe_to_updates(Client* client) {
+ ARROW_LOG(DEBUG) << "subscribing to updates on fd " << client->fd;
+ // TODO(rkn): The store could block here if the client doesn't send a file
+ // descriptor.
+ int fd = recv_fd(client->fd);
+ if (fd < 0) {
+ // This may mean that the client died before sending the file descriptor.
+ ARROW_LOG(WARNING) << "Failed to receive file descriptor from client on fd "
+ << client->fd << ".";
+ return;
+ }
+
+ // Create a new array to buffer notifications that can't be sent to the
+ // subscriber yet because the socket send buffer is full. TODO(rkn): the queue
+ // never gets freed.
+ // TODO(pcm): Is the following neccessary?
+ pending_notifications_[fd];
+
+ // Push notifications to the new subscriber about existing objects.
+ for (const auto& entry : store_info_.objects) {
+ push_notification(&entry.second->info);
+ }
+ send_notifications(fd);
+}
+
+Status PlasmaStore::process_message(Client* client) {
+ int64_t type;
+ Status s = ReadMessage(client->fd, &type, &input_buffer_);
+ ARROW_CHECK(s.ok() || s.IsIOError());
+
+ uint8_t* input = input_buffer_.data();
+ ObjectID object_id;
+ PlasmaObject object;
+ // TODO(pcm): Get rid of the following.
+ memset(&object, 0, sizeof(object));
+
+ // Process the different types of requests.
+ switch (type) {
+ case MessageType_PlasmaCreateRequest: {
+ int64_t data_size;
+ int64_t metadata_size;
+ RETURN_NOT_OK(ReadCreateRequest(input, &object_id, &data_size, &metadata_size));
+ int error_code =
+ create_object(object_id, data_size, metadata_size, client, &object);
+ HANDLE_SIGPIPE(
+ SendCreateReply(client->fd, object_id, &object, error_code), client->fd);
+ if (error_code == PlasmaError_OK) {
+ warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
+ }
+ } break;
+ case MessageType_PlasmaGetRequest: {
+ std::vector<ObjectID> object_ids_to_get;
+ int64_t timeout_ms;
+ RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms));
+ process_get_request(client, object_ids_to_get, timeout_ms);
+ } break;
+ case MessageType_PlasmaReleaseRequest:
+ RETURN_NOT_OK(ReadReleaseRequest(input, &object_id));
+ release_object(object_id, client);
+ break;
+ case MessageType_PlasmaContainsRequest:
+ RETURN_NOT_OK(ReadContainsRequest(input, &object_id));
+ if (contains_object(object_id) == OBJECT_FOUND) {
+ HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd);
+ } else {
+ HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd);
+ }
+ break;
+ case MessageType_PlasmaSealRequest: {
+ unsigned char digest[kDigestSize];
+ RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0]));
+ seal_object(object_id, &digest[0]);
+ } break;
+ case MessageType_PlasmaEvictRequest: {
+ // This code path should only be used for testing.
+ int64_t num_bytes;
+ RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes));
+ std::vector<ObjectID> objects_to_evict;
+ int64_t num_bytes_evicted =
+ eviction_policy_.choose_objects_to_evict(num_bytes, &objects_to_evict);
+ delete_objects(objects_to_evict);
+ HANDLE_SIGPIPE(SendEvictReply(client->fd, num_bytes_evicted), client->fd);
+ } break;
+ case MessageType_PlasmaSubscribeRequest:
+ subscribe_to_updates(client);
+ break;
+ case MessageType_PlasmaConnectRequest: {
+ HANDLE_SIGPIPE(
+ SendConnectReply(client->fd, store_info_.memory_capacity), client->fd);
+ } break;
+ case DISCONNECT_CLIENT:
+ ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd;
+ disconnect_client(client);
+ break;
+ default:
+ // This code should be unreachable.
+ ARROW_CHECK(0);
+ }
+ return Status::OK();
+}
+
+// Report "success" to valgrind.
+void signal_handler(int signal) {
+ if (signal == SIGTERM) { exit(0); }
+}
+
+void start_server(char* socket_name, int64_t system_memory) {
+ // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
+ // to a client that has already died, the store could die.
+ signal(SIGPIPE, SIG_IGN);
+ // Create the event loop.
+ EventLoop loop;
+ PlasmaStore store(&loop, system_memory);
+ int socket = bind_ipc_sock(socket_name, true);
+ ARROW_CHECK(socket >= 0);
+ // TODO(pcm): Check return value.
+ loop.add_file_event(socket, kEventLoopRead,
+ [&store, socket](int events) { store.connect_client(socket); });
+ loop.run();
+}
+
+int main(int argc, char* argv[]) {
+ signal(SIGTERM, signal_handler);
+ char* socket_name = NULL;
+ int64_t system_memory = -1;
+ int c;
+ while ((c = getopt(argc, argv, "s:m:")) != -1) {
+ switch (c) {
+ case 's':
+ socket_name = optarg;
+ break;
+ case 'm': {
+ char extra;
+ int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra);
+ ARROW_CHECK(scanned == 1);
+ ARROW_LOG(INFO) << "Allowing the Plasma store to use up to "
+ << static_cast<double>(system_memory) / 1000000000
+ << "GB of memory.";
+ break;
+ }
+ default:
+ exit(-1);
+ }
+ }
+ if (!socket_name) {
+ ARROW_LOG(FATAL) << "please specify socket for incoming connections with -s switch";
+ }
+ if (system_memory == -1) {
+ ARROW_LOG(FATAL) << "please specify the amount of system memory with -m switch";
+ }
+#ifdef __linux__
+ // On Linux, check that the amount of memory available in /dev/shm is large
+ // enough to accommodate the request. If it isn't, then fail.
+ int shm_fd = open("/dev/shm", O_RDONLY);
+ struct statvfs shm_vfs_stats;
+ fstatvfs(shm_fd, &shm_vfs_stats);
+ // The value shm_vfs_stats.f_bsize is the block size, and the value
+ // shm_vfs_stats.f_bavail is the number of available blocks.
+ int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail;
+ close(shm_fd);
+ if (system_memory > shm_mem_avail) {
+ ARROW_LOG(FATAL) << "System memory request exceeds memory available in /dev/shm. The "
+ "request is for "
+ << system_memory << " bytes, and the amount available is "
+ << shm_mem_avail
+ << " bytes. You may be able to free up space by deleting files in "
+ "/dev/shm. If you are inside a Docker container, you may need to "
+ "pass "
+ "an argument with the flag '--shm-size' to 'docker run'.";
+ }
+#endif
+ // Make it so dlmalloc fails if we try to request more memory than is
+ // available.
+ dlmalloc_set_footprint_limit((size_t)system_memory);
+ ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
+ start_server(socket_name, system_memory);
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/store.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h
new file mode 100644
index 0000000..8bd9426
--- /dev/null
+++ b/cpp/src/plasma/store.h
@@ -0,0 +1,169 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef PLASMA_STORE_H
+#define PLASMA_STORE_H
+
+#include <deque>
+#include <vector>
+
+#include "plasma/common.h"
+#include "plasma/events.h"
+#include "plasma/eviction_policy.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+struct GetRequest;
+
+struct NotificationQueue {
+ /// The object notifications for clients. We notify the client about the
+ /// objects in the order that the objects were sealed or deleted.
+ std::deque<uint8_t*> object_notifications;
+};
+
+/// Contains all information that is associated with a Plasma store client.
+struct Client {
+ explicit Client(int fd);
+
+ /// The file descriptor used to communicate with the client.
+ int fd;
+};
+
+class PlasmaStore {
+ public:
+ PlasmaStore(EventLoop* loop, int64_t system_memory);
+
+ ~PlasmaStore();
+
+ /// Create a new object. The client must do a call to release_object to tell
+ /// the store when it is done with the object.
+ ///
+ /// @param object_id Object ID of the object to be created.
+ /// @param data_size Size in bytes of the object to be created.
+ /// @param metadata_size Size in bytes of the object metadata.
+ /// @return One of the following error codes:
+ /// - PlasmaError_OK, if the object was created successfully.
+ /// - PlasmaError_ObjectExists, if an object with this ID is already
+ /// present in the store. In this case, the client should not call
+ /// plasma_release.
+ /// - PlasmaError_OutOfMemory, if the store is out of memory and
+ /// cannot create the object. In this case, the client should not call
+ /// plasma_release.
+ int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size,
+ Client* client, PlasmaObject* result);
+
+ /// Delete objects that have been created in the hash table. This should only
+ /// be called on objects that are returned by the eviction policy to evict.
+ ///
+ /// @param object_ids Object IDs of the objects to be deleted.
+ /// @return Void.
+ void delete_objects(const std::vector<ObjectID>& object_ids);
+
+ /// Process a get request from a client. This method assumes that we will
+ /// eventually have these objects sealed. If one of the objects has not yet
+ /// been sealed, the client that requested the object will be notified when it
+ /// is sealed.
+ ///
+ /// For each object, the client must do a call to release_object to tell the
+ /// store when it is done with the object.
+ ///
+ /// @param client The client making this request.
+ /// @param object_ids Object IDs of the objects to be gotten.
+ /// @param timeout_ms The timeout for the get request in milliseconds.
+ /// @return Void.
+ void process_get_request(
+ Client* client, const std::vector<ObjectID>& object_ids, int64_t timeout_ms);
+
+ /// Seal an object. The object is now immutable and can be accessed with get.
+ ///
+ /// @param object_id Object ID of the object to be sealed.
+ /// @param digest The digest of the object. This is used to tell if two
+ /// objects
+ /// with the same object ID are the same.
+ /// @return Void.
+ void seal_object(const ObjectID& object_id, unsigned char digest[]);
+
+ /// Check if the plasma store contains an object:
+ ///
+ /// @param object_id Object ID that will be checked.
+ /// @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if
+ /// not
+ int contains_object(const ObjectID& object_id);
+
+ /// Record the fact that a particular client is no longer using an object.
+ ///
+ /// @param object_id The object ID of the object that is being released.
+ /// @param client The client making this request.
+ /// @param Void.
+ void release_object(const ObjectID& object_id, Client* client);
+
+ /// Subscribe a file descriptor to updates about new sealed objects.
+ ///
+ /// @param client The client making this request.
+ /// @return Void.
+ void subscribe_to_updates(Client* client);
+
+ /// Connect a new client to the PlasmaStore.
+ ///
+ /// @param listener_sock The socket that is listening to incoming connections.
+ /// @return Void.
+ void connect_client(int listener_sock);
+
+ /// Disconnect a client from the PlasmaStore.
+ ///
+ /// @param client The client that is disconnected.
+ /// @return Void.
+ void disconnect_client(Client* client);
+
+ void send_notifications(int client_fd);
+
+ Status process_message(Client* client);
+
+ private:
+ void push_notification(ObjectInfoT* object_notification);
+
+ void add_client_to_object_clients(ObjectTableEntry* entry, Client* client);
+
+ void return_from_get(GetRequest* get_req);
+
+ void update_object_get_requests(const ObjectID& object_id);
+
+ int remove_client_from_object_clients(ObjectTableEntry* entry, Client* client);
+
+ /// Event loop of the plasma store.
+ EventLoop* loop_;
+ /// The plasma store information, including the object tables, that is exposed
+ /// to the eviction policy.
+ PlasmaStoreInfo store_info_;
+ /// The state that is managed by the eviction policy.
+ EvictionPolicy eviction_policy_;
+ /// Input buffer. This is allocated only once to avoid mallocs for every
+ /// call to process_message.
+ std::vector<uint8_t> input_buffer_;
+ /// A hash table mapping object IDs to a vector of the get requests that are
+ /// waiting for the object to arrive.
+ std::unordered_map<ObjectID, std::vector<GetRequest*>, UniqueIDHasher>
+ object_get_requests_;
+ /// The pending notifications that have not been sent to subscribers because
+ /// the socket send buffers were full. This is a hash table from client file
+ /// descriptor to an array of object_ids to send to that client.
+ /// TODO(pcm): Consider putting this into the Client data structure and
+ /// reorganize the code slightly.
+ std::unordered_map<int, NotificationQueue> pending_notifications_;
+};
+
+#endif // PLASMA_STORE_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/test/client_tests.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc
new file mode 100644
index 0000000..29b5b13
--- /dev/null
+++ b/cpp/src/plasma/test/client_tests.cc
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include <assert.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/client.h"
+#include "plasma/common.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+std::string g_test_executable; // NOLINT
+
+class TestPlasmaStore : public ::testing::Test {
+ public:
+ // TODO(pcm): At the moment, stdout of the test gets mixed up with
+ // stdout of the object store. Consider changing that.
+ void SetUp() {
+ std::string plasma_directory =
+ g_test_executable.substr(0, g_test_executable.find_last_of("/"));
+ std::string plasma_command =
+ plasma_directory +
+ "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &";
+ system(plasma_command.c_str());
+ ARROW_CHECK_OK(client_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY));
+ }
+ virtual void Finish() {
+ ARROW_CHECK_OK(client_.Disconnect());
+ system("killall plasma_store &");
+ }
+
+ protected:
+ PlasmaClient client_;
+};
+
+TEST_F(TestPlasmaStore, ContainsTest) {
+ ObjectID object_id = ObjectID::from_random();
+
+ // Test for object non-existence.
+ bool has_object;
+ ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+ ASSERT_EQ(has_object, false);
+
+ // Test for the object being in local Plasma store.
+ // First create object.
+ int64_t data_size = 100;
+ uint8_t metadata[] = {5};
+ int64_t metadata_size = sizeof(metadata);
+ uint8_t* data;
+ ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+ ARROW_CHECK_OK(client_.Seal(object_id));
+ // Avoid race condition of Plasma Manager waiting for notification.
+ ObjectBuffer object_buffer;
+ ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+ ARROW_CHECK_OK(client_.Contains(object_id, &has_object));
+ ASSERT_EQ(has_object, true);
+}
+
+TEST_F(TestPlasmaStore, GetTest) {
+ ObjectID object_id = ObjectID::from_random();
+ ObjectBuffer object_buffer;
+
+ // Test for object non-existence.
+ ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer));
+ ASSERT_EQ(object_buffer.data_size, -1);
+
+ // Test for the object being in local Plasma store.
+ // First create object.
+ int64_t data_size = 4;
+ uint8_t metadata[] = {5};
+ int64_t metadata_size = sizeof(metadata);
+ uint8_t* data;
+ ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data));
+ for (int64_t i = 0; i < data_size; i++) {
+ data[i] = static_cast<uint8_t>(i % 4);
+ }
+ ARROW_CHECK_OK(client_.Seal(object_id));
+
+ ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer));
+ for (int64_t i = 0; i < data_size; i++) {
+ ASSERT_EQ(data[i], object_buffer.data[i]);
+ }
+}
+
+TEST_F(TestPlasmaStore, MultipleGetTest) {
+ ObjectID object_id1 = ObjectID::from_random();
+ ObjectID object_id2 = ObjectID::from_random();
+ ObjectID object_ids[2] = {object_id1, object_id2};
+ ObjectBuffer object_buffer[2];
+
+ int64_t data_size = 4;
+ uint8_t metadata[] = {5};
+ int64_t metadata_size = sizeof(metadata);
+ uint8_t* data;
+ ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data));
+ data[0] = 1;
+ ARROW_CHECK_OK(client_.Seal(object_id1));
+
+ ARROW_CHECK_OK(client_.Create(object_id2, data_size, metadata, metadata_size, &data));
+ data[0] = 2;
+ ARROW_CHECK_OK(client_.Seal(object_id2));
+
+ ARROW_CHECK_OK(client_.Get(object_ids, 2, -1, object_buffer));
+ ASSERT_EQ(object_buffer[0].data[0], 1);
+ ASSERT_EQ(object_buffer[1].data[0], 2);
+}
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ g_test_executable = std::string(argv[0]);
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/test/run_tests.sh
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/test/run_tests.sh b/cpp/src/plasma/test/run_tests.sh
new file mode 100644
index 0000000..958bd08
--- /dev/null
+++ b/cpp/src/plasma/test/run_tests.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cause the script to exit if a single command fails.
+set -e
+
+./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
+sleep 1
+./src/plasma/manager_tests
+killall plasma_store
+./src/plasma/serialization_tests
+
+# Start the Redis shards.
+./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
+redis_pid1=$!
+./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
+redis_pid2=$!
+sleep 1
+
+# Flush the redis server
+./src/common/thirdparty/redis/src/redis-cli flushall
+# Register the shard location with the primary shard.
+./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
+./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
+sleep 1
+./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
+plasma1_pid=$!
+./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
+plasma2_pid=$!
+./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 &
+plasma3_pid=$!
+./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
+plasma4_pid=$!
+sleep 1
+
+./src/plasma/client_tests
+
+kill $plasma4_pid
+kill $plasma3_pid
+kill $plasma2_pid
+kill $plasma1_pid
+kill $redis_pid1
+wait $redis_pid1
+kill $redis_pid2
+wait $redis_pid2
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/test/run_valgrind.sh
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/test/run_valgrind.sh b/cpp/src/plasma/test/run_valgrind.sh
new file mode 100644
index 0000000..0472194
--- /dev/null
+++ b/cpp/src/plasma/test/run_valgrind.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cause the script to exit if a single command fails.
+set -e
+
+./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
+sleep 1
+valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests
+killall plasma_store
+valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/test/serialization_tests.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/test/serialization_tests.cc b/cpp/src/plasma/test/serialization_tests.cc
new file mode 100644
index 0000000..325cead
--- /dev/null
+++ b/cpp/src/plasma/test/serialization_tests.cc
@@ -0,0 +1,388 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "plasma/common.h"
+#include "plasma/io.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+
+/**
+ * Create a temporary file. Needs to be closed by the caller.
+ *
+ * @return File descriptor of the file.
+ */
+int create_temp_file(void) {
+ static char temp[] = "/tmp/tempfileXXXXXX";
+ char file_name[32];
+ strncpy(file_name, temp, 32);
+ return mkstemp(file_name);
+}
+
+/**
+ * Seek to the beginning of a file and read a message from it.
+ *
+ * @param fd File descriptor of the file.
+ * @param message type Message type that we expect in the file.
+ *
+ * @return Pointer to the content of the message. Needs to be freed by the
+ * caller.
+ */
+std::vector<uint8_t> read_message_from_file(int fd, int message_type) {
+ /* Go to the beginning of the file. */
+ lseek(fd, 0, SEEK_SET);
+ int64_t type;
+ std::vector<uint8_t> data;
+ ARROW_CHECK_OK(ReadMessage(fd, &type, &data));
+ ARROW_CHECK(type == message_type);
+ return data;
+}
+
+PlasmaObject random_plasma_object(void) {
+ unsigned int seed = static_cast<unsigned int>(time(NULL));
+ int random = rand_r(&seed);
+ PlasmaObject object;
+ memset(&object, 0, sizeof(object));
+ object.handle.store_fd = random + 7;
+ object.handle.mmap_size = random + 42;
+ object.data_offset = random + 1;
+ object.metadata_offset = random + 2;
+ object.data_size = random + 3;
+ object.metadata_size = random + 4;
+ return object;
+}
+
+TEST(PlasmaSerialization, CreateRequest) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ int64_t data_size1 = 42;
+ int64_t metadata_size1 = 11;
+ ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateRequest);
+ ObjectID object_id2;
+ int64_t data_size2;
+ int64_t metadata_size2;
+ ARROW_CHECK_OK(
+ ReadCreateRequest(data.data(), &object_id2, &data_size2, &metadata_size2));
+ ASSERT_EQ(data_size1, data_size2);
+ ASSERT_EQ(metadata_size1, metadata_size2);
+ ASSERT_EQ(object_id1, object_id2);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, CreateReply) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ PlasmaObject object1 = random_plasma_object();
+ ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaCreateReply);
+ ObjectID object_id2;
+ PlasmaObject object2;
+ memset(&object2, 0, sizeof(object2));
+ ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2));
+ ASSERT_EQ(object_id1, object_id2);
+ ASSERT_EQ(memcmp(&object1, &object2, sizeof(object1)), 0);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, SealRequest) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ unsigned char digest1[kDigestSize];
+ memset(&digest1[0], 7, kDigestSize);
+ ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0]));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealRequest);
+ ObjectID object_id2;
+ unsigned char digest2[kDigestSize];
+ ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0]));
+ ASSERT_EQ(object_id1, object_id2);
+ ASSERT_EQ(memcmp(&digest1[0], &digest2[0], kDigestSize), 0);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, SealReply) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaSealReply);
+ ObjectID object_id2;
+ Status s = ReadSealReply(data.data(), &object_id2);
+ ASSERT_EQ(object_id1, object_id2);
+ ASSERT_TRUE(s.IsPlasmaObjectExists());
+ close(fd);
+}
+
+TEST(PlasmaSerialization, GetRequest) {
+ int fd = create_temp_file();
+ ObjectID object_ids[2];
+ object_ids[0] = ObjectID::from_random();
+ object_ids[1] = ObjectID::from_random();
+ int64_t timeout_ms = 1234;
+ ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetRequest);
+ std::vector<ObjectID> object_ids_return;
+ int64_t timeout_ms_return;
+ ARROW_CHECK_OK(ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return));
+ ASSERT_EQ(object_ids[0], object_ids_return[0]);
+ ASSERT_EQ(object_ids[1], object_ids_return[1]);
+ ASSERT_EQ(timeout_ms, timeout_ms_return);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, GetReply) {
+ int fd = create_temp_file();
+ ObjectID object_ids[2];
+ object_ids[0] = ObjectID::from_random();
+ object_ids[1] = ObjectID::from_random();
+ std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> plasma_objects;
+ plasma_objects[object_ids[0]] = random_plasma_object();
+ plasma_objects[object_ids[1]] = random_plasma_object();
+ ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaGetReply);
+ ObjectID object_ids_return[2];
+ PlasmaObject plasma_objects_return[2];
+ memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
+ ARROW_CHECK_OK(
+ ReadGetReply(data.data(), object_ids_return, &plasma_objects_return[0], 2));
+ ASSERT_EQ(object_ids[0], object_ids_return[0]);
+ ASSERT_EQ(object_ids[1], object_ids_return[1]);
+ ASSERT_EQ(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
+ sizeof(PlasmaObject)),
+ 0);
+ ASSERT_EQ(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
+ sizeof(PlasmaObject)),
+ 0);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, ReleaseRequest) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1));
+ std::vector<uint8_t> data =
+ read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
+ ObjectID object_id2;
+ ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2));
+ ASSERT_EQ(object_id1, object_id2);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, ReleaseReply) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaReleaseReply);
+ ObjectID object_id2;
+ Status s = ReadReleaseReply(data.data(), &object_id2);
+ ASSERT_EQ(object_id1, object_id2);
+ ASSERT_TRUE(s.IsPlasmaObjectExists());
+ close(fd);
+}
+
+TEST(PlasmaSerialization, DeleteRequest) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
+ ObjectID object_id2;
+ ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2));
+ ASSERT_EQ(object_id1, object_id2);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, DeleteReply) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ int error1 = PlasmaError_ObjectExists;
+ ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDeleteReply);
+ ObjectID object_id2;
+ Status s = ReadDeleteReply(data.data(), &object_id2);
+ ASSERT_EQ(object_id1, object_id2);
+ ASSERT_TRUE(s.IsPlasmaObjectExists());
+ close(fd);
+}
+
+TEST(PlasmaSerialization, StatusRequest) {
+ int fd = create_temp_file();
+ int64_t num_objects = 2;
+ ObjectID object_ids[num_objects];
+ object_ids[0] = ObjectID::from_random();
+ object_ids[1] = ObjectID::from_random();
+ ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusRequest);
+ ObjectID object_ids_read[num_objects];
+ ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects));
+ ASSERT_EQ(object_ids[0], object_ids_read[0]);
+ ASSERT_EQ(object_ids[1], object_ids_read[1]);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, StatusReply) {
+ int fd = create_temp_file();
+ ObjectID object_ids[2];
+ object_ids[0] = ObjectID::from_random();
+ object_ids[1] = ObjectID::from_random();
+ int object_statuses[2] = {42, 43};
+ ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaStatusReply);
+ int64_t num_objects = ReadStatusReply_num_objects(data.data());
+ ObjectID object_ids_read[num_objects];
+ int object_statuses_read[num_objects];
+ ARROW_CHECK_OK(
+ ReadStatusReply(data.data(), object_ids_read, object_statuses_read, num_objects));
+ ASSERT_EQ(object_ids[0], object_ids_read[0]);
+ ASSERT_EQ(object_ids[1], object_ids_read[1]);
+ ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
+ ASSERT_EQ(object_statuses[1], object_statuses_read[1]);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, EvictRequest) {
+ int fd = create_temp_file();
+ int64_t num_bytes = 111;
+ ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictRequest);
+ int64_t num_bytes_received;
+ ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received));
+ ASSERT_EQ(num_bytes, num_bytes_received);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, EvictReply) {
+ int fd = create_temp_file();
+ int64_t num_bytes = 111;
+ ARROW_CHECK_OK(SendEvictReply(fd, num_bytes));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaEvictReply);
+ int64_t num_bytes_received;
+ ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received));
+ ASSERT_EQ(num_bytes, num_bytes_received);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, FetchRequest) {
+ int fd = create_temp_file();
+ ObjectID object_ids[2];
+ object_ids[0] = ObjectID::from_random();
+ object_ids[1] = ObjectID::from_random();
+ ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2));
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaFetchRequest);
+ std::vector<ObjectID> object_ids_read;
+ ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read));
+ ASSERT_EQ(object_ids[0], object_ids_read[0]);
+ ASSERT_EQ(object_ids[1], object_ids_read[1]);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, WaitRequest) {
+ int fd = create_temp_file();
+ const int num_objects_in = 2;
+ ObjectRequest object_requests_in[num_objects_in] = {
+ ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}),
+ ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})};
+ const int num_ready_objects_in = 1;
+ int64_t timeout_ms = 1000;
+
+ ARROW_CHECK_OK(SendWaitRequest(
+ fd, &object_requests_in[0], num_objects_in, num_ready_objects_in, timeout_ms));
+ /* Read message back. */
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitRequest);
+ int num_ready_objects_out;
+ int64_t timeout_ms_read;
+ ObjectRequestMap object_requests_out;
+ ARROW_CHECK_OK(ReadWaitRequest(
+ data.data(), object_requests_out, &timeout_ms_read, &num_ready_objects_out));
+ ASSERT_EQ(num_objects_in, object_requests_out.size());
+ ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
+ for (int i = 0; i < num_objects_in; i++) {
+ const ObjectID& object_id = object_requests_in[i].object_id;
+ ASSERT_EQ(1, object_requests_out.count(object_id));
+ const auto& entry = object_requests_out.find(object_id);
+ ASSERT_TRUE(entry != object_requests_out.end());
+ ASSERT_EQ(entry->second.object_id, object_requests_in[i].object_id);
+ ASSERT_EQ(entry->second.type, object_requests_in[i].type);
+ }
+ close(fd);
+}
+
+TEST(PlasmaSerialization, WaitReply) {
+ int fd = create_temp_file();
+ const int num_objects_in = 2;
+ /* Create a map with two ObjectRequests in it. */
+ ObjectRequestMap objects_in(num_objects_in);
+ ObjectID id1 = ObjectID::from_random();
+ objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local});
+ ObjectID id2 = ObjectID::from_random();
+ objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent});
+
+ ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in));
+ /* Read message back. */
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaWaitReply);
+ ObjectRequest objects_out[2];
+ int num_objects_out;
+ ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out));
+ ASSERT_EQ(num_objects_in, num_objects_out);
+ for (int i = 0; i < num_objects_out; i++) {
+ /* Each object request must appear exactly once. */
+ ASSERT_EQ(objects_in.count(objects_out[i].object_id), 1);
+ const auto& entry = objects_in.find(objects_out[i].object_id);
+ ASSERT_TRUE(entry != objects_in.end());
+ ASSERT_EQ(entry->second.object_id, objects_out[i].object_id);
+ ASSERT_EQ(entry->second.status, objects_out[i].status);
+ }
+ close(fd);
+}
+
+TEST(PlasmaSerialization, DataRequest) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ const char* address1 = "address1";
+ int port1 = 12345;
+ ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1));
+ /* Reading message back. */
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataRequest);
+ ObjectID object_id2;
+ char* address2;
+ int port2;
+ ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2));
+ ASSERT_EQ(object_id1, object_id2);
+ ASSERT_EQ(strcmp(address1, address2), 0);
+ ASSERT_EQ(port1, port2);
+ free(address2);
+ close(fd);
+}
+
+TEST(PlasmaSerialization, DataReply) {
+ int fd = create_temp_file();
+ ObjectID object_id1 = ObjectID::from_random();
+ int64_t object_size1 = 146;
+ int64_t metadata_size1 = 198;
+ ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1));
+ /* Reading message back. */
+ std::vector<uint8_t> data = read_message_from_file(fd, MessageType_PlasmaDataReply);
+ ObjectID object_id2;
+ int64_t object_size2;
+ int64_t metadata_size2;
+ ARROW_CHECK_OK(ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2));
+ ASSERT_EQ(object_id1, object_id2);
+ ASSERT_EQ(object_size1, object_size2);
+ ASSERT_EQ(metadata_size1, metadata_size2);
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/ae/ae.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae.c b/cpp/src/plasma/thirdparty/ae/ae.c
new file mode 100644
index 0000000..e66808a
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae.c
@@ -0,0 +1,465 @@
+/* A simple event-driven programming library. Originally I wrote this code
+ * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
+ * it in form of a library for easy reuse.
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <poll.h>
+#include <string.h>
+#include <time.h>
+#include <errno.h>
+
+#include "ae.h"
+#include "zmalloc.h"
+#include "config.h"
+
+/* Include the best multiplexing layer supported by this system.
+ * The following should be ordered by performances, descending. */
+#ifdef HAVE_EVPORT
+#include "ae_evport.c"
+#else
+ #ifdef HAVE_EPOLL
+ #include "ae_epoll.c"
+ #else
+ #ifdef HAVE_KQUEUE
+ #include "ae_kqueue.c"
+ #else
+ #include "ae_select.c"
+ #endif
+ #endif
+#endif
+
+aeEventLoop *aeCreateEventLoop(int setsize) {
+ aeEventLoop *eventLoop;
+ int i;
+
+ if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
+ eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
+ eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
+ if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
+ eventLoop->setsize = setsize;
+ eventLoop->lastTime = time(NULL);
+ eventLoop->timeEventHead = NULL;
+ eventLoop->timeEventNextId = 0;
+ eventLoop->stop = 0;
+ eventLoop->maxfd = -1;
+ eventLoop->beforesleep = NULL;
+ if (aeApiCreate(eventLoop) == -1) goto err;
+ /* Events with mask == AE_NONE are not set. So let's initialize the
+ * vector with it. */
+ for (i = 0; i < setsize; i++)
+ eventLoop->events[i].mask = AE_NONE;
+ return eventLoop;
+
+err:
+ if (eventLoop) {
+ zfree(eventLoop->events);
+ zfree(eventLoop->fired);
+ zfree(eventLoop);
+ }
+ return NULL;
+}
+
+/* Return the current set size. */
+int aeGetSetSize(aeEventLoop *eventLoop) {
+ return eventLoop->setsize;
+}
+
+/* Resize the maximum set size of the event loop.
+ * If the requested set size is smaller than the current set size, but
+ * there is already a file descriptor in use that is >= the requested
+ * set size minus one, AE_ERR is returned and the operation is not
+ * performed at all.
+ *
+ * Otherwise AE_OK is returned and the operation is successful. */
+int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
+ int i;
+
+ if (setsize == eventLoop->setsize) return AE_OK;
+ if (eventLoop->maxfd >= setsize) return AE_ERR;
+ if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
+
+ eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
+ eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
+ eventLoop->setsize = setsize;
+
+ /* Make sure that if we created new slots, they are initialized with
+ * an AE_NONE mask. */
+ for (i = eventLoop->maxfd+1; i < setsize; i++)
+ eventLoop->events[i].mask = AE_NONE;
+ return AE_OK;
+}
+
+void aeDeleteEventLoop(aeEventLoop *eventLoop) {
+ aeApiFree(eventLoop);
+ zfree(eventLoop->events);
+ zfree(eventLoop->fired);
+ zfree(eventLoop);
+}
+
+void aeStop(aeEventLoop *eventLoop) {
+ eventLoop->stop = 1;
+}
+
+int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
+ aeFileProc *proc, void *clientData)
+{
+ if (fd >= eventLoop->setsize) {
+ errno = ERANGE;
+ return AE_ERR;
+ }
+ aeFileEvent *fe = &eventLoop->events[fd];
+
+ if (aeApiAddEvent(eventLoop, fd, mask) == -1)
+ return AE_ERR;
+ fe->mask |= mask;
+ if (mask & AE_READABLE) fe->rfileProc = proc;
+ if (mask & AE_WRITABLE) fe->wfileProc = proc;
+ fe->clientData = clientData;
+ if (fd > eventLoop->maxfd)
+ eventLoop->maxfd = fd;
+ return AE_OK;
+}
+
+void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
+{
+ if (fd >= eventLoop->setsize) return;
+ aeFileEvent *fe = &eventLoop->events[fd];
+ if (fe->mask == AE_NONE) return;
+
+ aeApiDelEvent(eventLoop, fd, mask);
+ fe->mask = fe->mask & (~mask);
+ if (fd == eventLoop->maxfd && fe->mask == AE_NONE) {
+ /* Update the max fd */
+ int j;
+
+ for (j = eventLoop->maxfd-1; j >= 0; j--)
+ if (eventLoop->events[j].mask != AE_NONE) break;
+ eventLoop->maxfd = j;
+ }
+}
+
+int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
+ if (fd >= eventLoop->setsize) return 0;
+ aeFileEvent *fe = &eventLoop->events[fd];
+
+ return fe->mask;
+}
+
+static void aeGetTime(long *seconds, long *milliseconds)
+{
+ struct timeval tv;
+
+ gettimeofday(&tv, NULL);
+ *seconds = tv.tv_sec;
+ *milliseconds = tv.tv_usec/1000;
+}
+
+static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) {
+ long cur_sec, cur_ms, when_sec, when_ms;
+
+ aeGetTime(&cur_sec, &cur_ms);
+ when_sec = cur_sec + milliseconds/1000;
+ when_ms = cur_ms + milliseconds%1000;
+ if (when_ms >= 1000) {
+ when_sec ++;
+ when_ms -= 1000;
+ }
+ *sec = when_sec;
+ *ms = when_ms;
+}
+
+long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
+ aeTimeProc *proc, void *clientData,
+ aeEventFinalizerProc *finalizerProc)
+{
+ long long id = eventLoop->timeEventNextId++;
+ aeTimeEvent *te;
+
+ te = zmalloc(sizeof(*te));
+ if (te == NULL) return AE_ERR;
+ te->id = id;
+ aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms);
+ te->timeProc = proc;
+ te->finalizerProc = finalizerProc;
+ te->clientData = clientData;
+ te->next = eventLoop->timeEventHead;
+ eventLoop->timeEventHead = te;
+ return id;
+}
+
+int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
+{
+ aeTimeEvent *te = eventLoop->timeEventHead;
+ while(te) {
+ if (te->id == id) {
+ te->id = AE_DELETED_EVENT_ID;
+ return AE_OK;
+ }
+ te = te->next;
+ }
+ return AE_ERR; /* NO event with the specified ID found */
+}
+
+/* Search the first timer to fire.
+ * This operation is useful to know how many time the select can be
+ * put in sleep without to delay any event.
+ * If there are no timers NULL is returned.
+ *
+ * Note that's O(N) since time events are unsorted.
+ * Possible optimizations (not needed by Redis so far, but...):
+ * 1) Insert the event in order, so that the nearest is just the head.
+ * Much better but still insertion or deletion of timers is O(N).
+ * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)).
+ */
+static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
+{
+ aeTimeEvent *te = eventLoop->timeEventHead;
+ aeTimeEvent *nearest = NULL;
+
+ while(te) {
+ if (!nearest || te->when_sec < nearest->when_sec ||
+ (te->when_sec == nearest->when_sec &&
+ te->when_ms < nearest->when_ms))
+ nearest = te;
+ te = te->next;
+ }
+ return nearest;
+}
+
+/* Process time events */
+static int processTimeEvents(aeEventLoop *eventLoop) {
+ int processed = 0;
+ aeTimeEvent *te, *prev;
+ long long maxId;
+ time_t now = time(NULL);
+
+ /* If the system clock is moved to the future, and then set back to the
+ * right value, time events may be delayed in a random way. Often this
+ * means that scheduled operations will not be performed soon enough.
+ *
+ * Here we try to detect system clock skews, and force all the time
+ * events to be processed ASAP when this happens: the idea is that
+ * processing events earlier is less dangerous than delaying them
+ * indefinitely, and practice suggests it is. */
+ if (now < eventLoop->lastTime) {
+ te = eventLoop->timeEventHead;
+ while(te) {
+ te->when_sec = 0;
+ te = te->next;
+ }
+ }
+ eventLoop->lastTime = now;
+
+ prev = NULL;
+ te = eventLoop->timeEventHead;
+ maxId = eventLoop->timeEventNextId-1;
+ while(te) {
+ long now_sec, now_ms;
+ long long id;
+
+ /* Remove events scheduled for deletion. */
+ if (te->id == AE_DELETED_EVENT_ID) {
+ aeTimeEvent *next = te->next;
+ if (prev == NULL)
+ eventLoop->timeEventHead = te->next;
+ else
+ prev->next = te->next;
+ if (te->finalizerProc)
+ te->finalizerProc(eventLoop, te->clientData);
+ zfree(te);
+ te = next;
+ continue;
+ }
+
+ /* Make sure we don't process time events created by time events in
+ * this iteration. Note that this check is currently useless: we always
+ * add new timers on the head, however if we change the implementation
+ * detail, this check may be useful again: we keep it here for future
+ * defense. */
+ if (te->id > maxId) {
+ te = te->next;
+ continue;
+ }
+ aeGetTime(&now_sec, &now_ms);
+ if (now_sec > te->when_sec ||
+ (now_sec == te->when_sec && now_ms >= te->when_ms))
+ {
+ int retval;
+
+ id = te->id;
+ retval = te->timeProc(eventLoop, id, te->clientData);
+ processed++;
+ if (retval != AE_NOMORE) {
+ aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
+ } else {
+ te->id = AE_DELETED_EVENT_ID;
+ }
+ }
+ prev = te;
+ te = te->next;
+ }
+ return processed;
+}
+
+/* Process every pending time event, then every pending file event
+ * (that may be registered by time event callbacks just processed).
+ * Without special flags the function sleeps until some file event
+ * fires, or when the next time event occurs (if any).
+ *
+ * If flags is 0, the function does nothing and returns.
+ * if flags has AE_ALL_EVENTS set, all the kind of events are processed.
+ * if flags has AE_FILE_EVENTS set, file events are processed.
+ * if flags has AE_TIME_EVENTS set, time events are processed.
+ * if flags has AE_DONT_WAIT set the function returns ASAP until all
+ * the events that's possible to process without to wait are processed.
+ *
+ * The function returns the number of events processed. */
+int aeProcessEvents(aeEventLoop *eventLoop, int flags)
+{
+ int processed = 0, numevents;
+
+ /* Nothing to do? return ASAP */
+ if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0;
+
+ /* Note that we want call select() even if there are no
+ * file events to process as long as we want to process time
+ * events, in order to sleep until the next time event is ready
+ * to fire. */
+ if (eventLoop->maxfd != -1 ||
+ ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) {
+ int j;
+ aeTimeEvent *shortest = NULL;
+ struct timeval tv, *tvp;
+
+ if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT))
+ shortest = aeSearchNearestTimer(eventLoop);
+ if (shortest) {
+ long now_sec, now_ms;
+
+ aeGetTime(&now_sec, &now_ms);
+ tvp = &tv;
+
+ /* How many milliseconds we need to wait for the next
+ * time event to fire? */
+ long long ms =
+ (shortest->when_sec - now_sec)*1000 +
+ shortest->when_ms - now_ms;
+
+ if (ms > 0) {
+ tvp->tv_sec = ms/1000;
+ tvp->tv_usec = (ms % 1000)*1000;
+ } else {
+ tvp->tv_sec = 0;
+ tvp->tv_usec = 0;
+ }
+ } else {
+ /* If we have to check for events but need to return
+ * ASAP because of AE_DONT_WAIT we need to set the timeout
+ * to zero */
+ if (flags & AE_DONT_WAIT) {
+ tv.tv_sec = tv.tv_usec = 0;
+ tvp = &tv;
+ } else {
+ /* Otherwise we can block */
+ tvp = NULL; /* wait forever */
+ }
+ }
+
+ numevents = aeApiPoll(eventLoop, tvp);
+ for (j = 0; j < numevents; j++) {
+ aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd];
+ int mask = eventLoop->fired[j].mask;
+ int fd = eventLoop->fired[j].fd;
+ int rfired = 0;
+
+ /* note the fe->mask & mask & ... code: maybe an already processed
+ * event removed an element that fired and we still didn't
+ * processed, so we check if the event is still valid. */
+ if (fe->mask & mask & AE_READABLE) {
+ rfired = 1;
+ fe->rfileProc(eventLoop,fd,fe->clientData,mask);
+ }
+ if (fe->mask & mask & AE_WRITABLE) {
+ if (!rfired || fe->wfileProc != fe->rfileProc)
+ fe->wfileProc(eventLoop,fd,fe->clientData,mask);
+ }
+ processed++;
+ }
+ }
+ /* Check time events */
+ if (flags & AE_TIME_EVENTS)
+ processed += processTimeEvents(eventLoop);
+
+ return processed; /* return the number of processed file/time events */
+}
+
+/* Wait for milliseconds until the given file descriptor becomes
+ * writable/readable/exception */
+int aeWait(int fd, int mask, long long milliseconds) {
+ struct pollfd pfd;
+ int retmask = 0, retval;
+
+ memset(&pfd, 0, sizeof(pfd));
+ pfd.fd = fd;
+ if (mask & AE_READABLE) pfd.events |= POLLIN;
+ if (mask & AE_WRITABLE) pfd.events |= POLLOUT;
+
+ if ((retval = poll(&pfd, 1, milliseconds))== 1) {
+ if (pfd.revents & POLLIN) retmask |= AE_READABLE;
+ if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE;
+ if (pfd.revents & POLLERR) retmask |= AE_WRITABLE;
+ if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE;
+ return retmask;
+ } else {
+ return retval;
+ }
+}
+
+void aeMain(aeEventLoop *eventLoop) {
+ eventLoop->stop = 0;
+ while (!eventLoop->stop) {
+ if (eventLoop->beforesleep != NULL)
+ eventLoop->beforesleep(eventLoop);
+ aeProcessEvents(eventLoop, AE_ALL_EVENTS);
+ }
+}
+
+char *aeGetApiName(void) {
+ return aeApiName();
+}
+
+void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) {
+ eventLoop->beforesleep = beforesleep;
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/ae/ae.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae.h b/cpp/src/plasma/thirdparty/ae/ae.h
new file mode 100644
index 0000000..827c4c9
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae.h
@@ -0,0 +1,123 @@
+/* A simple event-driven programming library. Originally I wrote this code
+ * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
+ * it in form of a library for easy reuse.
+ *
+ * Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __AE_H__
+#define __AE_H__
+
+#include <time.h>
+
+#define AE_OK 0
+#define AE_ERR -1
+
+#define AE_NONE 0
+#define AE_READABLE 1
+#define AE_WRITABLE 2
+
+#define AE_FILE_EVENTS 1
+#define AE_TIME_EVENTS 2
+#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
+#define AE_DONT_WAIT 4
+
+#define AE_NOMORE -1
+#define AE_DELETED_EVENT_ID -1
+
+/* Macros */
+#define AE_NOTUSED(V) ((void) V)
+
+struct aeEventLoop;
+
+/* Types and data structures */
+typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
+typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData);
+typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData);
+typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop);
+
+/* File event structure */
+typedef struct aeFileEvent {
+ int mask; /* one of AE_(READABLE|WRITABLE) */
+ aeFileProc *rfileProc;
+ aeFileProc *wfileProc;
+ void *clientData;
+} aeFileEvent;
+
+/* Time event structure */
+typedef struct aeTimeEvent {
+ long long id; /* time event identifier. */
+ long when_sec; /* seconds */
+ long when_ms; /* milliseconds */
+ aeTimeProc *timeProc;
+ aeEventFinalizerProc *finalizerProc;
+ void *clientData;
+ struct aeTimeEvent *next;
+} aeTimeEvent;
+
+/* A fired event */
+typedef struct aeFiredEvent {
+ int fd;
+ int mask;
+} aeFiredEvent;
+
+/* State of an event based program */
+typedef struct aeEventLoop {
+ int maxfd; /* highest file descriptor currently registered */
+ int setsize; /* max number of file descriptors tracked */
+ long long timeEventNextId;
+ time_t lastTime; /* Used to detect system clock skew */
+ aeFileEvent *events; /* Registered events */
+ aeFiredEvent *fired; /* Fired events */
+ aeTimeEvent *timeEventHead;
+ int stop;
+ void *apidata; /* This is used for polling API specific data */
+ aeBeforeSleepProc *beforesleep;
+} aeEventLoop;
+
+/* Prototypes */
+aeEventLoop *aeCreateEventLoop(int setsize);
+void aeDeleteEventLoop(aeEventLoop *eventLoop);
+void aeStop(aeEventLoop *eventLoop);
+int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
+ aeFileProc *proc, void *clientData);
+void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask);
+int aeGetFileEvents(aeEventLoop *eventLoop, int fd);
+long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
+ aeTimeProc *proc, void *clientData,
+ aeEventFinalizerProc *finalizerProc);
+int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id);
+int aeProcessEvents(aeEventLoop *eventLoop, int flags);
+int aeWait(int fd, int mask, long long milliseconds);
+void aeMain(aeEventLoop *eventLoop);
+char *aeGetApiName(void);
+void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
+int aeGetSetSize(aeEventLoop *eventLoop);
+int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
+
+#endif
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/ae/ae_epoll.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/ae/ae_epoll.c b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
new file mode 100644
index 0000000..410aac7
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/ae/ae_epoll.c
@@ -0,0 +1,135 @@
+/* Linux epoll(2) based ae.c module
+ *
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/epoll.h>
+
+typedef struct aeApiState {
+ int epfd;
+ struct epoll_event *events;
+} aeApiState;
+
+static int aeApiCreate(aeEventLoop *eventLoop) {
+ aeApiState *state = zmalloc(sizeof(aeApiState));
+
+ if (!state) return -1;
+ state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize);
+ if (!state->events) {
+ zfree(state);
+ return -1;
+ }
+ state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */
+ if (state->epfd == -1) {
+ zfree(state->events);
+ zfree(state);
+ return -1;
+ }
+ eventLoop->apidata = state;
+ return 0;
+}
+
+static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+ aeApiState *state = eventLoop->apidata;
+
+ state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize);
+ return 0;
+}
+
+static void aeApiFree(aeEventLoop *eventLoop) {
+ aeApiState *state = eventLoop->apidata;
+
+ close(state->epfd);
+ zfree(state->events);
+ zfree(state);
+}
+
+static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
+ aeApiState *state = eventLoop->apidata;
+ struct epoll_event ee = {0}; /* avoid valgrind warning */
+ /* If the fd was already monitored for some event, we need a MOD
+ * operation. Otherwise we need an ADD operation. */
+ int op = eventLoop->events[fd].mask == AE_NONE ?
+ EPOLL_CTL_ADD : EPOLL_CTL_MOD;
+
+ ee.events = 0;
+ mask |= eventLoop->events[fd].mask; /* Merge old events */
+ if (mask & AE_READABLE) ee.events |= EPOLLIN;
+ if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
+ ee.data.fd = fd;
+ if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1;
+ return 0;
+}
+
+static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
+ aeApiState *state = eventLoop->apidata;
+ struct epoll_event ee = {0}; /* avoid valgrind warning */
+ int mask = eventLoop->events[fd].mask & (~delmask);
+
+ ee.events = 0;
+ if (mask & AE_READABLE) ee.events |= EPOLLIN;
+ if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
+ ee.data.fd = fd;
+ if (mask != AE_NONE) {
+ epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee);
+ } else {
+ /* Note, Kernel < 2.6.9 requires a non null event pointer even for
+ * EPOLL_CTL_DEL. */
+ epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee);
+ }
+}
+
+static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
+ aeApiState *state = eventLoop->apidata;
+ int retval, numevents = 0;
+
+ retval = epoll_wait(state->epfd,state->events,eventLoop->setsize,
+ tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
+ if (retval > 0) {
+ int j;
+
+ numevents = retval;
+ for (j = 0; j < numevents; j++) {
+ int mask = 0;
+ struct epoll_event *e = state->events+j;
+
+ if (e->events & EPOLLIN) mask |= AE_READABLE;
+ if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
+ if (e->events & EPOLLERR) mask |= AE_WRITABLE;
+ if (e->events & EPOLLHUP) mask |= AE_WRITABLE;
+ eventLoop->fired[j].fd = e->data.fd;
+ eventLoop->fired[j].mask = mask;
+ }
+ }
+ return numevents;
+}
+
+static char *aeApiName(void) {
+ return "epoll";
+}
[10/14] arrow git commit: [C++] Restore Plasma source tree after
0.5.0 release
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/dlmalloc.c
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/dlmalloc.c b/cpp/src/plasma/thirdparty/dlmalloc.c
new file mode 100644
index 0000000..84ccbd2
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/dlmalloc.c
@@ -0,0 +1,6281 @@
+/*
+ This is a version (aka dlmalloc) of malloc/free/realloc written by
+ Doug Lea and released to the public domain, as explained at
+ http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
+ comments, complaints, performance data, etc to dl@cs.oswego.edu
+
+* Version 2.8.6 Wed Aug 29 06:57:58 2012 Doug Lea
+ Note: There may be an updated version of this malloc obtainable at
+ ftp://gee.cs.oswego.edu/pub/misc/malloc.c
+ Check before installing!
+
+* Quickstart
+
+ This library is all in one file to simplify the most common usage:
+ ftp it, compile it (-O3), and link it into another program. All of
+ the compile-time options default to reasonable values for use on
+ most platforms. You might later want to step through various
+ compile-time and dynamic tuning options.
+
+ For convenience, an include file for code using this malloc is at:
+ ftp://gee.cs.oswego.edu/pub/misc/malloc-2.8.6.h
+ You don't really need this .h file unless you call functions not
+ defined in your system include files. The .h file contains only the
+ excerpts from this file needed for using this malloc on ANSI C/C++
+ systems, so long as you haven't changed compile-time options about
+ naming and tuning parameters. If you do, then you can create your
+ own malloc.h that does include all settings by cutting at the point
+ indicated below. Note that you may already by default be using a C
+ library containing a malloc that is based on some version of this
+ malloc (for example in linux). You might still want to use the one
+ in this file to customize settings or to avoid overheads associated
+ with library versions.
+
+* Vital statistics:
+
+ Supported pointer/size_t representation: 4 or 8 bytes
+ size_t MUST be an unsigned type of the same width as
+ pointers. (If you are using an ancient system that declares
+ size_t as a signed type, or need it to be a different width
+ than pointers, you can use a previous release of this malloc
+ (e.g. 2.7.2) supporting these.)
+
+ Alignment: 8 bytes (minimum)
+ This suffices for nearly all current machines and C compilers.
+ However, you can define MALLOC_ALIGNMENT to be wider than this
+ if necessary (up to 128bytes), at the expense of using more space.
+
+ Minimum overhead per allocated chunk: 4 or 8 bytes (if 4byte sizes)
+ 8 or 16 bytes (if 8byte sizes)
+ Each malloced chunk has a hidden word of overhead holding size
+ and status information, and additional cross-check word
+ if FOOTERS is defined.
+
+ Minimum allocated size: 4-byte ptrs: 16 bytes (including overhead)
+ 8-byte ptrs: 32 bytes (including overhead)
+
+ Even a request for zero bytes (i.e., malloc(0)) returns a
+ pointer to something of the minimum allocatable size.
+ The maximum overhead wastage (i.e., number of extra bytes
+ allocated than were requested in malloc) is less than or equal
+ to the minimum size, except for requests >= mmap_threshold that
+ are serviced via mmap(), where the worst case wastage is about
+ 32 bytes plus the remainder from a system page (the minimal
+ mmap unit); typically 4096 or 8192 bytes.
+
+ Security: static-safe; optionally more or less
+ The "security" of malloc refers to the ability of malicious
+ code to accentuate the effects of errors (for example, freeing
+ space that is not currently malloc'ed or overwriting past the
+ ends of chunks) in code that calls malloc. This malloc
+ guarantees not to modify any memory locations below the base of
+ heap, i.e., static variables, even in the presence of usage
+ errors. The routines additionally detect most improper frees
+ and reallocs. All this holds as long as the static bookkeeping
+ for malloc itself is not corrupted by some other means. This
+ is only one aspect of security -- these checks do not, and
+ cannot, detect all possible programming errors.
+
+ If FOOTERS is defined nonzero, then each allocated chunk
+ carries an additional check word to verify that it was malloced
+ from its space. These check words are the same within each
+ execution of a program using malloc, but differ across
+ executions, so externally crafted fake chunks cannot be
+ freed. This improves security by rejecting frees/reallocs that
+ could corrupt heap memory, in addition to the checks preventing
+ writes to statics that are always on. This may further improve
+ security at the expense of time and space overhead. (Note that
+ FOOTERS may also be worth using with MSPACES.)
+
+ By default detected errors cause the program to abort (calling
+ "abort()"). You can override this to instead proceed past
+ errors by defining PROCEED_ON_ERROR. In this case, a bad free
+ has no effect, and a malloc that encounters a bad address
+ caused by user overwrites will ignore the bad address by
+ dropping pointers and indices to all known memory. This may
+ be appropriate for programs that should continue if at all
+ possible in the face of programming errors, although they may
+ run out of memory because dropped memory is never reclaimed.
+
+ If you don't like either of these options, you can define
+ CORRUPTION_ERROR_ACTION and USAGE_ERROR_ACTION to do anything
+ else. And if if you are sure that your program using malloc has
+ no errors or vulnerabilities, you can define INSECURE to 1,
+ which might (or might not) provide a small performance improvement.
+
+ It is also possible to limit the maximum total allocatable
+ space, using malloc_set_footprint_limit. This is not
+ designed as a security feature in itself (calls to set limits
+ are not screened or privileged), but may be useful as one
+ aspect of a secure implementation.
+
+ Thread-safety: NOT thread-safe unless USE_LOCKS defined non-zero
+ When USE_LOCKS is defined, each public call to malloc, free,
+ etc is surrounded with a lock. By default, this uses a plain
+ pthread mutex, win32 critical section, or a spin-lock if if
+ available for the platform and not disabled by setting
+ USE_SPIN_LOCKS=0. However, if USE_RECURSIVE_LOCKS is defined,
+ recursive versions are used instead (which are not required for
+ base functionality but may be needed in layered extensions).
+ Using a global lock is not especially fast, and can be a major
+ bottleneck. It is designed only to provide minimal protection
+ in concurrent environments, and to provide a basis for
+ extensions. If you are using malloc in a concurrent program,
+ consider instead using nedmalloc
+ (http://www.nedprod.com/programs/portable/nedmalloc/) or
+ ptmalloc (See http://www.malloc.de), which are derived from
+ versions of this malloc.
+
+ System requirements: Any combination of MORECORE and/or MMAP/MUNMAP
+ This malloc can use unix sbrk or any emulation (invoked using
+ the CALL_MORECORE macro) and/or mmap/munmap or any emulation
+ (invoked using CALL_MMAP/CALL_MUNMAP) to get and release system
+ memory. On most unix systems, it tends to work best if both
+ MORECORE and MMAP are enabled. On Win32, it uses emulations
+ based on VirtualAlloc. It also uses common C library functions
+ like memset.
+
+ Compliance: I believe it is compliant with the Single Unix Specification
+ (See http://www.unix.org). Also SVID/XPG, ANSI C, and probably
+ others as well.
+
+* Overview of algorithms
+
+ This is not the fastest, most space-conserving, most portable, or
+ most tunable malloc ever written. However it is among the fastest
+ while also being among the most space-conserving, portable and
+ tunable. Consistent balance across these factors results in a good
+ general-purpose allocator for malloc-intensive programs.
+
+ In most ways, this malloc is a best-fit allocator. Generally, it
+ chooses the best-fitting existing chunk for a request, with ties
+ broken in approximately least-recently-used order. (This strategy
+ normally maintains low fragmentation.) However, for requests less
+ than 256bytes, it deviates from best-fit when there is not an
+ exactly fitting available chunk by preferring to use space adjacent
+ to that used for the previous small request, as well as by breaking
+ ties in approximately most-recently-used order. (These enhance
+ locality of series of small allocations.) And for very large requests
+ (>= 256Kb by default), it relies on system memory mapping
+ facilities, if supported. (This helps avoid carrying around and
+ possibly fragmenting memory used only for large chunks.)
+
+ All operations (except malloc_stats and mallinfo) have execution
+ times that are bounded by a constant factor of the number of bits in
+ a size_t, not counting any clearing in calloc or copying in realloc,
+ or actions surrounding MORECORE and MMAP that have times
+ proportional to the number of non-contiguous regions returned by
+ system allocation routines, which is often just 1. In real-time
+ applications, you can optionally suppress segment traversals using
+ NO_SEGMENT_TRAVERSAL, which assures bounded execution even when
+ system allocators return non-contiguous spaces, at the typical
+ expense of carrying around more memory and increased fragmentation.
+
+ The implementation is not very modular and seriously overuses
+ macros. Perhaps someday all C compilers will do as good a job
+ inlining modular code as can now be done by brute-force expansion,
+ but now, enough of them seem not to.
+
+ Some compilers issue a lot of warnings about code that is
+ dead/unreachable only on some platforms, and also about intentional
+ uses of negation on unsigned types. All known cases of each can be
+ ignored.
+
+ For a longer but out of date high-level description, see
+ http://gee.cs.oswego.edu/dl/html/malloc.html
+
+* MSPACES
+ If MSPACES is defined, then in addition to malloc, free, etc.,
+ this file also defines mspace_malloc, mspace_free, etc. These
+ are versions of malloc routines that take an "mspace" argument
+ obtained using create_mspace, to control all internal bookkeeping.
+ If ONLY_MSPACES is defined, only these versions are compiled.
+ So if you would like to use this allocator for only some allocations,
+ and your system malloc for others, you can compile with
+ ONLY_MSPACES and then do something like...
+ static mspace mymspace = create_mspace(0,0); // for example
+ #define mymalloc(bytes) mspace_malloc(mymspace, bytes)
+
+ (Note: If you only need one instance of an mspace, you can instead
+ use "USE_DL_PREFIX" to relabel the global malloc.)
+
+ You can similarly create thread-local allocators by storing
+ mspaces as thread-locals. For example:
+ static __thread mspace tlms = 0;
+ void* tlmalloc(size_t bytes) {
+ if (tlms == 0) tlms = create_mspace(0, 0);
+ return mspace_malloc(tlms, bytes);
+ }
+ void tlfree(void* mem) { mspace_free(tlms, mem); }
+
+ Unless FOOTERS is defined, each mspace is completely independent.
+ You cannot allocate from one and free to another (although
+ conformance is only weakly checked, so usage errors are not always
+ caught). If FOOTERS is defined, then each chunk carries around a tag
+ indicating its originating mspace, and frees are directed to their
+ originating spaces. Normally, this requires use of locks.
+
+ ------------------------- Compile-time options ---------------------------
+
+Be careful in setting #define values for numerical constants of type
+size_t. On some systems, literal values are not automatically extended
+to size_t precision unless they are explicitly casted. You can also
+use the symbolic values MAX_SIZE_T, SIZE_T_ONE, etc below.
+
+WIN32 default: defined if _WIN32 defined
+ Defining WIN32 sets up defaults for MS environment and compilers.
+ Otherwise defaults are for unix. Beware that there seem to be some
+ cases where this malloc might not be a pure drop-in replacement for
+ Win32 malloc: Random-looking failures from Win32 GDI API's (eg;
+ SetDIBits()) may be due to bugs in some video driver implementations
+ when pixel buffers are malloc()ed, and the region spans more than
+ one VirtualAlloc()ed region. Because dlmalloc uses a small (64Kb)
+ default granularity, pixel buffers may straddle virtual allocation
+ regions more often than when using the Microsoft allocator. You can
+ avoid this by using VirtualAlloc() and VirtualFree() for all pixel
+ buffers rather than using malloc(). If this is not possible,
+ recompile this malloc with a larger DEFAULT_GRANULARITY. Note:
+ in cases where MSC and gcc (cygwin) are known to differ on WIN32,
+ conditions use _MSC_VER to distinguish them.
+
+DLMALLOC_EXPORT default: extern
+ Defines how public APIs are declared. If you want to export via a
+ Windows DLL, you might define this as
+ #define DLMALLOC_EXPORT extern __declspec(dllexport)
+ If you want a POSIX ELF shared object, you might use
+ #define DLMALLOC_EXPORT extern __attribute__((visibility("default")))
+
+MALLOC_ALIGNMENT default: (size_t)(2 * sizeof(void *))
+ Controls the minimum alignment for malloc'ed chunks. It must be a
+ power of two and at least 8, even on machines for which smaller
+ alignments would suffice. It may be defined as larger than this
+ though. Note however that code and data structures are optimized for
+ the case of 8-byte alignment.
+
+MSPACES default: 0 (false)
+ If true, compile in support for independent allocation spaces.
+ This is only supported if HAVE_MMAP is true.
+
+ONLY_MSPACES default: 0 (false)
+ If true, only compile in mspace versions, not regular versions.
+
+USE_LOCKS default: 0 (false)
+ Causes each call to each public routine to be surrounded with
+ pthread or WIN32 mutex lock/unlock. (If set true, this can be
+ overridden on a per-mspace basis for mspace versions.) If set to a
+ non-zero value other than 1, locks are used, but their
+ implementation is left out, so lock functions must be supplied manually,
+ as described below.
+
+USE_SPIN_LOCKS default: 1 iff USE_LOCKS and spin locks available
+ If true, uses custom spin locks for locking. This is currently
+ supported only gcc >= 4.1, older gccs on x86 platforms, and recent
+ MS compilers. Otherwise, posix locks or win32 critical sections are
+ used.
+
+USE_RECURSIVE_LOCKS default: not defined
+ If defined nonzero, uses recursive (aka reentrant) locks, otherwise
+ uses plain mutexes. This is not required for malloc proper, but may
+ be needed for layered allocators such as nedmalloc.
+
+LOCK_AT_FORK default: not defined
+ If defined nonzero, performs pthread_atfork upon initialization
+ to initialize child lock while holding parent lock. The implementation
+ assumes that pthread locks (not custom locks) are being used. In other
+ cases, you may need to customize the implementation.
+
+FOOTERS default: 0
+ If true, provide extra checking and dispatching by placing
+ information in the footers of allocated chunks. This adds
+ space and time overhead.
+
+INSECURE default: 0
+ If true, omit checks for usage errors and heap space overwrites.
+
+USE_DL_PREFIX default: NOT defined
+ Causes compiler to prefix all public routines with the string 'dl'.
+ This can be useful when you only want to use this malloc in one part
+ of a program, using your regular system malloc elsewhere.
+
+MALLOC_INSPECT_ALL default: NOT defined
+ If defined, compiles malloc_inspect_all and mspace_inspect_all, that
+ perform traversal of all heap space. Unless access to these
+ functions is otherwise restricted, you probably do not want to
+ include them in secure implementations.
+
+ABORT default: defined as abort()
+ Defines how to abort on failed checks. On most systems, a failed
+ check cannot die with an "assert" or even print an informative
+ message, because the underlying print routines in turn call malloc,
+ which will fail again. Generally, the best policy is to simply call
+ abort(). It's not very useful to do more than this because many
+ errors due to overwriting will show up as address faults (null, odd
+ addresses etc) rather than malloc-triggered checks, so will also
+ abort. Also, most compilers know that abort() does not return, so
+ can better optimize code conditionally calling it.
+
+PROCEED_ON_ERROR default: defined as 0 (false)
+ Controls whether detected bad addresses cause them to bypassed
+ rather than aborting. If set, detected bad arguments to free and
+ realloc are ignored. And all bookkeeping information is zeroed out
+ upon a detected overwrite of freed heap space, thus losing the
+ ability to ever return it from malloc again, but enabling the
+ application to proceed. If PROCEED_ON_ERROR is defined, the
+ static variable malloc_corruption_error_count is compiled in
+ and can be examined to see if errors have occurred. This option
+ generates slower code than the default abort policy.
+
+DEBUG default: NOT defined
+ The DEBUG setting is mainly intended for people trying to modify
+ this code or diagnose problems when porting to new platforms.
+ However, it may also be able to better isolate user errors than just
+ using runtime checks. The assertions in the check routines spell
+ out in more detail the assumptions and invariants underlying the
+ algorithms. The checking is fairly extensive, and will slow down
+ execution noticeably. Calling malloc_stats or mallinfo with DEBUG
+ set will attempt to check every non-mmapped allocated and free chunk
+ in the course of computing the summaries.
+
+ABORT_ON_ASSERT_FAILURE default: defined as 1 (true)
+ Debugging assertion failures can be nearly impossible if your
+ version of the assert macro causes malloc to be called, which will
+ lead to a cascade of further failures, blowing the runtime stack.
+ ABORT_ON_ASSERT_FAILURE cause assertions failures to call abort(),
+ which will usually make debugging easier.
+
+MALLOC_FAILURE_ACTION default: sets errno to ENOMEM, or no-op on win32
+ The action to take before "return 0" when malloc fails to be able to
+ return memory because there is none available.
+
+HAVE_MORECORE default: 1 (true) unless win32 or ONLY_MSPACES
+ True if this system supports sbrk or an emulation of it.
+
+MORECORE default: sbrk
+ The name of the sbrk-style system routine to call to obtain more
+ memory. See below for guidance on writing custom MORECORE
+ functions. The type of the argument to sbrk/MORECORE varies across
+ systems. It cannot be size_t, because it supports negative
+ arguments, so it is normally the signed type of the same width as
+ size_t (sometimes declared as "intptr_t"). It doesn't much matter
+ though. Internally, we only call it with arguments less than half
+ the max value of a size_t, which should work across all reasonable
+ possibilities, although sometimes generating compiler warnings.
+
+MORECORE_CONTIGUOUS default: 1 (true) if HAVE_MORECORE
+ If true, take advantage of fact that consecutive calls to MORECORE
+ with positive arguments always return contiguous increasing
+ addresses. This is true of unix sbrk. It does not hurt too much to
+ set it true anyway, since malloc copes with non-contiguities.
+ Setting it false when definitely non-contiguous saves time
+ and possibly wasted space it would take to discover this though.
+
+MORECORE_CANNOT_TRIM default: NOT defined
+ True if MORECORE cannot release space back to the system when given
+ negative arguments. This is generally necessary only if you are
+ using a hand-crafted MORECORE function that cannot handle negative
+ arguments.
+
+NO_SEGMENT_TRAVERSAL default: 0
+ If non-zero, suppresses traversals of memory segments
+ returned by either MORECORE or CALL_MMAP. This disables
+ merging of segments that are contiguous, and selectively
+ releasing them to the OS if unused, but bounds execution times.
+
+HAVE_MMAP default: 1 (true)
+ True if this system supports mmap or an emulation of it. If so, and
+ HAVE_MORECORE is not true, MMAP is used for all system
+ allocation. If set and HAVE_MORECORE is true as well, MMAP is
+ primarily used to directly allocate very large blocks. It is also
+ used as a backup strategy in cases where MORECORE fails to provide
+ space from system. Note: A single call to MUNMAP is assumed to be
+ able to unmap memory that may have be allocated using multiple calls
+ to MMAP, so long as they are adjacent.
+
+HAVE_MREMAP default: 1 on linux, else 0
+ If true realloc() uses mremap() to re-allocate large blocks and
+ extend or shrink allocation spaces.
+
+MMAP_CLEARS default: 1 except on WINCE.
+ True if mmap clears memory so calloc doesn't need to. This is true
+ for standard unix mmap using /dev/zero and on WIN32 except for WINCE.
+
+USE_BUILTIN_FFS default: 0 (i.e., not used)
+ Causes malloc to use the builtin ffs() function to compute indices.
+ Some compilers may recognize and intrinsify ffs to be faster than the
+ supplied C version. Also, the case of x86 using gcc is special-cased
+ to an asm instruction, so is already as fast as it can be, and so
+ this setting has no effect. Similarly for Win32 under recent MS compilers.
+ (On most x86s, the asm version is only slightly faster than the C version.)
+
+malloc_getpagesize default: derive from system includes, or 4096.
+ The system page size. To the extent possible, this malloc manages
+ memory from the system in page-size units. This may be (and
+ usually is) a function rather than a constant. This is ignored
+ if WIN32, where page size is determined using getSystemInfo during
+ initialization.
+
+USE_DEV_RANDOM default: 0 (i.e., not used)
+ Causes malloc to use /dev/random to initialize secure magic seed for
+ stamping footers. Otherwise, the current time is used.
+
+NO_MALLINFO default: 0
+ If defined, don't compile "mallinfo". This can be a simple way
+ of dealing with mismatches between system declarations and
+ those in this file.
+
+MALLINFO_FIELD_TYPE default: size_t
+ The type of the fields in the mallinfo struct. This was originally
+ defined as "int" in SVID etc, but is more usefully defined as
+ size_t. The value is used only if HAVE_USR_INCLUDE_MALLOC_H is not set
+
+NO_MALLOC_STATS default: 0
+ If defined, don't compile "malloc_stats". This avoids calls to
+ fprintf and bringing in stdio dependencies you might not want.
+
+REALLOC_ZERO_BYTES_FREES default: not defined
+ This should be set if a call to realloc with zero bytes should
+ be the same as a call to free. Some people think it should. Otherwise,
+ since this malloc returns a unique pointer for malloc(0), so does
+ realloc(p, 0).
+
+LACKS_UNISTD_H, LACKS_FCNTL_H, LACKS_SYS_PARAM_H, LACKS_SYS_MMAN_H
+LACKS_STRINGS_H, LACKS_STRING_H, LACKS_SYS_TYPES_H, LACKS_ERRNO_H
+LACKS_STDLIB_H LACKS_SCHED_H LACKS_TIME_H default: NOT defined unless on WIN32
+ Define these if your system does not have these header files.
+ You might need to manually insert some of the declarations they provide.
+
+DEFAULT_GRANULARITY default: page size if MORECORE_CONTIGUOUS,
+ system_info.dwAllocationGranularity in WIN32,
+ otherwise 64K.
+ Also settable using mallopt(M_GRANULARITY, x)
+ The unit for allocating and deallocating memory from the system. On
+ most systems with contiguous MORECORE, there is no reason to
+ make this more than a page. However, systems with MMAP tend to
+ either require or encourage larger granularities. You can increase
+ this value to prevent system allocation functions to be called so
+ often, especially if they are slow. The value must be at least one
+ page and must be a power of two. Setting to 0 causes initialization
+ to either page size or win32 region size. (Note: In previous
+ versions of malloc, the equivalent of this option was called
+ "TOP_PAD")
+
+DEFAULT_TRIM_THRESHOLD default: 2MB
+ Also settable using mallopt(M_TRIM_THRESHOLD, x)
+ The maximum amount of unused top-most memory to keep before
+ releasing via malloc_trim in free(). Automatic trimming is mainly
+ useful in long-lived programs using contiguous MORECORE. Because
+ trimming via sbrk can be slow on some systems, and can sometimes be
+ wasteful (in cases where programs immediately afterward allocate
+ more large chunks) the value should be high enough so that your
+ overall system performance would improve by releasing this much
+ memory. As a rough guide, you might set to a value close to the
+ average size of a process (program) running on your system.
+ Releasing this much memory would allow such a process to run in
+ memory. Generally, it is worth tuning trim thresholds when a
+ program undergoes phases where several large chunks are allocated
+ and released in ways that can reuse each other's storage, perhaps
+ mixed with phases where there are no such chunks at all. The trim
+ value must be greater than page size to have any useful effect. To
+ disable trimming completely, you can set to MAX_SIZE_T. Note that the trick
+ some people use of mallocing a huge space and then freeing it at
+ program startup, in an attempt to reserve system memory, doesn't
+ have the intended effect under automatic trimming, since that memory
+ will immediately be returned to the system.
+
+DEFAULT_MMAP_THRESHOLD default: 256K
+ Also settable using mallopt(M_MMAP_THRESHOLD, x)
+ The request size threshold for using MMAP to directly service a
+ request. Requests of at least this size that cannot be allocated
+ using already-existing space will be serviced via mmap. (If enough
+ normal freed space already exists it is used instead.) Using mmap
+ segregates relatively large chunks of memory so that they can be
+ individually obtained and released from the host system. A request
+ serviced through mmap is never reused by any other request (at least
+ not directly; the system may just so happen to remap successive
+ requests to the same locations). Segregating space in this way has
+ the benefits that: Mmapped space can always be individually released
+ back to the system, which helps keep the system level memory demands
+ of a long-lived program low. Also, mapped memory doesn't become
+ `locked' between other chunks, as can happen with normally allocated
+ chunks, which means that even trimming via malloc_trim would not
+ release them. However, it has the disadvantage that the space
+ cannot be reclaimed, consolidated, and then used to service later
+ requests, as happens with normal chunks. The advantages of mmap
+ nearly always outweigh disadvantages for "large" chunks, but the
+ value of "large" may vary across systems. The default is an
+ empirically derived value that works well in most systems. You can
+ disable mmap by setting to MAX_SIZE_T.
+
+MAX_RELEASE_CHECK_RATE default: 4095 unless not HAVE_MMAP
+ The number of consolidated frees between checks to release
+ unused segments when freeing. When using non-contiguous segments,
+ especially with multiple mspaces, checking only for topmost space
+ doesn't always suffice to trigger trimming. To compensate for this,
+ free() will, with a period of MAX_RELEASE_CHECK_RATE (or the
+ current number of segments, if greater) try to release unused
+ segments to the OS when freeing chunks that result in
+ consolidation. The best value for this parameter is a compromise
+ between slowing down frees with relatively costly checks that
+ rarely trigger versus holding on to unused memory. To effectively
+ disable, set to MAX_SIZE_T. This may lead to a very slight speed
+ improvement at the expense of carrying around more memory.
+*/
+
+/* Version identifier to allow people to support multiple versions */
+#ifndef DLMALLOC_VERSION
+#define DLMALLOC_VERSION 20806
+#endif /* DLMALLOC_VERSION */
+
+#ifndef DLMALLOC_EXPORT
+#define DLMALLOC_EXPORT extern
+#endif
+
+#ifndef WIN32
+#ifdef _WIN32
+#define WIN32 1
+#endif /* _WIN32 */
+#ifdef _WIN32_WCE
+#define LACKS_FCNTL_H
+#define WIN32 1
+#endif /* _WIN32_WCE */
+#endif /* WIN32 */
+#ifdef WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <tchar.h>
+#define HAVE_MMAP 1
+#define HAVE_MORECORE 0
+#define LACKS_UNISTD_H
+#define LACKS_SYS_PARAM_H
+#define LACKS_SYS_MMAN_H
+#define LACKS_STRING_H
+#define LACKS_STRINGS_H
+#define LACKS_SYS_TYPES_H
+#define LACKS_ERRNO_H
+#define LACKS_SCHED_H
+#ifndef MALLOC_FAILURE_ACTION
+#define MALLOC_FAILURE_ACTION
+#endif /* MALLOC_FAILURE_ACTION */
+#ifndef MMAP_CLEARS
+#ifdef _WIN32_WCE /* WINCE reportedly does not clear */
+#define MMAP_CLEARS 0
+#else
+#define MMAP_CLEARS 1
+#endif /* _WIN32_WCE */
+#endif /*MMAP_CLEARS */
+#endif /* WIN32 */
+
+#if defined(DARWIN) || defined(_DARWIN)
+/* Mac OSX docs advise not to use sbrk; it seems better to use mmap */
+#ifndef HAVE_MORECORE
+#define HAVE_MORECORE 0
+#define HAVE_MMAP 1
+/* OSX allocators provide 16 byte alignment */
+#ifndef MALLOC_ALIGNMENT
+#define MALLOC_ALIGNMENT ((size_t)16U)
+#endif
+#endif /* HAVE_MORECORE */
+#endif /* DARWIN */
+
+#ifndef LACKS_SYS_TYPES_H
+#include <sys/types.h> /* For size_t */
+#endif /* LACKS_SYS_TYPES_H */
+
+/* The maximum possible size_t value has all bits set */
+#define MAX_SIZE_T (~(size_t)0)
+
+#ifndef USE_LOCKS /* ensure true if spin or recursive locks set */
+#define USE_LOCKS ((defined(USE_SPIN_LOCKS) && USE_SPIN_LOCKS != 0) || \
+ (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0))
+#endif /* USE_LOCKS */
+
+#if USE_LOCKS /* Spin locks for gcc >= 4.1, older gcc on x86, MSC >= 1310 */
+#if ((defined(__GNUC__) && \
+ ((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) || \
+ defined(__i386__) || defined(__x86_64__))) || \
+ (defined(_MSC_VER) && _MSC_VER>=1310))
+#ifndef USE_SPIN_LOCKS
+#define USE_SPIN_LOCKS 1
+#endif /* USE_SPIN_LOCKS */
+#elif USE_SPIN_LOCKS
+#error "USE_SPIN_LOCKS defined without implementation"
+#endif /* ... locks available... */
+#elif !defined(USE_SPIN_LOCKS)
+#define USE_SPIN_LOCKS 0
+#endif /* USE_LOCKS */
+
+#ifndef ONLY_MSPACES
+#define ONLY_MSPACES 0
+#endif /* ONLY_MSPACES */
+#ifndef MSPACES
+#if ONLY_MSPACES
+#define MSPACES 1
+#else /* ONLY_MSPACES */
+#define MSPACES 0
+#endif /* ONLY_MSPACES */
+#endif /* MSPACES */
+#ifndef MALLOC_ALIGNMENT
+#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *)))
+#endif /* MALLOC_ALIGNMENT */
+#ifndef FOOTERS
+#define FOOTERS 0
+#endif /* FOOTERS */
+#ifndef ABORT
+#define ABORT abort()
+#endif /* ABORT */
+#ifndef ABORT_ON_ASSERT_FAILURE
+#define ABORT_ON_ASSERT_FAILURE 1
+#endif /* ABORT_ON_ASSERT_FAILURE */
+#ifndef PROCEED_ON_ERROR
+#define PROCEED_ON_ERROR 0
+#endif /* PROCEED_ON_ERROR */
+
+#ifndef INSECURE
+#define INSECURE 0
+#endif /* INSECURE */
+#ifndef MALLOC_INSPECT_ALL
+#define MALLOC_INSPECT_ALL 0
+#endif /* MALLOC_INSPECT_ALL */
+#ifndef HAVE_MMAP
+#define HAVE_MMAP 1
+#endif /* HAVE_MMAP */
+#ifndef MMAP_CLEARS
+#define MMAP_CLEARS 1
+#endif /* MMAP_CLEARS */
+#ifndef HAVE_MREMAP
+#ifdef linux
+#define HAVE_MREMAP 1
+#define _GNU_SOURCE /* Turns on mremap() definition */
+#else /* linux */
+#define HAVE_MREMAP 0
+#endif /* linux */
+#endif /* HAVE_MREMAP */
+#ifndef MALLOC_FAILURE_ACTION
+#define MALLOC_FAILURE_ACTION errno = ENOMEM;
+#endif /* MALLOC_FAILURE_ACTION */
+#ifndef HAVE_MORECORE
+#if ONLY_MSPACES
+#define HAVE_MORECORE 0
+#else /* ONLY_MSPACES */
+#define HAVE_MORECORE 1
+#endif /* ONLY_MSPACES */
+#endif /* HAVE_MORECORE */
+#if !HAVE_MORECORE
+#define MORECORE_CONTIGUOUS 0
+#else /* !HAVE_MORECORE */
+#define MORECORE_DEFAULT sbrk
+#ifndef MORECORE_CONTIGUOUS
+#define MORECORE_CONTIGUOUS 1
+#endif /* MORECORE_CONTIGUOUS */
+#endif /* HAVE_MORECORE */
+#ifndef DEFAULT_GRANULARITY
+#if (MORECORE_CONTIGUOUS || defined(WIN32))
+#define DEFAULT_GRANULARITY (0) /* 0 means to compute in init_mparams */
+#else /* MORECORE_CONTIGUOUS */
+#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U)
+#endif /* MORECORE_CONTIGUOUS */
+#endif /* DEFAULT_GRANULARITY */
+#ifndef DEFAULT_TRIM_THRESHOLD
+#ifndef MORECORE_CANNOT_TRIM
+#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U)
+#else /* MORECORE_CANNOT_TRIM */
+#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T
+#endif /* MORECORE_CANNOT_TRIM */
+#endif /* DEFAULT_TRIM_THRESHOLD */
+#ifndef DEFAULT_MMAP_THRESHOLD
+#if HAVE_MMAP
+#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U)
+#else /* HAVE_MMAP */
+#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
+#endif /* HAVE_MMAP */
+#endif /* DEFAULT_MMAP_THRESHOLD */
+#ifndef MAX_RELEASE_CHECK_RATE
+#if HAVE_MMAP
+#define MAX_RELEASE_CHECK_RATE 4095
+#else
+#define MAX_RELEASE_CHECK_RATE MAX_SIZE_T
+#endif /* HAVE_MMAP */
+#endif /* MAX_RELEASE_CHECK_RATE */
+#ifndef USE_BUILTIN_FFS
+#define USE_BUILTIN_FFS 0
+#endif /* USE_BUILTIN_FFS */
+#ifndef USE_DEV_RANDOM
+#define USE_DEV_RANDOM 0
+#endif /* USE_DEV_RANDOM */
+#ifndef NO_MALLINFO
+#define NO_MALLINFO 0
+#endif /* NO_MALLINFO */
+#ifndef MALLINFO_FIELD_TYPE
+#define MALLINFO_FIELD_TYPE size_t
+#endif /* MALLINFO_FIELD_TYPE */
+#ifndef NO_MALLOC_STATS
+#define NO_MALLOC_STATS 0
+#endif /* NO_MALLOC_STATS */
+#ifndef NO_SEGMENT_TRAVERSAL
+#define NO_SEGMENT_TRAVERSAL 0
+#endif /* NO_SEGMENT_TRAVERSAL */
+
+/*
+ mallopt tuning options. SVID/XPG defines four standard parameter
+ numbers for mallopt, normally defined in malloc.h. None of these
+ are used in this malloc, so setting them has no effect. But this
+ malloc does support the following options.
+*/
+
+#define M_TRIM_THRESHOLD (-1)
+#define M_GRANULARITY (-2)
+#define M_MMAP_THRESHOLD (-3)
+
+/* ------------------------ Mallinfo declarations ------------------------ */
+
+#if !NO_MALLINFO
+/*
+ This version of malloc supports the standard SVID/XPG mallinfo
+ routine that returns a struct containing usage properties and
+ statistics. It should work on any system that has a
+ /usr/include/malloc.h defining struct mallinfo. The main
+ declaration needed is the mallinfo struct that is returned (by-copy)
+ by mallinfo(). The malloinfo struct contains a bunch of fields that
+ are not even meaningful in this version of malloc. These fields are
+ are instead filled by mallinfo() with other numbers that might be of
+ interest.
+
+ HAVE_USR_INCLUDE_MALLOC_H should be set if you have a
+ /usr/include/malloc.h file that includes a declaration of struct
+ mallinfo. If so, it is included; else a compliant version is
+ declared below. These must be precisely the same for mallinfo() to
+ work. The original SVID version of this struct, defined on most
+ systems with mallinfo, declares all fields as ints. But some others
+ define as unsigned long. If your system defines the fields using a
+ type of different width than listed here, you MUST #include your
+ system version and #define HAVE_USR_INCLUDE_MALLOC_H.
+*/
+
+/* #define HAVE_USR_INCLUDE_MALLOC_H */
+
+#ifdef HAVE_USR_INCLUDE_MALLOC_H
+#include "/usr/include/malloc.h"
+#else /* HAVE_USR_INCLUDE_MALLOC_H */
+#ifndef STRUCT_MALLINFO_DECLARED
+/* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is defined */
+#define _STRUCT_MALLINFO
+#define STRUCT_MALLINFO_DECLARED 1
+struct mallinfo {
+ MALLINFO_FIELD_TYPE arena; /* non-mmapped space allocated from system */
+ MALLINFO_FIELD_TYPE ordblks; /* number of free chunks */
+ MALLINFO_FIELD_TYPE smblks; /* always 0 */
+ MALLINFO_FIELD_TYPE hblks; /* always 0 */
+ MALLINFO_FIELD_TYPE hblkhd; /* space in mmapped regions */
+ MALLINFO_FIELD_TYPE usmblks; /* maximum total allocated space */
+ MALLINFO_FIELD_TYPE fsmblks; /* always 0 */
+ MALLINFO_FIELD_TYPE uordblks; /* total allocated space */
+ MALLINFO_FIELD_TYPE fordblks; /* total free space */
+ MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */
+};
+#endif /* STRUCT_MALLINFO_DECLARED */
+#endif /* HAVE_USR_INCLUDE_MALLOC_H */
+#endif /* NO_MALLINFO */
+
+/*
+ Try to persuade compilers to inline. The most critical functions for
+ inlining are defined as macros, so these aren't used for them.
+*/
+
+#ifndef FORCEINLINE
+ #if defined(__GNUC__)
+#define FORCEINLINE __inline __attribute__ ((always_inline))
+ #elif defined(_MSC_VER)
+ #define FORCEINLINE __forceinline
+ #endif
+#endif
+#ifndef NOINLINE
+ #if defined(__GNUC__)
+ #define NOINLINE __attribute__ ((noinline))
+ #elif defined(_MSC_VER)
+ #define NOINLINE __declspec(noinline)
+ #else
+ #define NOINLINE
+ #endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#ifndef FORCEINLINE
+ #define FORCEINLINE inline
+#endif
+#endif /* __cplusplus */
+#ifndef FORCEINLINE
+ #define FORCEINLINE
+#endif
+
+#if !ONLY_MSPACES
+
+/* ------------------- Declarations of public routines ------------------- */
+
+#ifndef USE_DL_PREFIX
+#define dlcalloc calloc
+#define dlfree free
+#define dlmalloc malloc
+#define dlmemalign memalign
+#define dlposix_memalign posix_memalign
+#define dlrealloc realloc
+#define dlrealloc_in_place realloc_in_place
+#define dlvalloc valloc
+#define dlpvalloc pvalloc
+#define dlmallinfo mallinfo
+#define dlmallopt mallopt
+#define dlmalloc_trim malloc_trim
+#define dlmalloc_stats malloc_stats
+#define dlmalloc_usable_size malloc_usable_size
+#define dlmalloc_footprint malloc_footprint
+#define dlmalloc_max_footprint malloc_max_footprint
+#define dlmalloc_footprint_limit malloc_footprint_limit
+#define dlmalloc_set_footprint_limit malloc_set_footprint_limit
+#define dlmalloc_inspect_all malloc_inspect_all
+#define dlindependent_calloc independent_calloc
+#define dlindependent_comalloc independent_comalloc
+#define dlbulk_free bulk_free
+#endif /* USE_DL_PREFIX */
+
+/*
+ malloc(size_t n)
+ Returns a pointer to a newly allocated chunk of at least n bytes, or
+ null if no space is available, in which case errno is set to ENOMEM
+ on ANSI C systems.
+
+ If n is zero, malloc returns a minimum-sized chunk. (The minimum
+ size is 16 bytes on most 32bit systems, and 32 bytes on 64bit
+ systems.) Note that size_t is an unsigned type, so calls with
+ arguments that would be negative if signed are interpreted as
+ requests for huge amounts of space, which will often fail. The
+ maximum supported value of n differs across systems, but is in all
+ cases less than the maximum representable value of a size_t.
+*/
+DLMALLOC_EXPORT void* dlmalloc(size_t);
+
+/*
+ free(void* p)
+ Releases the chunk of memory pointed to by p, that had been previously
+ allocated using malloc or a related routine such as realloc.
+ It has no effect if p is null. If p was not malloced or already
+ freed, free(p) will by default cause the current program to abort.
+*/
+DLMALLOC_EXPORT void dlfree(void*);
+
+/*
+ calloc(size_t n_elements, size_t element_size);
+ Returns a pointer to n_elements * element_size bytes, with all locations
+ set to zero.
+*/
+DLMALLOC_EXPORT void* dlcalloc(size_t, size_t);
+
+/*
+ realloc(void* p, size_t n)
+ Returns a pointer to a chunk of size n that contains the same data
+ as does chunk p up to the minimum of (n, p's size) bytes, or null
+ if no space is available.
+
+ The returned pointer may or may not be the same as p. The algorithm
+ prefers extending p in most cases when possible, otherwise it
+ employs the equivalent of a malloc-copy-free sequence.
+
+ If p is null, realloc is equivalent to malloc.
+
+ If space is not available, realloc returns null, errno is set (if on
+ ANSI) and p is NOT freed.
+
+ if n is for fewer bytes than already held by p, the newly unused
+ space is lopped off and freed if possible. realloc with a size
+ argument of zero (re)allocates a minimum-sized chunk.
+
+ The old unix realloc convention of allowing the last-free'd chunk
+ to be used as an argument to realloc is not supported.
+*/
+DLMALLOC_EXPORT void* dlrealloc(void*, size_t);
+
+/*
+ realloc_in_place(void* p, size_t n)
+ Resizes the space allocated for p to size n, only if this can be
+ done without moving p (i.e., only if there is adjacent space
+ available if n is greater than p's current allocated size, or n is
+ less than or equal to p's size). This may be used instead of plain
+ realloc if an alternative allocation strategy is needed upon failure
+ to expand space; for example, reallocation of a buffer that must be
+ memory-aligned or cleared. You can use realloc_in_place to trigger
+ these alternatives only when needed.
+
+ Returns p if successful; otherwise null.
+*/
+DLMALLOC_EXPORT void* dlrealloc_in_place(void*, size_t);
+
+/*
+ memalign(size_t alignment, size_t n);
+ Returns a pointer to a newly allocated chunk of n bytes, aligned
+ in accord with the alignment argument.
+
+ The alignment argument should be a power of two. If the argument is
+ not a power of two, the nearest greater power is used.
+ 8-byte alignment is guaranteed by normal malloc calls, so don't
+ bother calling memalign with an argument of 8 or less.
+
+ Overreliance on memalign is a sure way to fragment space.
+*/
+DLMALLOC_EXPORT void* dlmemalign(size_t, size_t);
+
+/*
+ int posix_memalign(void** pp, size_t alignment, size_t n);
+ Allocates a chunk of n bytes, aligned in accord with the alignment
+ argument. Differs from memalign only in that it (1) assigns the
+ allocated memory to *pp rather than returning it, (2) fails and
+ returns EINVAL if the alignment is not a power of two (3) fails and
+ returns ENOMEM if memory cannot be allocated.
+*/
+DLMALLOC_EXPORT int dlposix_memalign(void**, size_t, size_t);
+
+/*
+ valloc(size_t n);
+ Equivalent to memalign(pagesize, n), where pagesize is the page
+ size of the system. If the pagesize is unknown, 4096 is used.
+*/
+DLMALLOC_EXPORT void* dlvalloc(size_t);
+
+/*
+ mallopt(int parameter_number, int parameter_value)
+ Sets tunable parameters The format is to provide a
+ (parameter-number, parameter-value) pair. mallopt then sets the
+ corresponding parameter to the argument value if it can (i.e., so
+ long as the value is meaningful), and returns 1 if successful else
+ 0. To workaround the fact that mallopt is specified to use int,
+ not size_t parameters, the value -1 is specially treated as the
+ maximum unsigned size_t value.
+
+ SVID/XPG/ANSI defines four standard param numbers for mallopt,
+ normally defined in malloc.h. None of these are use in this malloc,
+ so setting them has no effect. But this malloc also supports other
+ options in mallopt. See below for details. Briefly, supported
+ parameters are as follows (listed defaults are for "typical"
+ configurations).
+
+ Symbol param # default allowed param values
+ M_TRIM_THRESHOLD -1 2*1024*1024 any (-1 disables)
+ M_GRANULARITY -2 page size any power of 2 >= page size
+ M_MMAP_THRESHOLD -3 256*1024 any (or 0 if no MMAP support)
+*/
+DLMALLOC_EXPORT int dlmallopt(int, int);
+
+/*
+ malloc_footprint();
+ Returns the number of bytes obtained from the system. The total
+ number of bytes allocated by malloc, realloc etc., is less than this
+ value. Unlike mallinfo, this function returns only a precomputed
+ result, so can be called frequently to monitor memory consumption.
+ Even if locks are otherwise defined, this function does not use them,
+ so results might not be up to date.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_footprint(void);
+
+/*
+ malloc_max_footprint();
+ Returns the maximum number of bytes obtained from the system. This
+ value will be greater than current footprint if deallocated space
+ has been reclaimed by the system. The peak number of bytes allocated
+ by malloc, realloc etc., is less than this value. Unlike mallinfo,
+ this function returns only a precomputed result, so can be called
+ frequently to monitor memory consumption. Even if locks are
+ otherwise defined, this function does not use them, so results might
+ not be up to date.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_max_footprint(void);
+
+/*
+ malloc_footprint_limit();
+ Returns the number of bytes that the heap is allowed to obtain from
+ the system, returning the last value returned by
+ malloc_set_footprint_limit, or the maximum size_t value if
+ never set. The returned value reflects a permission. There is no
+ guarantee that this number of bytes can actually be obtained from
+ the system.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_footprint_limit();
+
+/*
+ malloc_set_footprint_limit();
+ Sets the maximum number of bytes to obtain from the system, causing
+ failure returns from malloc and related functions upon attempts to
+ exceed this value. The argument value may be subject to page
+ rounding to an enforceable limit; this actual value is returned.
+ Using an argument of the maximum possible size_t effectively
+ disables checks. If the argument is less than or equal to the
+ current malloc_footprint, then all future allocations that require
+ additional system memory will fail. However, invocation cannot
+ retroactively deallocate existing used memory.
+*/
+DLMALLOC_EXPORT size_t dlmalloc_set_footprint_limit(size_t bytes);
+
+#if MALLOC_INSPECT_ALL
+/*
+ malloc_inspect_all(void(*handler)(void *start,
+ void *end,
+ size_t used_bytes,
+ void* callback_arg),
+ void* arg);
+ Traverses the heap and calls the given handler for each managed
+ region, skipping all bytes that are (or may be) used for bookkeeping
+ purposes. Traversal does not include include chunks that have been
+ directly memory mapped. Each reported region begins at the start
+ address, and continues up to but not including the end address. The
+ first used_bytes of the region contain allocated data. If
+ used_bytes is zero, the region is unallocated. The handler is
+ invoked with the given callback argument. If locks are defined, they
+ are held during the entire traversal. It is a bad idea to invoke
+ other malloc functions from within the handler.
+
+ For example, to count the number of in-use chunks with size greater
+ than 1000, you could write:
+ static int count = 0;
+ void count_chunks(void* start, void* end, size_t used, void* arg) {
+ if (used >= 1000) ++count;
+ }
+ then:
+ malloc_inspect_all(count_chunks, NULL);
+
+ malloc_inspect_all is compiled only if MALLOC_INSPECT_ALL is defined.
+*/
+DLMALLOC_EXPORT void dlmalloc_inspect_all(void(*handler)(void*, void *, size_t, void*),
+ void* arg);
+
+#endif /* MALLOC_INSPECT_ALL */
+
+#if !NO_MALLINFO
+/*
+ mallinfo()
+ Returns (by copy) a struct containing various summary statistics:
+
+ arena: current total non-mmapped bytes allocated from system
+ ordblks: the number of free chunks
+ smblks: always zero.
+ hblks: current number of mmapped regions
+ hblkhd: total bytes held in mmapped regions
+ usmblks: the maximum total allocated space. This will be greater
+ than current total if trimming has occurred.
+ fsmblks: always zero
+ uordblks: current total allocated space (normal or mmapped)
+ fordblks: total free space
+ keepcost: the maximum number of bytes that could ideally be released
+ back to system via malloc_trim. ("ideally" means that
+ it ignores page restrictions etc.)
+
+ Because these fields are ints, but internal bookkeeping may
+ be kept as longs, the reported values may wrap around zero and
+ thus be inaccurate.
+*/
+DLMALLOC_EXPORT struct mallinfo dlmallinfo(void);
+#endif /* NO_MALLINFO */
+
+/*
+ independent_calloc(size_t n_elements, size_t element_size, void* chunks[]);
+
+ independent_calloc is similar to calloc, but instead of returning a
+ single cleared space, it returns an array of pointers to n_elements
+ independent elements that can hold contents of size elem_size, each
+ of which starts out cleared, and can be independently freed,
+ realloc'ed etc. The elements are guaranteed to be adjacently
+ allocated (this is not guaranteed to occur with multiple callocs or
+ mallocs), which may also improve cache locality in some
+ applications.
+
+ The "chunks" argument is optional (i.e., may be null, which is
+ probably the most typical usage). If it is null, the returned array
+ is itself dynamically allocated and should also be freed when it is
+ no longer needed. Otherwise, the chunks array must be of at least
+ n_elements in length. It is filled in with the pointers to the
+ chunks.
+
+ In either case, independent_calloc returns this pointer array, or
+ null if the allocation failed. If n_elements is zero and "chunks"
+ is null, it returns a chunk representing an array with zero elements
+ (which should be freed if not wanted).
+
+ Each element must be freed when it is no longer needed. This can be
+ done all at once using bulk_free.
+
+ independent_calloc simplifies and speeds up implementations of many
+ kinds of pools. It may also be useful when constructing large data
+ structures that initially have a fixed number of fixed-sized nodes,
+ but the number is not known at compile time, and some of the nodes
+ may later need to be freed. For example:
+
+ struct Node { int item; struct Node* next; };
+
+ struct Node* build_list() {
+ struct Node** pool;
+ int n = read_number_of_nodes_needed();
+ if (n <= 0) return 0;
+ pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0);
+ if (pool == 0) die();
+ // organize into a linked list...
+ struct Node* first = pool[0];
+ for (i = 0; i < n-1; ++i)
+ pool[i]->next = pool[i+1];
+ free(pool); // Can now free the array (or not, if it is needed later)
+ return first;
+ }
+*/
+DLMALLOC_EXPORT void** dlindependent_calloc(size_t, size_t, void**);
+
+/*
+ independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
+
+ independent_comalloc allocates, all at once, a set of n_elements
+ chunks with sizes indicated in the "sizes" array. It returns
+ an array of pointers to these elements, each of which can be
+ independently freed, realloc'ed etc. The elements are guaranteed to
+ be adjacently allocated (this is not guaranteed to occur with
+ multiple callocs or mallocs), which may also improve cache locality
+ in some applications.
+
+ The "chunks" argument is optional (i.e., may be null). If it is null
+ the returned array is itself dynamically allocated and should also
+ be freed when it is no longer needed. Otherwise, the chunks array
+ must be of at least n_elements in length. It is filled in with the
+ pointers to the chunks.
+
+ In either case, independent_comalloc returns this pointer array, or
+ null if the allocation failed. If n_elements is zero and chunks is
+ null, it returns a chunk representing an array with zero elements
+ (which should be freed if not wanted).
+
+ Each element must be freed when it is no longer needed. This can be
+ done all at once using bulk_free.
+
+ independent_comallac differs from independent_calloc in that each
+ element may have a different size, and also that it does not
+ automatically clear elements.
+
+ independent_comalloc can be used to speed up allocation in cases
+ where several structs or objects must always be allocated at the
+ same time. For example:
+
+ struct Head { ... }
+ struct Foot { ... }
+
+ void send_message(char* msg) {
+ int msglen = strlen(msg);
+ size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) };
+ void* chunks[3];
+ if (independent_comalloc(3, sizes, chunks) == 0)
+ die();
+ struct Head* head = (struct Head*)(chunks[0]);
+ char* body = (char*)(chunks[1]);
+ struct Foot* foot = (struct Foot*)(chunks[2]);
+ // ...
+ }
+
+ In general though, independent_comalloc is worth using only for
+ larger values of n_elements. For small values, you probably won't
+ detect enough difference from series of malloc calls to bother.
+
+ Overuse of independent_comalloc can increase overall memory usage,
+ since it cannot reuse existing noncontiguous small chunks that
+ might be available for some of the elements.
+*/
+DLMALLOC_EXPORT void** dlindependent_comalloc(size_t, size_t*, void**);
+
+/*
+ bulk_free(void* array[], size_t n_elements)
+ Frees and clears (sets to null) each non-null pointer in the given
+ array. This is likely to be faster than freeing them one-by-one.
+ If footers are used, pointers that have been allocated in different
+ mspaces are not freed or cleared, and the count of all such pointers
+ is returned. For large arrays of pointers with poor locality, it
+ may be worthwhile to sort this array before calling bulk_free.
+*/
+DLMALLOC_EXPORT size_t dlbulk_free(void**, size_t n_elements);
+
+/*
+ pvalloc(size_t n);
+ Equivalent to valloc(minimum-page-that-holds(n)), that is,
+ round up n to nearest pagesize.
+ */
+DLMALLOC_EXPORT void* dlpvalloc(size_t);
+
+/*
+ malloc_trim(size_t pad);
+
+ If possible, gives memory back to the system (via negative arguments
+ to sbrk) if there is unused memory at the `high' end of the malloc
+ pool or in unused MMAP segments. You can call this after freeing
+ large blocks of memory to potentially reduce the system-level memory
+ requirements of a program. However, it cannot guarantee to reduce
+ memory. Under some allocation patterns, some large free blocks of
+ memory will be locked between two used chunks, so they cannot be
+ given back to the system.
+
+ The `pad' argument to malloc_trim represents the amount of free
+ trailing space to leave untrimmed. If this argument is zero, only
+ the minimum amount of memory to maintain internal data structures
+ will be left. Non-zero arguments can be supplied to maintain enough
+ trailing space to service future expected allocations without having
+ to re-obtain memory from the system.
+
+ Malloc_trim returns 1 if it actually released any memory, else 0.
+*/
+DLMALLOC_EXPORT int dlmalloc_trim(size_t);
+
+/*
+ malloc_stats();
+ Prints on stderr the amount of space obtained from the system (both
+ via sbrk and mmap), the maximum amount (which may be more than
+ current if malloc_trim and/or munmap got called), and the current
+ number of bytes allocated via malloc (or realloc, etc) but not yet
+ freed. Note that this is the number of bytes allocated, not the
+ number requested. It will be larger than the number requested
+ because of alignment and bookkeeping overhead. Because it includes
+ alignment wastage as being in use, this figure may be greater than
+ zero even when no user-level chunks are allocated.
+
+ The reported current and maximum system memory can be inaccurate if
+ a program makes other calls to system memory allocation functions
+ (normally sbrk) outside of malloc.
+
+ malloc_stats prints only the most commonly interesting statistics.
+ More information can be obtained by calling mallinfo.
+*/
+DLMALLOC_EXPORT void dlmalloc_stats(void);
+
+/*
+ malloc_usable_size(void* p);
+
+ Returns the number of bytes you can actually use in
+ an allocated chunk, which may be more than you requested (although
+ often not) due to alignment and minimum size constraints.
+ You can use this many bytes without worrying about
+ overwriting other allocated objects. This is not a particularly great
+ programming practice. malloc_usable_size can be more useful in
+ debugging and assertions, for example:
+
+ p = malloc(n);
+ assert(malloc_usable_size(p) >= 256);
+*/
+size_t dlmalloc_usable_size(void*);
+
+#endif /* ONLY_MSPACES */
+
+#if MSPACES
+
+/*
+ mspace is an opaque type representing an independent
+ region of space that supports mspace_malloc, etc.
+*/
+typedef void* mspace;
+
+/*
+ create_mspace creates and returns a new independent space with the
+ given initial capacity, or, if 0, the default granularity size. It
+ returns null if there is no system memory available to create the
+ space. If argument locked is non-zero, the space uses a separate
+ lock to control access. The capacity of the space will grow
+ dynamically as needed to service mspace_malloc requests. You can
+ control the sizes of incremental increases of this space by
+ compiling with a different DEFAULT_GRANULARITY or dynamically
+ setting with mallopt(M_GRANULARITY, value).
+*/
+DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked);
+
+/*
+ destroy_mspace destroys the given space, and attempts to return all
+ of its memory back to the system, returning the total number of
+ bytes freed. After destruction, the results of access to all memory
+ used by the space become undefined.
+*/
+DLMALLOC_EXPORT size_t destroy_mspace(mspace msp);
+
+/*
+ create_mspace_with_base uses the memory supplied as the initial base
+ of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this
+ space is used for bookkeeping, so the capacity must be at least this
+ large. (Otherwise 0 is returned.) When this initial space is
+ exhausted, additional memory will be obtained from the system.
+ Destroying this space will deallocate all additionally allocated
+ space (if possible) but not the initial base.
+*/
+DLMALLOC_EXPORT mspace create_mspace_with_base(void* base, size_t capacity, int locked);
+
+/*
+ mspace_track_large_chunks controls whether requests for large chunks
+ are allocated in their own untracked mmapped regions, separate from
+ others in this mspace. By default large chunks are not tracked,
+ which reduces fragmentation. However, such chunks are not
+ necessarily released to the system upon destroy_mspace. Enabling
+ tracking by setting to true may increase fragmentation, but avoids
+ leakage when relying on destroy_mspace to release all memory
+ allocated using this space. The function returns the previous
+ setting.
+*/
+DLMALLOC_EXPORT int mspace_track_large_chunks(mspace msp, int enable);
+
+
+/*
+ mspace_malloc behaves as malloc, but operates within
+ the given space.
+*/
+DLMALLOC_EXPORT void* mspace_malloc(mspace msp, size_t bytes);
+
+/*
+ mspace_free behaves as free, but operates within
+ the given space.
+
+ If compiled with FOOTERS==1, mspace_free is not actually needed.
+ free may be called instead of mspace_free because freed chunks from
+ any space are handled by their originating spaces.
+*/
+DLMALLOC_EXPORT void mspace_free(mspace msp, void* mem);
+
+/*
+ mspace_realloc behaves as realloc, but operates within
+ the given space.
+
+ If compiled with FOOTERS==1, mspace_realloc is not actually
+ needed. realloc may be called instead of mspace_realloc because
+ realloced chunks from any space are handled by their originating
+ spaces.
+*/
+DLMALLOC_EXPORT void* mspace_realloc(mspace msp, void* mem, size_t newsize);
+
+/*
+ mspace_calloc behaves as calloc, but operates within
+ the given space.
+*/
+DLMALLOC_EXPORT void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size);
+
+/*
+ mspace_memalign behaves as memalign, but operates within
+ the given space.
+*/
+DLMALLOC_EXPORT void* mspace_memalign(mspace msp, size_t alignment, size_t bytes);
+
+/*
+ mspace_independent_calloc behaves as independent_calloc, but
+ operates within the given space.
+*/
+DLMALLOC_EXPORT void** mspace_independent_calloc(mspace msp, size_t n_elements,
+ size_t elem_size, void* chunks[]);
+
+/*
+ mspace_independent_comalloc behaves as independent_comalloc, but
+ operates within the given space.
+*/
+DLMALLOC_EXPORT void** mspace_independent_comalloc(mspace msp, size_t n_elements,
+ size_t sizes[], void* chunks[]);
+
+/*
+ mspace_footprint() returns the number of bytes obtained from the
+ system for this space.
+*/
+DLMALLOC_EXPORT size_t mspace_footprint(mspace msp);
+
+/*
+ mspace_max_footprint() returns the peak number of bytes obtained from the
+ system for this space.
+*/
+DLMALLOC_EXPORT size_t mspace_max_footprint(mspace msp);
+
+
+#if !NO_MALLINFO
+/*
+ mspace_mallinfo behaves as mallinfo, but reports properties of
+ the given space.
+*/
+DLMALLOC_EXPORT struct mallinfo mspace_mallinfo(mspace msp);
+#endif /* NO_MALLINFO */
+
+/*
+ malloc_usable_size(void* p) behaves the same as malloc_usable_size;
+*/
+DLMALLOC_EXPORT size_t mspace_usable_size(const void* mem);
+
+/*
+ mspace_malloc_stats behaves as malloc_stats, but reports
+ properties of the given space.
+*/
+DLMALLOC_EXPORT void mspace_malloc_stats(mspace msp);
+
+/*
+ mspace_trim behaves as malloc_trim, but
+ operates within the given space.
+*/
+DLMALLOC_EXPORT int mspace_trim(mspace msp, size_t pad);
+
+/*
+ An alias for mallopt.
+*/
+DLMALLOC_EXPORT int mspace_mallopt(int, int);
+
+#endif /* MSPACES */
+
+#ifdef __cplusplus
+} /* end of extern "C" */
+#endif /* __cplusplus */
+
+/*
+ ========================================================================
+ To make a fully customizable malloc.h header file, cut everything
+ above this line, put into file malloc.h, edit to suit, and #include it
+ on the next line, as well as in programs that use this malloc.
+ ========================================================================
+*/
+
+/* #include "malloc.h" */
+
+/*------------------------------ internal #includes ---------------------- */
+
+#ifdef _MSC_VER
+#pragma warning( disable : 4146 ) /* no "unsigned" warnings */
+#endif /* _MSC_VER */
+#if !NO_MALLOC_STATS
+#include <stdio.h> /* for printing in malloc_stats */
+#endif /* NO_MALLOC_STATS */
+#ifndef LACKS_ERRNO_H
+#include <errno.h> /* for MALLOC_FAILURE_ACTION */
+#endif /* LACKS_ERRNO_H */
+#ifdef DEBUG
+#if ABORT_ON_ASSERT_FAILURE
+#undef assert
+#define assert(x) if(!(x)) ABORT
+#else /* ABORT_ON_ASSERT_FAILURE */
+#include <assert.h>
+#endif /* ABORT_ON_ASSERT_FAILURE */
+#else /* DEBUG */
+#ifndef assert
+#define assert(x)
+#endif
+#define DEBUG 0
+#endif /* DEBUG */
+#if !defined(WIN32) && !defined(LACKS_TIME_H)
+#include <time.h> /* for magic initialization */
+#endif /* WIN32 */
+#ifndef LACKS_STDLIB_H
+#include <stdlib.h> /* for abort() */
+#endif /* LACKS_STDLIB_H */
+#ifndef LACKS_STRING_H
+#include <string.h> /* for memset etc */
+#endif /* LACKS_STRING_H */
+#if USE_BUILTIN_FFS
+#ifndef LACKS_STRINGS_H
+#include <strings.h> /* for ffs */
+#endif /* LACKS_STRINGS_H */
+#endif /* USE_BUILTIN_FFS */
+#if HAVE_MMAP
+#ifndef LACKS_SYS_MMAN_H
+/* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */
+#if (defined(linux) && !defined(__USE_GNU))
+#define __USE_GNU 1
+#include <sys/mman.h> /* for mmap */
+#undef __USE_GNU
+#else
+#include <sys/mman.h> /* for mmap */
+#endif /* linux */
+#endif /* LACKS_SYS_MMAN_H */
+#ifndef LACKS_FCNTL_H
+#include <fcntl.h>
+#endif /* LACKS_FCNTL_H */
+#endif /* HAVE_MMAP */
+#ifndef LACKS_UNISTD_H
+#include <unistd.h> /* for sbrk, sysconf */
+#else /* LACKS_UNISTD_H */
+#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__)
+extern void* sbrk(ptrdiff_t);
+#endif /* FreeBSD etc */
+#endif /* LACKS_UNISTD_H */
+
+/* Declarations for locking */
+#if USE_LOCKS
+#ifndef WIN32
+#if defined (__SVR4) && defined (__sun) /* solaris */
+#include <thread.h>
+#elif !defined(LACKS_SCHED_H)
+#include <sched.h>
+#endif /* solaris or LACKS_SCHED_H */
+#if (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0) || !USE_SPIN_LOCKS
+#include <pthread.h>
+#endif /* USE_RECURSIVE_LOCKS ... */
+#elif defined(_MSC_VER)
+#ifndef _M_AMD64
+/* These are already defined on AMD64 builds */
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+LONG __cdecl _InterlockedCompareExchange(LONG volatile *Dest, LONG Exchange, LONG Comp);
+LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value);
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+#endif /* _M_AMD64 */
+#pragma intrinsic (_InterlockedCompareExchange)
+#pragma intrinsic (_InterlockedExchange)
+#define interlockedcompareexchange _InterlockedCompareExchange
+#define interlockedexchange _InterlockedExchange
+#elif defined(WIN32) && defined(__GNUC__)
+#define interlockedcompareexchange(a, b, c) __sync_val_compare_and_swap(a, c, b)
+#define interlockedexchange __sync_lock_test_and_set
+#endif /* Win32 */
+#else /* USE_LOCKS */
+#endif /* USE_LOCKS */
+
+#ifndef LOCK_AT_FORK
+#define LOCK_AT_FORK 0
+#endif
+
+/* Declarations for bit scanning on win32 */
+#if defined(_MSC_VER) && _MSC_VER>=1300
+#ifndef BitScanForward /* Try to avoid pulling in WinNT.h */
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+unsigned char _BitScanForward(unsigned long *index, unsigned long mask);
+unsigned char _BitScanReverse(unsigned long *index, unsigned long mask);
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#define BitScanForward _BitScanForward
+#define BitScanReverse _BitScanReverse
+#pragma intrinsic(_BitScanForward)
+#pragma intrinsic(_BitScanReverse)
+#endif /* BitScanForward */
+#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */
+
+#ifndef WIN32
+#ifndef malloc_getpagesize
+# ifdef _SC_PAGESIZE /* some SVR4 systems omit an underscore */
+# ifndef _SC_PAGE_SIZE
+# define _SC_PAGE_SIZE _SC_PAGESIZE
+# endif
+# endif
+# ifdef _SC_PAGE_SIZE
+# define malloc_getpagesize sysconf(_SC_PAGE_SIZE)
+# else
+# if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE)
+ extern size_t getpagesize();
+# define malloc_getpagesize getpagesize()
+# else
+# ifdef WIN32 /* use supplied emulation of getpagesize */
+# define malloc_getpagesize getpagesize()
+# else
+# ifndef LACKS_SYS_PARAM_H
+# include <sys/param.h>
+# endif
+# ifdef EXEC_PAGESIZE
+# define malloc_getpagesize EXEC_PAGESIZE
+# else
+# ifdef NBPG
+# ifndef CLSIZE
+# define malloc_getpagesize NBPG
+# else
+# define malloc_getpagesize (NBPG * CLSIZE)
+# endif
+# else
+# ifdef NBPC
+# define malloc_getpagesize NBPC
+# else
+# ifdef PAGESIZE
+# define malloc_getpagesize PAGESIZE
+# else /* just guess */
+# define malloc_getpagesize ((size_t)4096U)
+# endif
+# endif
+# endif
+# endif
+# endif
+# endif
+# endif
+#endif
+#endif
+
+/* ------------------- size_t and alignment properties -------------------- */
+
+/* The byte and bit size of a size_t */
+#define SIZE_T_SIZE (sizeof(size_t))
+#define SIZE_T_BITSIZE (sizeof(size_t) << 3)
+
+/* Some constants coerced to size_t */
+/* Annoying but necessary to avoid errors on some platforms */
+#define SIZE_T_ZERO ((size_t)0)
+#define SIZE_T_ONE ((size_t)1)
+#define SIZE_T_TWO ((size_t)2)
+#define SIZE_T_FOUR ((size_t)4)
+#define TWO_SIZE_T_SIZES (SIZE_T_SIZE<<1)
+#define FOUR_SIZE_T_SIZES (SIZE_T_SIZE<<2)
+#define SIX_SIZE_T_SIZES (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES)
+#define HALF_MAX_SIZE_T (MAX_SIZE_T / 2U)
+
+/* The bit mask value corresponding to MALLOC_ALIGNMENT */
+#define CHUNK_ALIGN_MASK (MALLOC_ALIGNMENT - SIZE_T_ONE)
+
+/* True if address a has acceptable alignment */
+#define is_aligned(A) (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0)
+
+/* the number of bytes to offset an address to align it */
+#define align_offset(A)\
+ ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\
+ ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK))
+
+/* -------------------------- MMAP preliminaries ------------------------- */
+
+/*
+ If HAVE_MORECORE or HAVE_MMAP are false, we just define calls and
+ checks to fail so compiler optimizer can delete code rather than
+ using so many "#if"s.
+*/
+
+
+/* MORECORE and MMAP must return MFAIL on failure */
+#define MFAIL ((void*)(MAX_SIZE_T))
+#define CMFAIL ((char*)(MFAIL)) /* defined for convenience */
+
+#if HAVE_MMAP
+
+#ifndef WIN32
+#define MUNMAP_DEFAULT(a, s) munmap((a), (s))
+#define MMAP_PROT (PROT_READ|PROT_WRITE)
+#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
+#define MAP_ANONYMOUS MAP_ANON
+#endif /* MAP_ANON */
+#ifdef MAP_ANONYMOUS
+#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS)
+#define MMAP_DEFAULT(s) mmap(0, (s), MMAP_PROT, MMAP_FLAGS, -1, 0)
+#else /* MAP_ANONYMOUS */
+/*
+ Nearly all versions of mmap support MAP_ANONYMOUS, so the following
+ is unlikely to be needed, but is supplied just in case.
+*/
+#define MMAP_FLAGS (MAP_PRIVATE)
+static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */
+#define MMAP_DEFAULT(s) ((dev_zero_fd < 0) ? \
+ (dev_zero_fd = open("/dev/zero", O_RDWR), \
+ mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) : \
+ mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0))
+#endif /* MAP_ANONYMOUS */
+
+#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s)
+
+#else /* WIN32 */
+
+/* Win32 MMAP via VirtualAlloc */
+static FORCEINLINE void* win32mmap(size_t size) {
+ void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
+ return (ptr != 0)? ptr: MFAIL;
+}
+
+/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
+static FORCEINLINE void* win32direct_mmap(size_t size) {
+ void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN,
+ PAGE_READWRITE);
+ return (ptr != 0)? ptr: MFAIL;
+}
+
+/* This function supports releasing coalesed segments */
+static FORCEINLINE int win32munmap(void* ptr, size_t size) {
+ MEMORY_BASIC_INFORMATION minfo;
+ char* cptr = (char*)ptr;
+ while (size) {
+ if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0)
+ return -1;
+ if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr ||
+ minfo.State != MEM_COMMIT || minfo.RegionSize > size)
+ return -1;
+ if (VirtualFree(cptr, 0, MEM_RELEASE) == 0)
+ return -1;
+ cptr += minfo.RegionSize;
+ size -= minfo.RegionSize;
+ }
+ return 0;
+}
+
+#define MMAP_DEFAULT(s) win32mmap(s)
+#define MUNMAP_DEFAULT(a, s) win32munmap((a), (s))
+#define DIRECT_MMAP_DEFAULT(s) win32direct_mmap(s)
+#endif /* WIN32 */
+#endif /* HAVE_MMAP */
+
+#if HAVE_MREMAP
+#ifndef WIN32
+#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv))
+#endif /* WIN32 */
+#endif /* HAVE_MREMAP */
+
+/**
+ * Define CALL_MORECORE
+ */
+#if HAVE_MORECORE
+ #ifdef MORECORE
+ #define CALL_MORECORE(S) MORECORE(S)
+ #else /* MORECORE */
+ #define CALL_MORECORE(S) MORECORE_DEFAULT(S)
+ #endif /* MORECORE */
+#else /* HAVE_MORECORE */
+ #define CALL_MORECORE(S) MFAIL
+#endif /* HAVE_MORECORE */
+
+/**
+ * Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP
+ */
+#if HAVE_MMAP
+ #define USE_MMAP_BIT (SIZE_T_ONE)
+
+ #ifdef MMAP
+ #define CALL_MMAP(s) MMAP(s)
+ #else /* MMAP */
+ #define CALL_MMAP(s) MMAP_DEFAULT(s)
+ #endif /* MMAP */
+ #ifdef MUNMAP
+ #define CALL_MUNMAP(a, s) MUNMAP((a), (s))
+ #else /* MUNMAP */
+ #define CALL_MUNMAP(a, s) MUNMAP_DEFAULT((a), (s))
+ #endif /* MUNMAP */
+ #ifdef DIRECT_MMAP
+ #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
+ #else /* DIRECT_MMAP */
+ #define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s)
+ #endif /* DIRECT_MMAP */
+#else /* HAVE_MMAP */
+ #define USE_MMAP_BIT (SIZE_T_ZERO)
+
+ #define MMAP(s) MFAIL
+ #define MUNMAP(a, s) (-1)
+ #define DIRECT_MMAP(s) MFAIL
+ #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
+ #define CALL_MMAP(s) MMAP(s)
+ #define CALL_MUNMAP(a, s) MUNMAP((a), (s))
+#endif /* HAVE_MMAP */
+
+/**
+ * Define CALL_MREMAP
+ */
+#if HAVE_MMAP && HAVE_MREMAP
+ #ifdef MREMAP
+ #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv))
+ #else /* MREMAP */
+ #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP_DEFAULT((addr), (osz), (nsz), (mv))
+ #endif /* MREMAP */
+#else /* HAVE_MMAP && HAVE_MREMAP */
+ #define CALL_MREMAP(addr, osz, nsz, mv) MFAIL
+#endif /* HAVE_MMAP && HAVE_MREMAP */
+
+/* mstate bit set if continguous morecore disabled or failed */
+#define USE_NONCONTIGUOUS_BIT (4U)
+
+/* segment bit set in create_mspace_with_base */
+#define EXTERN_BIT (8U)
+
+
+/* --------------------------- Lock preliminaries ------------------------ */
+
+/*
+ When locks are defined, there is one global lock, plus
+ one per-mspace lock.
+
+ The global lock_ensures that mparams.magic and other unique
+ mparams values are initialized only once. It also protects
+ sequences of calls to MORECORE. In many cases sys_alloc requires
+ two calls, that should not be interleaved with calls by other
+ threads. This does not protect against direct calls to MORECORE
+ by other threads not using this lock, so there is still code to
+ cope the best we can on interference.
+
+ Per-mspace locks surround calls to malloc, free, etc.
+ By default, locks are simple non-reentrant mutexes.
+
+ Because lock-protected regions generally have bounded times, it is
+ OK to use the supplied simple spinlocks. Spinlocks are likely to
+ improve performance for lightly contended applications, but worsen
+ performance under heavy contention.
+
+ If USE_LOCKS is > 1, the definitions of lock routines here are
+ bypassed, in which case you will need to define the type MLOCK_T,
+ and at least INITIAL_LOCK, DESTROY_LOCK, ACQUIRE_LOCK, RELEASE_LOCK
+ and TRY_LOCK. You must also declare a
+ static MLOCK_T malloc_global_mutex = { initialization values };.
+
+*/
+
+#if !USE_LOCKS
+#define USE_LOCK_BIT (0U)
+#define INITIAL_LOCK(l) (0)
+#define DESTROY_LOCK(l) (0)
+#define ACQUIRE_MALLOC_GLOBAL_LOCK()
+#define RELEASE_MALLOC_GLOBAL_LOCK()
+
+#else
+#if USE_LOCKS > 1
+/* ----------------------- User-defined locks ------------------------ */
+/* Define your own lock implementation here */
+/* #define INITIAL_LOCK(lk) ... */
+/* #define DESTROY_LOCK(lk) ... */
+/* #define ACQUIRE_LOCK(lk) ... */
+/* #define RELEASE_LOCK(lk) ... */
+/* #define TRY_LOCK(lk) ... */
+/* static MLOCK_T malloc_global_mutex = ... */
+
+#elif USE_SPIN_LOCKS
+
+/* First, define CAS_LOCK and CLEAR_LOCK on ints */
+/* Note CAS_LOCK defined to return 0 on success */
+
+#if defined(__GNUC__)&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))
+#define CAS_LOCK(sl) __sync_lock_test_and_set(sl, 1)
+#define CLEAR_LOCK(sl) __sync_lock_release(sl)
+
+#elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)))
+/* Custom spin locks for older gcc on x86 */
+static FORCEINLINE int x86_cas_lock(int *sl) {
+ int ret;
+ int val = 1;
+ int cmp = 0;
+ __asm__ __volatile__ ("lock; cmpxchgl %1, %2"
+ : "=a" (ret)
+ : "r" (val), "m" (*(sl)), "0"(cmp)
+ : "memory", "cc");
+ return ret;
+}
+
+static FORCEINLINE void x86_clear_lock(int* sl) {
+ assert(*sl != 0);
+ int prev = 0;
+ int ret;
+ __asm__ __volatile__ ("lock; xchgl %0, %1"
+ : "=r" (ret)
+ : "m" (*(sl)), "0"(prev)
+ : "memory");
+}
+
+#define CAS_LOCK(sl) x86_cas_lock(sl)
+#define CLEAR_LOCK(sl) x86_clear_lock(sl)
+
+#else /* Win32 MSC */
+#define CAS_LOCK(sl) interlockedexchange(sl, (LONG)1)
+#define CLEAR_LOCK(sl) interlockedexchange (sl, (LONG)0)
+
+#endif /* ... gcc spins locks ... */
+
+/* How to yield for a spin lock */
+#define SPINS_PER_YIELD 63
+#if defined(_MSC_VER)
+#define SLEEP_EX_DURATION 50 /* delay for yield/sleep */
+#define SPIN_LOCK_YIELD SleepEx(SLEEP_EX_DURATION, FALSE)
+#elif defined (__SVR4) && defined (__sun) /* solaris */
+#define SPIN_LOCK_YIELD thr_yield();
+#elif !defined(LACKS_SCHED_H)
+#define SPIN_LOCK_YIELD sched_yield();
+#else
+#define SPIN_LOCK_YIELD
+#endif /* ... yield ... */
+
+#if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0
+/* Plain spin locks use single word (embedded in malloc_states) */
+static int spin_acquire_lock(int *sl) {
+ int spins = 0;
+ while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) {
+ if ((++spins & SPINS_PER_YIELD) == 0) {
+ SPIN_LOCK_YIELD;
+ }
+ }
+ return 0;
+}
+
+#define MLOCK_T int
+#define TRY_LOCK(sl) !CAS_LOCK(sl)
+#define RELEASE_LOCK(sl) CLEAR_LOCK(sl)
+#define ACQUIRE_LOCK(sl) (CAS_LOCK(sl)? spin_acquire_lock(sl) : 0)
+#define INITIAL_LOCK(sl) (*sl = 0)
+#define DESTROY_LOCK(sl) (0)
+static MLOCK_T malloc_global_mutex = 0;
+
+#else /* USE_RECURSIVE_LOCKS */
+/* types for lock owners */
+#ifdef WIN32
+#define THREAD_ID_T DWORD
+#define CURRENT_THREAD GetCurrentThreadId()
+#define EQ_OWNER(X,Y) ((X) == (Y))
+#else
+/*
+ Note: the following assume that pthread_t is a type that can be
+ initialized to (casted) zero. If this is not the case, you will need to
+ somehow redefine these or not use spin locks.
+*/
+#define THREAD_ID_T pthread_t
+#define CURRENT_THREAD pthread_self()
+#define EQ_OWNER(X,Y) pthread_equal(X, Y)
+#endif
+
+struct malloc_recursive_lock {
+ int sl;
+ unsigned int c;
+ THREAD_ID_T threadid;
+};
+
+#define MLOCK_T struct malloc_recursive_lock
+static MLOCK_T malloc_global_mutex = { 0, 0, (THREAD_ID_T)0};
+
+static FORCEINLINE void recursive_release_lock(MLOCK_T *lk) {
+ assert(lk->sl != 0);
+ if (--lk->c == 0) {
+ CLEAR_LOCK(&lk->sl);
+ }
+}
+
+static FORCEINLINE int recursive_acquire_lock(MLOCK_T *lk) {
+ THREAD_ID_T mythreadid = CURRENT_THREAD;
+ int spins = 0;
+ for (;;) {
+ if (*((volatile int *)(&lk->sl)) == 0) {
+ if (!CAS_LOCK(&lk->sl)) {
+ lk->threadid = mythreadid;
+ lk->c = 1;
+ return 0;
+ }
+ }
+ else if (EQ_OWNER(lk->threadid, mythreadid)) {
+ ++lk->c;
+ return 0;
+ }
+ if ((++spins & SPINS_PER_YIELD) == 0) {
+ SPIN_LOCK_YIELD;
+ }
+ }
+}
+
+static FORCEINLINE int recursive_try_lock(MLOCK_T *lk) {
+ THREAD_ID_T mythreadid = CURRENT_THREAD;
+ if (*((volatile int *)(&lk->sl)) == 0) {
+ if (!CAS_LOCK(&lk->sl)) {
+ lk->threadid = mythreadid;
+ lk->c = 1;
+ return 1;
+ }
+ }
+ else if (EQ_OWNER(lk->threadid, mythreadid)) {
+ ++lk->c;
+ return 1;
+ }
+ return 0;
+}
+
+#define RELEASE_LOCK(lk) recursive_release_lock(lk)
+#define TRY_LOCK(lk) recursive_try_lock(lk)
+#define ACQUIRE_LOCK(lk) recursive_acquire_lock(lk)
+#define INITIAL_LOCK(lk) ((lk)->threadid = (THREAD_ID_T)0, (lk)->sl = 0, (lk)->c = 0)
+#define DESTROY_LOCK(lk) (0)
+#endif /* USE_RECURSIVE_LOCKS */
+
+#elif defined(WIN32) /* Win32 critical sections */
+#define MLOCK_T CRITICAL_SECTION
+#define ACQUIRE_LOCK(lk) (EnterCriticalSection(lk), 0)
+#define RELEASE_LOCK(lk) LeaveCriticalSection(lk)
+#define TRY_LOCK(lk) TryEnterCriticalSection(lk)
+#define INITIAL_LOCK(lk) (!InitializeCriticalSectionAndSpinCount((lk), 0x80000000|4000))
+#define DESTROY_LOCK(lk) (DeleteCriticalSection(lk), 0)
+#define NEED_GLOBAL_LOCK_INIT
+
+static MLOCK_T malloc_global_mutex;
+static volatile LONG malloc_global_mutex_status;
+
+/* Use spin loop to initialize global lock */
+static void init_malloc_global_mutex() {
+ for (;;) {
+ long stat = malloc_global_mutex_status;
+ if (stat > 0)
+ return;
+ /* transition to < 0 while initializing, then to > 0) */
+ if (stat == 0 &&
+ interlockedcompareexchange(&malloc_global_mutex_status, (LONG)-1, (LONG)0) == 0) {
+ InitializeCriticalSection(&malloc_global_mutex);
+ interlockedexchange(&malloc_global_mutex_status, (LONG)1);
+ return;
+ }
+ SleepEx(0, FALSE);
+ }
+}
+
+#else /* pthreads-based locks */
+#define MLOCK_T pthread_mutex_t
+#define ACQUIRE_LOCK(lk) pthread_mutex_lock(lk)
+#define RELEASE_LOCK(lk) pthread_mutex_unlock(lk)
+#define TRY_LOCK(lk) (!pthread_mutex_trylock(lk))
+#define INITIAL_LOCK(lk) pthread_init_lock(lk)
+#define DESTROY_LOCK(lk) pthread_mutex_destroy(lk)
+
+#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 && defined(linux) && !defined(PTHREAD_MUTEX_RECURSIVE)
+/* Cope with old-style linux recursive lock initialization by adding */
+/* skipped internal declaration from pthread.h */
+extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr,
+ int __kind));
+#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP
+#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y)
+#endif /* USE_RECURSIVE_LOCKS ... */
+
+static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static int pthread_init_lock (MLOCK_T *lk) {
+ pthread_mutexattr_t attr;
+ if (pthread_mutexattr_init(&attr)) return 1;
+#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0
+ if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1;
+#endif
+ if (pthread_mutex_init(lk, &attr)) return 1;
+ if (pthread_mutexattr_destroy(&attr)) return 1;
+ return 0;
+}
+
+#endif /* ... lock types ... */
+
+/* Common code for all lock types */
+#define USE_LOCK_BIT (2U)
+
+#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK
+#define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex);
+#endif
+
+#ifndef RELEASE_MALLOC_GLOBAL_LOCK
+#define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex);
+#endif
+
+#endif /* USE_LOCKS */
+
+/* ----------------------- Chunk representations ------------------------ */
+
+/*
+ (The following includes lightly edited explanations by Colin Plumb.)
+
+ The malloc_chunk declaration below is misleading (but accurate and
+ necessary). It declares a "view" into memory allowing access to
+ necessary fields at known offsets from a given base.
+
+ Chunks of memory are maintained using a `boundary tag' method as
+ originally described by Knuth. (See the paper by Paul Wilson
+ ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a survey of such
+ techniques.) Sizes of free chunks are stored both in the front of
+ each chunk and at the end. This makes consolidating fragmented
+ chunks into bigger chunks fast. The head fields also hold bits
+ representing whether chunks are free or in use.
+
+ Here are some pictures to make it clearer. They are "exploded" to
+ show that the state of a chunk can be thought of as extending from
+ the high 31 bits of the head field of its header through the
+ prev_foot and PINUSE_BIT bit of the following chunk header.
+
+ A chunk that's in use looks like:
+
+ chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Size of previous chunk (if P = 0) |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
+ | Size of this chunk 1| +-+
+ mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | |
+ +- -+
+ | |
+ +- -+
+ | :
+ +- size - sizeof(size_t) available payload bytes -+
+ : |
+ chunk-> +- -+
+ | |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |1|
+ | Size of next chunk (may or may not be in use) | +-+
+ mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+ And if it's free, it looks like this:
+
+ chunk-> +- -+
+ | User payload (must be in use, or we would have merged!) |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
+ | Size of this chunk 0| +-+
+ mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Next pointer |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Prev pointer |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | :
+ +- size - sizeof(struct chunk) unused bytes -+
+ : |
+ chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Size of this chunk |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |0|
+ | Size of next chunk (must be in use, or we would have merged)| +-+
+ mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | :
+ +- User payload -+
+ : |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ |0|
+ +-+
+ Note that since we always merge adjacent free chunks, the chunks
+ adjacent to a free chunk must be in use.
+
+ Given a pointer to a chunk (which can be derived trivially from the
+ payload pointer) we can, in O(1) time, find out whether the adjacent
+ chunks are free, and if so, unlink them from the lists that they
+ are on and merge them with the current chunk.
+
+ Chunks always begin on even word boundaries, so the mem portion
+ (which is returned to the user) is also on an even word boundary, and
+ thus at least double-word aligned.
+
+ The P (PINUSE_BIT) bit, stored in the unused low-order bit of the
+ chunk size (which is always a multiple of two words), is an in-use
+ bit for the *previous* chunk. If that bit is *clear*, then the
+ word before the current chunk size contains the previous chunk
+ size, and can be used to find the front of the previous chunk.
+ The very first chunk allocated always has this bit set, preventing
+ access to non-existent (or non-owned) memory. If pinuse is set for
+ any given chunk, then you CANNOT determine the size of the
+ previous chunk, and might even get a memory addressing fault when
+ trying to do so.
+
+ The C (CINUSE_BIT) bit, stored in the unused second-lowest bit of
+ the chunk size redundantly records whether the current chunk is
+ inuse (unless the chunk is mmapped). This redundancy enables usage
+ checks within free and realloc, and reduces indirection when freeing
+ and consolidating chunks.
+
+ Each freshly allocated chunk must have both cinuse and pinuse set.
+ That is, each allocated chunk borders either a previously allocated
+ and still in-use chunk, or the base of its memory arena. This is
+ ensured by making all allocatio
<TRUNCATED>
[09/14] arrow git commit: [C++] Restore Plasma source tree after
0.5.0 release
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/xxhash.cc
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/xxhash.cc b/cpp/src/plasma/thirdparty/xxhash.cc
new file mode 100644
index 0000000..f74880b
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/xxhash.cc
@@ -0,0 +1,889 @@
+/*
+* xxHash - Fast Hash algorithm
+* Copyright (C) 2012-2016, Yann Collet
+*
+* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are
+* met:
+*
+* * Redistributions of source code must retain the above copyright
+* notice, this list of conditions and the following disclaimer.
+* * Redistributions in binary form must reproduce the above
+* copyright notice, this list of conditions and the following disclaimer
+* in the documentation and/or other materials provided with the
+* distribution.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+* You can contact the author at :
+* - xxHash homepage: http://www.xxhash.com
+* - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+
+/* *************************************
+* Tuning parameters
+***************************************/
+/*!XXH_FORCE_MEMORY_ACCESS :
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
+ * It can generate buggy code on targets which do not support unaligned memory accesses.
+ * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See http://stackoverflow.com/a/32095106/646947 for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
+ */
+#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
+# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+# define XXH_FORCE_MEMORY_ACCESS 2
+# elif defined(__INTEL_COMPILER) || \
+ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+# define XXH_FORCE_MEMORY_ACCESS 1
+# endif
+#endif
+
+/*!XXH_ACCEPT_NULL_INPUT_POINTER :
+ * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
+ * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
+ * By default, this option is disabled. To enable it, uncomment below define :
+ */
+/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
+
+/*!XXH_FORCE_NATIVE_FORMAT :
+ * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
+ * Results are therefore identical for little-endian and big-endian CPU.
+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+ * Should endian-independence be of no importance for your application, you may set the #define below to 1,
+ * to improve speed for Big-endian CPU.
+ * This option has no impact on Little_Endian CPU.
+ */
+#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */
+# define XXH_FORCE_NATIVE_FORMAT 0
+#endif
+
+/*!XXH_FORCE_ALIGN_CHECK :
+ * This is a minor performance trick, only useful with lots of very small keys.
+ * It means : check for aligned/unaligned input.
+ * The check costs one initial branch per hash; set to 0 when the input data
+ * is guaranteed to be aligned.
+ */
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+# define XXH_FORCE_ALIGN_CHECK 0
+# else
+# define XXH_FORCE_ALIGN_CHECK 1
+# endif
+#endif
+
+
+/* *************************************
+* Includes & Memory related functions
+***************************************/
+/*! Modify the local functions below should you wish to use some other memory routines
+* for malloc(), free() */
+#include <stdlib.h>
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void XXH_free (void* p) { free(p); }
+/*! and for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
+
+
+/* *************************************
+* Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER /* Visual Studio */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# define FORCE_INLINE static __forceinline
+#else
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
+# else
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
+#endif
+
+
+/* *************************************
+* Basic Types
+***************************************/
+#ifndef MEM_MODULE
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+# else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+# endif
+#endif
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; } __attribute__((packed)) unalign;
+static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+static U32 XXH_read32(const void* memPtr)
+{
+ U32 val;
+ memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* ****************************************
+* Compiler-specific Functions and Macros
+******************************************/
+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
+#if defined(_MSC_VER)
+# define XXH_rotl32(x,r) _rotl(x,r)
+# define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
+#endif
+
+#if defined(_MSC_VER) /* Visual Studio */
+# define XXH_swap32 _byteswap_ulong
+#elif XXH_GCC_VERSION >= 403
+# define XXH_swap32 __builtin_bswap32
+#else
+static U32 XXH_swap32 (U32 x)
+{
+ return ((x << 24) & 0xff000000 ) |
+ ((x << 8) & 0x00ff0000 ) |
+ ((x >> 8) & 0x0000ff00 ) |
+ ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* *************************************
+* Architecture Macros
+***************************************/
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+
+/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+ static const int g_one = 1;
+# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))
+#endif
+
+
+/* ***************************
+* Memory reads
+*****************************/
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+ if (align==XXH_unaligned)
+ return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+ else
+ return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
+}
+
+FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+{
+ return XXH_readLE32_align(ptr, endian, XXH_unaligned);
+}
+
+static U32 XXH_readBE32(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+
+
+/* *************************************
+* Macros
+***************************************/
+#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+* 32-bits hash functions
+*********************************************************************/
+static const U32 PRIME32_1 = 2654435761U;
+static const U32 PRIME32_2 = 2246822519U;
+static const U32 PRIME32_3 = 3266489917U;
+static const U32 PRIME32_4 = 668265263U;
+static const U32 PRIME32_5 = 374761393U;
+
+static U32 XXH32_round(U32 seed, U32 input)
+{
+ seed += input * PRIME32_2;
+ seed = XXH_rotl32(seed, 13);
+ seed *= PRIME32_1;
+ return seed;
+}
+
+FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
+{
+ const BYTE* p = (const BYTE*)input;
+ const BYTE* bEnd = p + len;
+ U32 h32;
+#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (p==NULL) {
+ len=0;
+ bEnd=p=(const BYTE*)(size_t)16;
+ }
+#endif
+
+ if (len>=16) {
+ const BYTE* const limit = bEnd - 16;
+ U32 v1 = seed + PRIME32_1 + PRIME32_2;
+ U32 v2 = seed + PRIME32_2;
+ U32 v3 = seed + 0;
+ U32 v4 = seed - PRIME32_1;
+
+ do {
+ v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
+ v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
+ v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
+ v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
+ } while (p<=limit);
+
+ h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+ } else {
+ h32 = seed + PRIME32_5;
+ }
+
+ h32 += (U32) len;
+
+ while (p+4<=bEnd) {
+ h32 += XXH_get32bits(p) * PRIME32_3;
+ h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
+ p+=4;
+ }
+
+ while (p<bEnd) {
+ h32 += (*p) * PRIME32_5;
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
+{
+#if 0
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+ XXH32_state_t state;
+ XXH32_reset(&state, seed);
+ XXH32_update(&state, input, len);
+ return XXH32_digest(&state);
+#else
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+ else
+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+ } }
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+ else
+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+
+
+/*====== Hash streaming ======*/
+
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+ XXH_free(statePtr);
+ return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+ memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
+{
+ XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+ memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */
+ state.v1 = seed + PRIME32_1 + PRIME32_2;
+ state.v2 = seed + PRIME32_2;
+ state.v3 = seed + 0;
+ state.v4 = seed - PRIME32_1;
+ memcpy(statePtr, &state, sizeof(state));
+ return XXH_OK;
+}
+
+
+FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+ const BYTE* p = (const BYTE*)input;
+ const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (input==NULL) return XXH_ERROR;
+#endif
+
+ state->total_len_32 += (unsigned)len;
+ state->large_len |= (len>=16) | (state->total_len_32>=16);
+
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
+ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
+ state->memsize += (unsigned)len;
+ return XXH_OK;
+ }
+
+ if (state->memsize) { /* some data left from previous update */
+ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
+ { const U32* p32 = state->mem32;
+ state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
+ state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
+ state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
+ state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
+ }
+ p += 16-state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p <= bEnd-16) {
+ const BYTE* const limit = bEnd - 16;
+ U32 v1 = state->v1;
+ U32 v2 = state->v2;
+ U32 v3 = state->v3;
+ U32 v4 = state->v4;
+
+ do {
+ v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
+ v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
+ v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
+ v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
+ } while (p<=limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
+
+ if (p < bEnd) {
+ XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
+ }
+
+ return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
+{
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+ else
+ return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
+{
+ const BYTE * p = (const BYTE*)state->mem32;
+ const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
+ U32 h32;
+
+ if (state->large_len) {
+ h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
+ } else {
+ h32 = state->v3 /* == seed */ + PRIME32_5;
+ }
+
+ h32 += state->total_len_32;
+
+ while (p+4<=bEnd) {
+ h32 += XXH_readLE32(p, endian) * PRIME32_3;
+ h32 = XXH_rotl32(h32, 17) * PRIME32_4;
+ p+=4;
+ }
+
+ while (p<bEnd) {
+ h32 += (*p) * PRIME32_5;
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+
+
+XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
+{
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_digest_endian(state_in, XXH_littleEndian);
+ else
+ return XXH32_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*====== Canonical representation ======*/
+
+/*! Default XXH result types are basic unsigned 32 and 64 bits.
+* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
+* These functions allow transformation of hash result into and from its canonical format.
+* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
+*/
+
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+ memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+ return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+* 64-bits hash functions
+*********************************************************************/
+
+/*====== Memory access ======*/
+
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint64_t U64;
+# else
+ typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
+# endif
+#endif
+
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
+static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+
+static U64 XXH_read64(const void* memPtr)
+{
+ U64 val;
+ memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER) /* Visual Studio */
+# define XXH_swap64 _byteswap_uint64
+#elif XXH_GCC_VERSION >= 403
+# define XXH_swap64 __builtin_bswap64
+#else
+static U64 XXH_swap64 (U64 x)
+{
+ return ((x << 56) & 0xff00000000000000ULL) |
+ ((x << 40) & 0x00ff000000000000ULL) |
+ ((x << 24) & 0x0000ff0000000000ULL) |
+ ((x << 8) & 0x000000ff00000000ULL) |
+ ((x >> 8) & 0x00000000ff000000ULL) |
+ ((x >> 24) & 0x0000000000ff0000ULL) |
+ ((x >> 40) & 0x000000000000ff00ULL) |
+ ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+ if (align==XXH_unaligned)
+ return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+ else
+ return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
+}
+
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+{
+ return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+}
+
+static U64 XXH_readBE64(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+
+
+/*====== xxh64 ======*/
+
+static const U64 PRIME64_1 = 11400714785074694791ULL;
+static const U64 PRIME64_2 = 14029467366897019727ULL;
+static const U64 PRIME64_3 = 1609587929392839161ULL;
+static const U64 PRIME64_4 = 9650029242287828579ULL;
+static const U64 PRIME64_5 = 2870177450012600261ULL;
+
+static U64 XXH64_round(U64 acc, U64 input)
+{
+ acc += input * PRIME64_2;
+ acc = XXH_rotl64(acc, 31);
+ acc *= PRIME64_1;
+ return acc;
+}
+
+static U64 XXH64_mergeRound(U64 acc, U64 val)
+{
+ val = XXH64_round(0, val);
+ acc ^= val;
+ acc = acc * PRIME64_1 + PRIME64_4;
+ return acc;
+}
+
+FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
+{
+ const BYTE* p = (const BYTE*)input;
+ const BYTE* bEnd = p + len;
+ U64 h64;
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (p==NULL) {
+ len=0;
+ bEnd=p=(const BYTE*)(size_t)32;
+ }
+#endif
+
+ if (len>=32) {
+ const BYTE* const limit = bEnd - 32;
+ U64 v1 = seed + PRIME64_1 + PRIME64_2;
+ U64 v2 = seed + PRIME64_2;
+ U64 v3 = seed + 0;
+ U64 v4 = seed - PRIME64_1;
+
+ do {
+ v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
+ v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
+ v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
+ v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
+ } while (p<=limit);
+
+ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+ h64 = XXH64_mergeRound(h64, v1);
+ h64 = XXH64_mergeRound(h64, v2);
+ h64 = XXH64_mergeRound(h64, v3);
+ h64 = XXH64_mergeRound(h64, v4);
+
+ } else {
+ h64 = seed + PRIME64_5;
+ }
+
+ h64 += (U64) len;
+
+ while (p+8<=bEnd) {
+ U64 const k1 = XXH64_round(0, XXH_get64bits(p));
+ h64 ^= k1;
+ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+ p+=8;
+ }
+
+ if (p+4<=bEnd) {
+ h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
+ h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ p+=4;
+ }
+
+ while (p<bEnd) {
+ h64 ^= (*p) * PRIME64_5;
+ h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+ p++;
+ }
+
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+
+ return h64;
+}
+
+
+XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+{
+#if 0
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+ XXH64_state_t state;
+ XXH64_reset(&state, seed);
+ XXH64_update(&state, input, len);
+ return XXH64_digest(&state);
+#else
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+ else
+ return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+ } }
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+ else
+ return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+/*====== Hash Streaming ======*/
+
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+ XXH_free(statePtr);
+ return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+ memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
+{
+ XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+ memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */
+ state.v1 = seed + PRIME64_1 + PRIME64_2;
+ state.v2 = seed + PRIME64_2;
+ state.v3 = seed + 0;
+ state.v4 = seed - PRIME64_1;
+ memcpy(statePtr, &state, sizeof(state));
+ return XXH_OK;
+}
+
+FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
+{
+ const BYTE* p = (const BYTE*)input;
+ const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (input==NULL) return XXH_ERROR;
+#endif
+
+ state->total_len += len;
+
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
+ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
+ state->memsize += (U32)len;
+ return XXH_OK;
+ }
+
+ if (state->memsize) { /* tmp buffer is full */
+ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
+ state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
+ state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
+ state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
+ state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
+ p += 32-state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p+32 <= bEnd) {
+ const BYTE* const limit = bEnd - 32;
+ U64 v1 = state->v1;
+ U64 v2 = state->v2;
+ U64 v3 = state->v3;
+ U64 v4 = state->v4;
+
+ do {
+ v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
+ v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
+ v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
+ v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
+ } while (p<=limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
+
+ if (p < bEnd) {
+ XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
+ }
+
+ return XXH_OK;
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
+{
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
+ else
+ return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
+{
+ const BYTE * p = (const BYTE*)state->mem64;
+ const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
+ U64 h64;
+
+ if (state->total_len >= 32) {
+ U64 const v1 = state->v1;
+ U64 const v2 = state->v2;
+ U64 const v3 = state->v3;
+ U64 const v4 = state->v4;
+
+ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+ h64 = XXH64_mergeRound(h64, v1);
+ h64 = XXH64_mergeRound(h64, v2);
+ h64 = XXH64_mergeRound(h64, v3);
+ h64 = XXH64_mergeRound(h64, v4);
+ } else {
+ h64 = state->v3 + PRIME64_5;
+ }
+
+ h64 += (U64) state->total_len;
+
+ while (p+8<=bEnd) {
+ U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
+ h64 ^= k1;
+ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+ p+=8;
+ }
+
+ if (p+4<=bEnd) {
+ h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
+ h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ p+=4;
+ }
+
+ while (p<bEnd) {
+ h64 ^= (*p) * PRIME64_5;
+ h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+ p++;
+ }
+
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+
+ return h64;
+}
+
+XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
+{
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH64_digest_endian(state_in, XXH_littleEndian);
+ else
+ return XXH64_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+/*====== Canonical representation ======*/
+
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+ memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+ return XXH_readBE64(src);
+}
+
+#endif /* XXH_NO_LONG_LONG */
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c810151/cpp/src/plasma/thirdparty/xxhash.h
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/thirdparty/xxhash.h b/cpp/src/plasma/thirdparty/xxhash.h
new file mode 100644
index 0000000..9d831e0
--- /dev/null
+++ b/cpp/src/plasma/thirdparty/xxhash.h
@@ -0,0 +1,293 @@
+/*
+ xxHash - Extremely Fast Hash algorithm
+ Header File
+ Copyright (C) 2012-2016, Yann Collet.
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name Speed Q.Score Author
+xxHash 5.4 GB/s 10
+CrapWow 3.2 GB/s 2 Andrew
+MumurHash 3a 2.7 GB/s 10 Austin Appleby
+SpookyHash 2.0 GB/s 10 Bob Jenkins
+SBox 1.4 GB/s 9 Bret Mulvey
+Lookup3 1.2 GB/s 9 Bob Jenkins
+SuperFastHash 1.2 GB/s 1 Paul Hsieh
+CityHash64 1.05 GB/s 10 Pike & Alakuijala
+FNV 0.55 GB/s 5 Fowler, Noll, Vo
+CRC32 0.43 GB/s 9
+MD5-32 0.33 GB/s 10 Ronald L. Rivest
+SHA1-32 0.28 GB/s 10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+A 64-bits version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bits applications only.
+Name Speed on 64 bits Speed on 32 bits
+XXH64 13.8 GB/s 1.9 GB/s
+XXH32 6.8 GB/s 6.0 GB/s
+*/
+
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************
+* Definitions
+******************************/
+#include <stddef.h> /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/* ****************************
+* API modifier
+******************************/
+/** XXH_PRIVATE_API
+* This is useful to include xxhash functions in `static` mode
+* in order to inline them, and remove their symbol from the public list.
+* Methodology :
+* #define XXH_PRIVATE_API
+* #include "xxhash.h"
+* `xxhash.c` is automatically included.
+* It's not useful to compile and link it as a separate module.
+*/
+#ifdef XXH_PRIVATE_API
+# ifndef XXH_STATIC_LINKING_ONLY
+# define XXH_STATIC_LINKING_ONLY
+# endif
+# if defined(__GNUC__)
+# define XXH_PUBLIC_API static __inline __attribute__((unused))
+# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define XXH_PUBLIC_API static inline
+# elif defined(_MSC_VER)
+# define XXH_PUBLIC_API static __inline
+# else
+# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */
+# endif
+#else
+# define XXH_PUBLIC_API /* do nothing */
+#endif /* XXH_PRIVATE_API */
+
+/*!XXH_NAMESPACE, aka Namespace Emulation :
+
+If you want to include _and expose_ xxHash functions from within your own library,
+but also want to avoid symbol collisions with other libraries which may also include xxHash,
+
+you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
+with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
+
+Note that no change is required within the calling program as long as it includes `xxhash.h` :
+regular symbol name will be automatically translated by this header.
+*/
+#ifdef XXH_NAMESPACE
+# define XXH_CAT(A,B) A##B
+# define XXH_NAME2(A,B) XXH_CAT(A,B)
+# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+#endif
+
+
+/* *************************************
+* Version
+***************************************/
+#define XXH_VERSION_MAJOR 0
+#define XXH_VERSION_MINOR 6
+#define XXH_VERSION_RELEASE 2
+#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+XXH_PUBLIC_API unsigned XXH_versionNumber (void);
+
+
+/*-**********************************************************************
+* 32-bits hash
+************************************************************************/
+typedef unsigned int XXH32_hash_t;
+
+/*! XXH32() :
+ Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
+ The memory between input & input+length must be valid (allocated and read-accessible).
+ "seed" can be used to alter the result predictably.
+ Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
+
+/*====== Streaming ======*/
+typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
+
+/*
+These functions generate the xxHash of an input provided in multiple segments.
+Note that, for small input, they are slower than single-call functions, due to state management.
+For small input, prefer `XXH32()` and `XXH64()` .
+
+XXH state must first be allocated, using XXH*_createState() .
+
+Start a new hash by initializing state with a seed, using XXH*_reset().
+
+Then, feed the hash state by calling XXH*_update() as many times as necessary.
+Obviously, input must be allocated and read accessible.
+The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+
+Finally, a hash value can be produced anytime, by using XXH*_digest().
+This function returns the nn-bits hash as an int or long long.
+
+It's still possible to continue inserting input into the hash state after a digest,
+and generate some new hashes later on, by calling again XXH*_digest().
+
+When done, free XXH state space if it was allocated dynamically.
+*/
+
+/*====== Canonical representation ======*/
+
+typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+* These functions allow transformation of hash result into and from its canonical format.
+* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+*/
+
+
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+* 64-bits hash
+************************************************************************/
+typedef unsigned long long XXH64_hash_t;
+
+/*! XXH64() :
+ Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
+ "seed" can be used to alter the result predictably.
+ This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
+*/
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
+
+/*====== Streaming ======*/
+typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
+
+/*====== Canonical representation ======*/
+typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+#endif /* XXH_NO_LONG_LONG */
+
+
+#ifdef XXH_STATIC_LINKING_ONLY
+
+/* ================================================================================================
+ This section contains definitions which are not guaranteed to remain stable.
+ They may change in future versions, becoming incompatible with a different version of the library.
+ They shall only be used with static linking.
+ Never use these definitions in association with dynamic linking !
+=================================================================================================== */
+
+/* These definitions are only meant to make possible
+ static allocation of XXH state, on stack or in a struct for example.
+ Never use members directly. */
+
+struct XXH32_state_s {
+ unsigned total_len_32;
+ unsigned large_len;
+ unsigned v1;
+ unsigned v2;
+ unsigned v3;
+ unsigned v4;
+ unsigned mem32[4]; /* buffer defined as U32 for alignment */
+ unsigned memsize;
+ unsigned reserved; /* never read nor write, will be removed in a future version */
+}; /* typedef'd to XXH32_state_t */
+
+#ifndef XXH_NO_LONG_LONG /* remove 64-bits support */
+struct XXH64_state_s {
+ unsigned long long total_len;
+ unsigned long long v1;
+ unsigned long long v2;
+ unsigned long long v3;
+ unsigned long long v4;
+ unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
+ unsigned memsize;
+ unsigned reserved[2]; /* never read nor write, will be removed in a future version */
+}; /* typedef'd to XXH64_state_t */
+#endif
+
+#ifdef XXH_PRIVATE_API
+# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */
+#endif
+
+#endif /* XXH_STATIC_LINKING_ONLY */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* XXHASH_H_5627135585666179 */