You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/09/06 03:49:14 UTC
[46/53] [abbrv] git commit: ACCUMULO-658 Move native library to its
own module
ACCUMULO-658 Move native library to its own module
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d1b449d3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d1b449d3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d1b449d3
Branch: refs/heads/ACCUMULO-210
Commit: d1b449d3f615ce6df1ff8498343ccb150be4e130
Parents: fc9363a
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu Aug 22 22:07:07 2013 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Sep 5 21:47:33 2013 -0400
----------------------------------------------------------------------
assemble/pom.xml | 11 +-
pom.xml | 35 ++-
proxy/pom.xml | 3 +-
server/native/pom.xml | 73 +++++
.../src/main/c++/nativeMap/BlockAllocator.h | 268 +++++++++++++++++++
server/native/src/main/c++/nativeMap/Field.h | 143 ++++++++++
server/native/src/main/c++/nativeMap/Key.h | 143 ++++++++++
.../native/src/main/c++/nativeMap/NativeMap.h | 209 +++++++++++++++
server/native/src/main/c++/nativeMap/SubKey.h | 187 +++++++++++++
.../org_apache_accumulo_tserver_NativeMap.cc | 130 +++++++++
server/native/src/test/c++/nativeMap/test.cc | 134 ++++++++++
server/native/src/test/c++/nativeMap/util.cc | 32 +++
server/native/src/test/c++/nativeMap/util.h | 19 ++
server/server/pom.xml | 54 ----
server/server/src/main/c++/Makefile | 23 --
.../src/main/c++/nativeMap/BlockAllocator.h | 268 -------------------
server/server/src/main/c++/nativeMap/Field.h | 143 ----------
server/server/src/main/c++/nativeMap/Key.h | 143 ----------
server/server/src/main/c++/nativeMap/Makefile | 75 ------
.../server/src/main/c++/nativeMap/NativeMap.h | 209 ---------------
server/server/src/main/c++/nativeMap/SubKey.h | 193 -------------
...he_accumulo_server_tabletserver_NativeMap.cc | 149 -----------
...che_accumulo_server_tabletserver_NativeMap.h | 133 ---------
..._tabletserver_NativeMap_ConcurrentIterator.h | 33 ---
...mulo_server_tabletserver_NativeMap_NMEntry.h | 29 --
...o_server_tabletserver_NativeMap_NMIterator.h | 29 --
...lo_server_tabletserver_NativeMap_NMSKVIter.h | 29 --
server/server/src/main/c++/nativeMap/test.cc | 152 -----------
server/server/src/main/c++/nativeMap/util.cc | 40 ---
server/server/src/main/c++/nativeMap/util.h | 19 --
30 files changed, 1375 insertions(+), 1733 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/assemble/pom.xml
----------------------------------------------------------------------
diff --git a/assemble/pom.xml b/assemble/pom.xml
index db40b65..62cd0a2 100644
--- a/assemble/pom.xml
+++ b/assemble/pom.xml
@@ -68,10 +68,6 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-start</artifactId>
</dependency>
<dependency>
@@ -83,6 +79,11 @@
<artifactId>accumulo-trace</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>libaccumulo</artifactId>
+ <type>so</type>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-vfs2</artifactId>
</dependency>
@@ -368,7 +369,7 @@
<classifier>native</classifier>
<needarch>x86_64</needarch>
<provides>
- <provide>accumulo-native</provide>
+ <provide>libaccumulo</provide>
</provides>
<requires>
<require>accumulo</require>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d42b45a..f2e9eef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -87,7 +87,7 @@
<module>server/gc</module>
<module>server/master</module>
<module>server/monitor</module>
- <module>server/server</module>
+ <module>server/native</module>
<module>server/tserver</module>
<module>server/utils</module>
</modules>
@@ -267,11 +267,6 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-server-base</artifactId>
<version>${project.version}</version>
</dependency>
@@ -296,6 +291,12 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>libaccumulo</artifactId>
+ <version>${project.version}</version>
+ <type>so</type>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-jci-core</artifactId>
<version>1.0</version>
@@ -430,6 +431,11 @@
</configuration>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>native-maven-plugin</artifactId>
+ <version>1.0-alpha-7</version>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
@@ -606,6 +612,23 @@
<pluginExecution>
<pluginExecutionFilter>
<groupId>org.codehaus.mojo</groupId>
+ <artifactId>native-maven-plugin</artifactId>
+ <versionRange>[1.0-alpha-7,)</versionRange>
+ <goals>
+ <goal>compile</goal>
+ <goal>initialize</goal>
+ <goal>javah</goal>
+ <goal>link</goal>
+ <goal>unzipinc</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<versionRange>[1.0,)</versionRange>
<goals>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/proxy/pom.xml b/proxy/pom.xml
index ce9e747..e1edcc3 100644
--- a/proxy/pom.xml
+++ b/proxy/pom.xml
@@ -43,7 +43,8 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server</artifactId>
+ <artifactId>libaccumulo</artifactId>
+ <type>so</type>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/pom.xml
----------------------------------------------------------------------
diff --git a/server/native/pom.xml b/server/native/pom.xml
new file mode 100644
index 0000000..c360ed4
--- /dev/null
+++ b/server/native/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-project</artifactId>
+ <version>1.6.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <artifactId>libaccumulo</artifactId>
+ <packaging>so</packaging>
+ <name>Native Libraries</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-tserver</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>native-maven-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <javahOS>linux</javahOS>
+ <javahClassNames>
+ <javahClassName>org.apache.accumulo.tserver.NativeMap</javahClassName>
+ </javahClassNames>
+ <compilerProvider>generic</compilerProvider>
+ <compilerStartOptions>
+ <compilerStartOption>-g</compilerStartOption>
+ <compilerStartOption>-shared</compilerStartOption>
+ <compilerStartOption>-fPIC</compilerStartOption>
+ <compilerStartOption>-O2</compilerStartOption>
+ <compilerStartOption>-fno-omit-frame-pointer</compilerStartOption>
+ <compilerStartOption>-fno-strict-aliasing</compilerStartOption>
+ <compilerStartOption>-Wall</compilerStartOption>
+ </compilerStartOptions>
+ <sources>
+ <source>
+ <directory>src/main/c++/nativeMap</directory>
+ <fileNames>
+ <fileName>org_apache_accumulo_tserver_NativeMap.cc</fileName>
+ </fileNames>
+ </source>
+ </sources>
+ <linkerStartOptions>
+ <linkerStartOption>-g</linkerStartOption>
+ <linkerStartOption>-shared</linkerStartOption>
+ <linkerStartOption>-fPIC</linkerStartOption>
+ </linkerStartOptions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/main/c++/nativeMap/BlockAllocator.h
----------------------------------------------------------------------
diff --git a/server/native/src/main/c++/nativeMap/BlockAllocator.h b/server/native/src/main/c++/nativeMap/BlockAllocator.h
new file mode 100644
index 0000000..3126217
--- /dev/null
+++ b/server/native/src/main/c++/nativeMap/BlockAllocator.h
@@ -0,0 +1,268 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+
+#ifndef _BLOCK_ALLOCATOR_H_
+#define _BLOCK_ALLOCATOR_H_ 1
+
+#include <new>
+#include <bits/functexcept.h>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <stdlib.h>
+#include <stddef.h>
+
+struct Block {
+ unsigned char *data;
+ unsigned char *currentPos;
+ unsigned char *end;
+ unsigned char *prevPos;
+
+ Block(uint32_t size){
+ data = new unsigned char[size];
+ end = data + size;
+ currentPos = data;
+ prevPos = NULL;
+ }
+
+ ~Block(){
+ }
+
+ void *allocate(size_t amount){
+ unsigned char *nextPos = currentPos + amount;
+
+ if(nextPos > end){
+ return NULL;
+ }
+
+ prevPos = currentPos;
+ currentPos = nextPos;
+ return prevPos;
+ }
+
+ size_t rollback(void *p){
+ if(p == prevPos){
+ size_t diff = currentPos - prevPos;
+ currentPos = prevPos;
+ return diff;
+ }else{
+ std::cerr << "Tried to delete something that was not previous allocation " << p << " " << prevPos << std::endl;
+ exit(-1);
+ }
+
+ return 0;
+ }
+
+ size_t getMemoryFree() {
+ return end - currentPos;
+ }
+};
+
+struct BigBlock {
+ unsigned char *ptr;
+ size_t length;
+
+ BigBlock(unsigned char *p, size_t len):ptr(p),length(len){}
+};
+
+struct LinkedBlockAllocator {
+
+ std::vector<Block> blocks;
+ std::vector<BigBlock> bigBlocks;
+ int blockSize;
+ int bigBlockSize;
+ int64_t memused;
+ void *lastAlloc;
+
+ LinkedBlockAllocator(int blockSize, int bigBlockSize){
+ this->blockSize = blockSize;
+ this->bigBlockSize = bigBlockSize;
+ lastAlloc = NULL;
+ memused = 0;
+ }
+
+ void *allocate(size_t amount){
+
+ if(amount > (size_t)bigBlockSize){
+ unsigned char *p = new unsigned char[amount];
+ bigBlocks.push_back(BigBlock(p, amount));
+ memused += sizeof(BigBlock) + amount;
+ return p;
+ }else{
+ if(blocks.size() == 0){
+ //do lazy allocation of memory, do not allocate a block until it is used
+ blocks.push_back(Block(blockSize));
+ memused += sizeof(Block) + blockSize;
+ }
+
+ lastAlloc = blocks.back().allocate(amount);
+ if(lastAlloc == NULL){
+ blocks.push_back(Block(blockSize));
+ lastAlloc = blocks.back().allocate(amount);
+ memused += sizeof(Block) + blockSize;
+ }
+
+ return lastAlloc;
+ }
+ }
+
+ void deleteLast(void *p){
+ if(p != NULL){
+ if(p == lastAlloc){
+ blocks.back().rollback(p);
+ lastAlloc = NULL;
+ return;
+ }else if(bigBlocks.back().ptr == p){
+ memused -= (sizeof(BigBlock) + bigBlocks.back().length);
+ bigBlocks.pop_back();
+ delete((unsigned char *)p);
+ return;
+ }
+ }
+
+ std::cerr << "Tried to delete something that was not last allocation " << p << " " << lastAlloc << std::endl;
+ exit(-1);
+ }
+
+ size_t getMemoryUsed(){
+ if(blocks.size() == 0)
+ return memused;
+ else
+ return memused - blocks.back().getMemoryFree();
+ }
+
+ ~LinkedBlockAllocator(){
+ //std::cout << "Deleting " << blocks.size() << " blocks, memused : " << memused << std::endl;
+ std::vector<Block>::iterator iter = blocks.begin();
+ while(iter != blocks.end()){
+ delete(iter->data);
+ iter++;
+ }
+
+ std::vector<BigBlock>::iterator iter2 = bigBlocks.begin();
+ while(iter2 != bigBlocks.end()){
+ delete(iter2->ptr);
+ iter2++;
+ }
+ }
+
+};
+
+
+ /**
+ * @brief An allocator that uses global new, as per [20.4].
+ *
+ * This is precisely the allocator defined in the C++ Standard.
+ * - all allocation calls operator new
+ * - all deallocation calls operator delete
+ */
+ template<typename _Tp>
+ class BlockAllocator
+ {
+ public:
+ typedef size_t size_type;
+ typedef ptrdiff_t difference_type;
+ typedef _Tp* pointer;
+ typedef const _Tp* const_pointer;
+ typedef _Tp& reference;
+ typedef const _Tp& const_reference;
+ typedef _Tp value_type;
+
+ LinkedBlockAllocator *lba;
+
+ template<typename _Tp1>
+ struct rebind
+ { typedef BlockAllocator<_Tp1> other; };
+
+ BlockAllocator() throw() {
+ lba = NULL;
+ }
+
+ BlockAllocator(LinkedBlockAllocator *lba) throw() {
+ this->lba = lba;
+ }
+
+ BlockAllocator(const BlockAllocator& ba) throw() {
+ lba = ba.lba;
+ }
+
+ template<typename _Tp1>
+ BlockAllocator(const BlockAllocator<_Tp1>& ba) throw() {
+ lba = ba.lba;
+ }
+
+ ~BlockAllocator() throw() { }
+
+ pointer
+ address(reference __x) const { return &__x; }
+
+ const_pointer
+ address(const_reference __x) const { return &__x; }
+
+ // NB: __n is permitted to be 0. The C++ standard says nothing
+ // about what the return value is when __n == 0.
+ pointer
+ allocate(size_type __n, const void* = 0)
+ {
+ if (__builtin_expect(__n > this->max_size(), false))
+ std::__throw_bad_alloc();
+
+
+ //void *p = ::operator new(__n * sizeof(_Tp));
+ void *p = lba->allocate(__n * sizeof(_Tp));
+
+ //std::cout << "Allocating "<< name <<" " << __n * sizeof(_Tp) << " " << ((unsigned long long)p) % 4 << " " << ((unsigned long long)p) % 8 << std::endl;
+
+ return static_cast<_Tp*>(p);
+ }
+
+ // __p is not permitted to be a null pointer.
+ void
+ deallocate(pointer __p, size_type)
+ {
+ //::operator delete(__p);
+ }
+
+ size_type
+ max_size() const throw()
+ { return size_t(-1) / sizeof(_Tp); }
+
+ // _GLIBCXX_RESOLVE_LIB_DEFECTS
+ // 402. wrong new expression in [some_] allocator::construct
+ void
+ construct(pointer __p, const _Tp& __val)
+ { ::new(__p) _Tp(__val); }
+
+ void
+ destroy(pointer __p) { __p->~_Tp(); }
+
+
+ };
+
+
+ template<typename _Tp>
+ inline bool
+ operator==(const BlockAllocator<_Tp>& ba1, const BlockAllocator<_Tp>& ba2)
+ { return true; }
+
+ template<typename _Tp>
+ inline bool
+ operator!=(const BlockAllocator<_Tp>& ba1, const BlockAllocator<_Tp>& ba2)
+ { return false; }
+
+#endif
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/main/c++/nativeMap/Field.h
----------------------------------------------------------------------
diff --git a/server/native/src/main/c++/nativeMap/Field.h b/server/native/src/main/c++/nativeMap/Field.h
new file mode 100644
index 0000000..7c3884e
--- /dev/null
+++ b/server/native/src/main/c++/nativeMap/Field.h
@@ -0,0 +1,143 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#include <stdint.h>
+#include <string.h>
+#include <jni.h>
+#include <string>
+#include <string.h>
+#include <iostream>
+#include <stdlib.h>
+#include "BlockAllocator.h"
+
+using namespace std;
+
+#ifndef __FIELD__
+#define __FIELD__
+
+struct Field {
+ uint8_t *field;
+ int32_t len;
+
+ int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{
+ int result = memcmp(d1, d2, len1 < len2 ? len1 : len2);
+
+ if(result != 0)
+ return result;
+ if(len1 == len2)
+ return 0;
+ if(len1 < len2)
+ return -1;
+
+ return 1;
+ }
+
+ Field(){}
+
+ Field(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray f, int l){
+ len = l;
+ field=(uint8_t *)lba->allocate(len);
+ env->GetByteArrayRegion(f, 0, len, (jbyte *)field);
+ }
+
+ Field(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray f){
+ len = env->GetArrayLength(f);
+ field=(uint8_t *)lba->allocate(len);
+ env->GetByteArrayRegion(f, 0, len, (jbyte *)field);
+ }
+
+ Field(uint8_t *f, int32_t l):field(f),len(l){
+ }
+
+ Field(const char *cstr){
+ //constructor for testing C++
+ len = strlen(cstr);
+ field=new uint8_t[len];
+ memcpy(field, cstr, len);
+ }
+
+ Field(LinkedBlockAllocator *lba, const char *cstr){
+ //constructor for testing C++
+ len = strlen(cstr);
+ field=(uint8_t *)lba->allocate(len);
+ memcpy(field, cstr, len);
+ }
+
+ void set(const char *d, int l){
+ if(l < 0 || l > len){
+ cerr << "Tried to set field with value that is too long " << l << " " << len << endl;
+ }
+ memcpy(field, d, l);
+ len = l;
+ }
+
+ void set(JNIEnv *env, jbyteArray f, int l){
+ if(l < 0 || l > len){
+ cerr << "Tried to set field with value that is too long " << l << " " << len << endl;
+ }
+ len = l;
+ env->GetByteArrayRegion(f, 0, len, (jbyte *)field);
+ }
+
+ int compare(const Field &of) const{
+ return compare(field, len, of.field, of.len);
+ }
+
+ bool operator<(const Field &of) const{
+ return compare(of) < 0;
+ }
+
+ int32_t length() const {
+ return len;
+ }
+
+ void fillIn(JNIEnv *env, jbyteArray d) const {
+ //TODO ensure lengths match up
+ env->SetByteArrayRegion(d, 0, len, (jbyte *)field);
+ }
+
+ jbyteArray createJByteArray(JNIEnv *env) const{
+ jbyteArray valData = env->NewByteArray(len);
+ env->SetByteArrayRegion(valData, 0, len, (jbyte *)field);
+ return valData;
+ }
+
+ string toString() const{
+ return string((char *)field, len);
+ }
+
+ void clear(){
+ //delete(field);
+ }
+
+ void clear(LinkedBlockAllocator *lba){
+ lba->deleteLast(field);
+ }
+};
+
+struct LocalField : public Field {
+ LocalField(JNIEnv *env, jbyteArray f){
+ len = env->GetArrayLength(f);
+ field= new uint8_t[len];
+ env->GetByteArrayRegion(f, 0, len, (jbyte *)field);
+ }
+
+ ~LocalField(){
+ delete(field);
+ }
+};
+#endif
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/main/c++/nativeMap/Key.h
----------------------------------------------------------------------
diff --git a/server/native/src/main/c++/nativeMap/Key.h b/server/native/src/main/c++/nativeMap/Key.h
new file mode 100644
index 0000000..f902426
--- /dev/null
+++ b/server/native/src/main/c++/nativeMap/Key.h
@@ -0,0 +1,143 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#include <algorithm>
+#include <stdint.h>
+#include <string.h>
+#include <string>
+#include <jni.h>
+
+using namespace std;
+
+class Key {
+
+public:
+ int32_t colFamilyOffset;
+ int32_t colQualifierOffset;
+ int32_t colVisibilityOffset;
+ int32_t totalLen;
+
+ uint8_t *keyData;
+
+ int64_t timestamp;
+ bool deleted;
+
+ int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{
+ int result = memcmp(d1, d2, len1 < len2 ? len1 : len2);
+
+ if(result != 0)
+ return result;
+ if(len1 == len2)
+ return 0;
+ if(len1 < len2)
+ return -1;
+
+ return 1;
+ }
+
+ /**
+ * Constructor for testing purposes
+ */
+
+ Key(){}
+
+ Key(const string &r, const string &cf, const string &cq, const string &cv, long ts, bool del){
+
+ colFamilyOffset = r.length();
+ colQualifierOffset = colFamilyOffset + cf.length();
+ colVisibilityOffset = colQualifierOffset + cq.length();
+ totalLen = colVisibilityOffset + cv.length();
+
+ keyData = new uint8_t[totalLen];
+
+ copy(r.begin(), r.end(), keyData);
+ copy(cf.begin(), cf.end(), keyData+colFamilyOffset);
+ copy(cq.begin(), cq.end(), keyData+colQualifierOffset);
+ copy(cv.begin(), cv.end(), keyData+colVisibilityOffset);
+
+ timestamp = ts;
+ deleted = del;
+ }
+
+ /**
+ * Constructor used for taking data from Java Key
+ */
+
+ Key(JNIEnv *env, jbyteArray kd, jint cfo, jint cqo, jint cvo, jint tl, jlong ts, jboolean del){
+
+ colFamilyOffset = cfo;
+ colQualifierOffset = cqo;
+ colVisibilityOffset = cvo;
+ totalLen = tl;
+ timestamp = ts;
+ deleted = del == JNI_TRUE ? true : false;
+
+ keyData = new uint8_t[totalLen];
+ env->GetByteArrayRegion(kd, 0, totalLen, (jbyte *)keyData);
+ }
+
+ bool operator<(const Key &key) const{
+ int result = compare(keyData, colFamilyOffset, key.keyData, key.colFamilyOffset);
+ if(result != 0) return result < 0;
+
+ result = compare(keyData + colFamilyOffset, colQualifierOffset - colFamilyOffset, key.keyData + key.colFamilyOffset, key.colQualifierOffset - key.colFamilyOffset);
+ if(result != 0) return result < 0;
+
+ result = compare(keyData + colQualifierOffset, colVisibilityOffset - colQualifierOffset, key.keyData + key.colQualifierOffset, key.colVisibilityOffset - key.colQualifierOffset);
+ if(result != 0) return result < 0;
+
+ result = compare(keyData + colVisibilityOffset, totalLen - colVisibilityOffset, key.keyData + key.colVisibilityOffset, key.totalLen - key.colVisibilityOffset);
+ if(result != 0) return result < 0;
+
+ if(timestamp < key.timestamp){
+ return false;
+ }else if(timestamp > key.timestamp){
+ return true;
+ }
+
+ return deleted && !key.deleted;
+ }
+
+};
+
+
+class LocalKey : public Key {
+
+public:
+
+ JNIEnv *envPtr;
+ jbyteArray kd;
+
+ LocalKey(JNIEnv *env, jbyteArray kd, jint cfo, jint cqo, jint cvo, jint tl, jlong ts, jboolean del){
+ envPtr = env;
+
+ colFamilyOffset = cfo;
+ colQualifierOffset = cqo;
+ colVisibilityOffset = cvo;
+ totalLen = tl;
+ timestamp = ts;
+ deleted = del == JNI_TRUE ? true : false;
+
+ this->kd = kd;
+ keyData = (uint8_t *)env->GetByteArrayElements((jbyteArray)kd, NULL);
+ }
+
+ ~LocalKey(){
+ envPtr->ReleaseByteArrayElements(kd, (jbyte *)keyData, JNI_ABORT);
+ }
+
+};
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/main/c++/nativeMap/NativeMap.h
----------------------------------------------------------------------
diff --git a/server/native/src/main/c++/nativeMap/NativeMap.h b/server/native/src/main/c++/nativeMap/NativeMap.h
new file mode 100644
index 0000000..85060d6
--- /dev/null
+++ b/server/native/src/main/c++/nativeMap/NativeMap.h
@@ -0,0 +1,209 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#ifndef __NATIVE_MAP_A12__
+#define __NATIVE_MAP_A12__
+
+#include "SubKey.h"
+#include "Field.h"
+#include "BlockAllocator.h"
+#include <map>
+#include <vector>
+#include <iostream>
+
+using namespace std;
+
+typedef map<SubKey, Field, std::less<SubKey>, BlockAllocator<std::pair<const SubKey, Field> > > ColumnMap;
+typedef map<Field, ColumnMap, std::less<Field>, BlockAllocator<std::pair<const Field, ColumnMap> > > RowMap;
+
+
+struct NativeMapData {
+ LinkedBlockAllocator *lba;
+ RowMap rowmap;
+ int count;
+
+ NativeMapData(int blockSize, int bigBlockSize):lba(new LinkedBlockAllocator(blockSize, bigBlockSize)),
+ rowmap(RowMap(std::less<Field>(), BlockAllocator<std::pair<Field, ColumnMap> >(lba))){
+ this->lba = lba;
+ }
+
+ ~NativeMapData(){
+ rowmap.clear(); //if row map is not cleared here, it will be deconstructed after lba is deleted
+ delete(lba);
+ }
+};
+
+
+struct Iterator {
+ NativeMapData &nativeMap;
+ RowMap::iterator rowIter;
+ ColumnMap::iterator colIter;
+
+ Iterator(NativeMapData &nm, int32_t *ia):nativeMap(nm){
+ rowIter = nativeMap.rowmap.begin();
+ if(rowIter == nativeMap.rowmap.end()){
+ return;
+ }
+
+ colIter = rowIter->second.begin();
+
+ skipAndFillIn(ia, true);
+ }
+
+
+ Iterator(NativeMapData &nm, Field &row, SubKey &sk, int32_t *ia):nativeMap(nm){
+ rowIter = nativeMap.rowmap.lower_bound(row);
+ if(rowIter == nativeMap.rowmap.end()){
+ return;
+ }
+
+ //TODO use equals instead of compare
+ if(rowIter->first.compare(row) == 0){
+ colIter = rowIter->second.lower_bound(sk);
+ }else{
+ colIter = rowIter->second.begin();
+ }
+
+ skipAndFillIn(ia, true);
+ }
+
+ bool skipAndFillIn(int32_t *ia, bool firstCall)
+ {
+ bool rowChanged = false;
+
+ while(colIter == rowIter->second.end()){
+ rowIter++;
+ rowChanged = true;
+ if(rowIter == nativeMap.rowmap.end()){
+ return false;
+ }
+ colIter = rowIter->second.begin();
+ }
+
+ ia[0] = (firstCall || rowChanged) ? rowIter->first.length() : -1;
+ ia[1] = colIter->first.getCFLen();
+ ia[2] = colIter->first.getCQLen();
+ ia[3] = colIter->first.getCVLen();
+ ia[4] = colIter->first.isDeleted() ? 1 : 0;
+ ia[5] = colIter->second.length();
+ ia[6] = colIter->first.getMC();
+
+ return true;
+ }
+
+ bool atEnd(){
+ return rowIter == nativeMap.rowmap.end();
+ }
+
+ void advance(int32_t *ia){
+ colIter++;
+ skipAndFillIn(ia, false);
+ }
+};
+
+struct NativeMap : public NativeMapData {
+
+ NativeMap(int blockSize, int bigBlockSize):NativeMapData(blockSize, bigBlockSize){
+ count = 0;
+ }
+
+ ~NativeMap(){
+
+ }
+
+
+ ColumnMap *startUpdate(JNIEnv * env, jbyteArray r){
+ Field row(lba, env, r);
+ return startUpdate(row);
+ }
+
+ ColumnMap *startUpdate(const char *r){
+
+ Field row(lba, r);
+ return startUpdate(row);
+ }
+
+ ColumnMap *startUpdate(Field &row){
+ //cout << "Starting update "<<row.toString()<<endl;
+
+ pair<RowMap::iterator, bool> insertResult = rowmap.insert(pair<Field, ColumnMap>(row, ColumnMap(std::less<SubKey>(), BlockAllocator<std::pair<SubKey, Field> >(lba))));
+
+ if(!insertResult.second)
+ row.clear(lba);
+
+ return &(insertResult.first->second);
+
+ }
+
+ void update(ColumnMap *cm, JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jbyteArray val, jint mutationCount){
+
+ SubKey sk(lba, env, cf, cq, cv, ts, del, mutationCount);
+ //cout << "Updating " << sk.toString() << " " << sk.getTimestamp() << " " << sk.isDeleted() << endl;
+ //do not bother allocating value if not needed
+ Field value(NULL, 0);
+
+ pair<ColumnMap::iterator, bool> insertResult = cm->insert(pair<SubKey, Field>(sk, value));
+ if(insertResult.second){
+ insertResult.first->second = Field(lba, env, val);
+ count++;
+ }else{
+ sk.clear(lba);
+ int valLen = env->GetArrayLength(val);
+ if(valLen <= insertResult.first->second.length()){
+ insertResult.first->second.set(env, val, valLen);
+ }else{
+ insertResult.first->second.clear();
+ insertResult.first->second = Field(lba, env, val, valLen);
+ }
+ }
+ }
+
+ void update(ColumnMap *cm, const char *cf, const char *cq, const char *cv, long ts, bool del, const char *val, int valLen, int mutationCount){
+
+ SubKey sk(lba, cf, cq, cv, ts, del, mutationCount);
+ //do not bother allocating value if not needed
+ Field value(NULL, 0);
+
+ pair<ColumnMap::iterator, bool> insertResult = cm->insert(pair<SubKey, Field>(sk, value));
+ if(insertResult.second){
+ insertResult.first->second = Field(lba, val);
+ count++;
+ }else{
+ sk.clear(lba);
+ if(insertResult.first->second.length() <= valLen){
+ insertResult.first->second.set(val, valLen);
+ }else{
+ insertResult.first->second.clear();
+ insertResult.first->second = Field(lba, val);
+ }
+ }
+ }
+
+ Iterator *iterator(int32_t *ia){
+ return new Iterator(*this, ia);
+ }
+
+ Iterator *iterator(Field &row, SubKey &sk, int32_t *ia){
+ return new Iterator(*this, row, sk, ia);
+ }
+
+ int64_t getMemoryUsed(){
+ return lba->getMemoryUsed();
+ }
+};
+
+#endif
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/main/c++/nativeMap/SubKey.h
----------------------------------------------------------------------
diff --git a/server/native/src/main/c++/nativeMap/SubKey.h b/server/native/src/main/c++/nativeMap/SubKey.h
new file mode 100644
index 0000000..0522308
--- /dev/null
+++ b/server/native/src/main/c++/nativeMap/SubKey.h
@@ -0,0 +1,187 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#include <jni.h>
+#include <limits.h>
+#include "Field.h"
+#include "BlockAllocator.h"
+
+using namespace std;
+
+#ifndef __SUB_KEY__
+#define __SUB_KEY__
+
+class SubKey {
+
+public:
+
+ int32_t colQualifierOffset;
+ int32_t colVisibilityOffset;
+ int32_t totalLen;
+ uint8_t *keyData;
+ int64_t timestamp;
+ int32_t mutationCount;
+ bool deleted;
+
+ int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{
+ int result = memcmp(d1, d2, len1 < len2 ? len1 : len2);
+
+ if(result != 0)
+ return result;
+ if(len1 == len2)
+ return 0;
+ if(len1 < len2)
+ return -1;
+
+ return 1;
+ }
+
+ /**
+ * Constructor for testing purposes
+ */
+ SubKey(LinkedBlockAllocator *lba, const string &cf, const string &cq, const string &cv, int64_t ts, bool del, int32_t mc){
+
+ colQualifierOffset = cf.length();
+ colVisibilityOffset = colQualifierOffset + cq.length();
+ totalLen = colVisibilityOffset + cv.length();
+
+ keyData = (uint8_t *)lba->allocate(totalLen);
+
+ copy(cf.begin(), cf.end(), keyData);
+ copy(cq.begin(), cq.end(), keyData+colQualifierOffset);
+ copy(cv.begin(), cv.end(), keyData+colVisibilityOffset);
+
+ timestamp = ts;
+ deleted = del;
+
+ mutationCount = mc;
+ }
+
+ SubKey(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, int32_t mc){
+
+ int cfLen = env->GetArrayLength(cf);
+ int cqLen = env->GetArrayLength(cq);
+ int cvLen = env->GetArrayLength(cv);
+
+ colQualifierOffset = cfLen;
+ colVisibilityOffset = colQualifierOffset + cqLen;
+ totalLen = colVisibilityOffset + cvLen;
+
+ if(lba == NULL)
+ keyData = new uint8_t[totalLen];
+ else
+ keyData = (uint8_t *)lba->allocate(totalLen);
+
+
+ env->GetByteArrayRegion(cf, 0, cfLen, (jbyte *)keyData);
+ env->GetByteArrayRegion(cq, 0, cqLen, (jbyte *)(keyData+colQualifierOffset));
+ env->GetByteArrayRegion(cv, 0, cvLen, (jbyte *)(keyData+colVisibilityOffset));
+
+ timestamp = ts;
+ deleted = del;
+
+ mutationCount = mc;
+ }
+
+
+ bool operator<(const SubKey &key) const{
+
+ int result = compare(keyData, colQualifierOffset, key.keyData, key.colQualifierOffset);
+ if(result != 0) return result < 0;
+
+ result = compare(keyData + colQualifierOffset, colVisibilityOffset - colQualifierOffset, key.keyData + key.colQualifierOffset, key.colVisibilityOffset - key.colQualifierOffset);
+ if(result != 0) return result < 0;
+
+ result = compare(keyData + colVisibilityOffset, totalLen - colVisibilityOffset, key.keyData + key.colVisibilityOffset, key.totalLen - key.colVisibilityOffset);
+ if(result != 0) return result < 0;
+
+ if(timestamp < key.timestamp){
+ return false;
+ }else if(timestamp > key.timestamp){
+ return true;
+ }
+
+ if(deleted != key.deleted)
+ return deleted && !key.deleted;
+
+ return mutationCount > key.mutationCount;
+ }
+
+ void clear(){
+ //delete(keyData);
+ }
+
+ void clear(LinkedBlockAllocator *lba){
+ lba->deleteLast(keyData);
+ }
+
+ int64_t bytesUsed() const{
+ return totalLen + 9;
+ }
+
+ bool isDeleted() const{
+ return deleted;
+ }
+
+ int32_t getCFLen() const{
+ return colQualifierOffset;
+ }
+
+ int32_t getCQLen() const {
+ return colVisibilityOffset - colQualifierOffset;
+ }
+
+ int32_t getCVLen() const {
+ return totalLen - colVisibilityOffset;
+ }
+
+ const Field getCF() const{
+ return Field(keyData, getCFLen());
+ }
+
+ const Field getCQ() const{
+ return Field(keyData + colQualifierOffset, getCQLen());
+ }
+
+ const Field getCV() const{
+ return Field(keyData + colVisibilityOffset, getCVLen());
+ }
+
+ string toString() const{
+ return getCF().toString()+":"+getCQ().toString()+":"+getCV().toString();
+ }
+
+ int64_t getTimestamp() const{
+ return timestamp;
+ }
+
+ int32_t getMC() const{
+ return mutationCount;
+ }
+
+};
+
+struct LocalSubKey : public SubKey {
+
+ LocalSubKey(JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del):SubKey(NULL, env, cf, cq, cv, ts, del, INT_MAX){}
+
+ ~LocalSubKey(){
+ delete(keyData);
+ }
+};
+
+#endif
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/main/c++/nativeMap/org_apache_accumulo_tserver_NativeMap.cc
----------------------------------------------------------------------
diff --git a/server/native/src/main/c++/nativeMap/org_apache_accumulo_tserver_NativeMap.cc b/server/native/src/main/c++/nativeMap/org_apache_accumulo_tserver_NativeMap.cc
new file mode 100644
index 0000000..4c17fb3
--- /dev/null
+++ b/server/native/src/main/c++/nativeMap/org_apache_accumulo_tserver_NativeMap.cc
@@ -0,0 +1,130 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#include "org_apache_accumulo_tserver_NativeMap.h"
+#include "SubKey.h"
+#include "Field.h"
+#include "NativeMap.h"
+#include <map>
+#include <vector>
+#include <jni.h>
+#include <iostream>
+
+#ifdef _POSIX_MEMLOCK
+#include <sys/mman.h>
+#endif
+
+using namespace std;
+
+JNIEXPORT jlong JNICALL Java_org_apache_accumulo_tserver_NativeMap_createNM(JNIEnv *env, jclass cls) {
+ return (jlong)(new NativeMap(1<<17, 1<<11));
+}
+
+JNIEXPORT jint JNICALL Java_org_apache_accumulo_tserver_NativeMap_sizeNM(JNIEnv *env, jclass cls, jlong nm) {
+ return ((NativeMap *)nm)->count;
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_accumulo_tserver_NativeMap_memoryUsedNM(JNIEnv *env, jclass cls, jlong nm) {
+ return ((NativeMap *)nm)->getMemoryUsed();
+}
+
+JNIEXPORT void JNICALL Java_org_apache_accumulo_tserver_NativeMap_singleUpdate(JNIEnv *env, jclass cls, jlong nm, jbyteArray r, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jbyteArray val, jint mutationCount) {
+ jlong uid = Java_org_apache_accumulo_tserver_NativeMap_startUpdate(env, cls, nm, r);
+ Java_org_apache_accumulo_tserver_NativeMap_update(env, cls, nm, uid, cf, cq, cv, ts, del, val, mutationCount);
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_accumulo_tserver_NativeMap_startUpdate(JNIEnv *env, jclass cls, jlong nm, jbyteArray r) {
+ NativeMap *nativeMap = (NativeMap *)nm;
+ ColumnMap *cm = nativeMap->startUpdate(env, r);
+ return (jlong)cm;
+}
+
+JNIEXPORT void JNICALL Java_org_apache_accumulo_tserver_NativeMap_update(JNIEnv *env, jclass cls, jlong nm, jlong uid, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jbyteArray val, jint mutationCount) {
+ NativeMap *nativeMap = (NativeMap *)nm;
+ nativeMap->update((ColumnMap *)uid, env, cf, cq, cv, ts, del, val, mutationCount);
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_accumulo_tserver_NativeMap_deleteNM(JNIEnv *env, jclass cls, jlong nm) {
+ NativeMap *nativeMap = (NativeMap *)nm;
+ delete(nativeMap);
+ return 0;
+}
+
+
+JNIEXPORT jlong JNICALL Java_org_apache_accumulo_tserver_NativeMap_createNMI__J_3I(JNIEnv *env, jclass cls, jlong nm, jintArray lens) {
+ NativeMap *nativeMap = (NativeMap *)nm;
+ int32_t ia[7];
+ Iterator *iter = nativeMap->iterator(ia);
+ if (iter->atEnd()) {
+ delete(iter);
+ return 0;
+ }
+ env->SetIntArrayRegion(lens, 0, 7, ia);
+ return (jlong)iter;
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_accumulo_tserver_NativeMap_createNMI__J_3B_3B_3B_3BJZ_3I(JNIEnv *env, jclass cls, jlong nm, jbyteArray r, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jintArray lens) {
+
+ NativeMap *nativeMap = (NativeMap *)nm;
+ LocalField row(env, r);
+ LocalSubKey sk(env, cf, cq, cv, ts, del);
+
+ int32_t ia[7];
+ Iterator *iter = nativeMap->iterator(row, sk, ia);
+
+ if(iter->atEnd()) {
+ delete(iter);
+ return 0;
+ }
+
+ env->SetIntArrayRegion(lens, 0, 7, ia);
+ return (jlong)iter;
+}
+
+JNIEXPORT jboolean JNICALL Java_org_apache_accumulo_tserver_NativeMap_nmiNext(JNIEnv *env, jclass cls, jlong ip, jintArray lens) {
+ Iterator &iter = *((Iterator *)ip);
+
+ int32_t ia[7];
+ iter.advance(ia);
+ if(iter.atEnd()) {
+ return false;
+ }
+
+ env->SetIntArrayRegion(lens, 0, 7, ia);
+ return true;
+}
+
+JNIEXPORT void JNICALL Java_org_apache_accumulo_tserver_NativeMap_nmiGetData(JNIEnv *env, jclass cls, jlong ip, jbyteArray r, jbyteArray cf, jbyteArray cq, jbyteArray cv, jbyteArray val) {
+ Iterator &iter = *((Iterator *)ip);
+ if(r != NULL) {
+ iter.rowIter->first.fillIn(env, r);
+ }
+
+ iter.colIter->first.getCF().fillIn(env, cf);
+ iter.colIter->first.getCQ().fillIn(env, cq);
+ iter.colIter->first.getCV().fillIn(env, cv);
+ iter.colIter->second.fillIn(env, val);
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_accumulo_tserver_NativeMap_nmiGetTS(JNIEnv *env, jclass cls, jlong ip) {
+ Iterator &iter = *((Iterator *)ip);
+ return iter.colIter->first.getTimestamp();
+}
+
+JNIEXPORT void JNICALL Java_org_apache_accumulo_tserver_NativeMap_deleteNMI(JNIEnv *env, jclass cls, jlong ip) {
+ delete((Iterator *)ip);
+}
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/test/c++/nativeMap/test.cc
----------------------------------------------------------------------
diff --git a/server/native/src/test/c++/nativeMap/test.cc b/server/native/src/test/c++/nativeMap/test.cc
new file mode 100644
index 0000000..786e0c1
--- /dev/null
+++ b/server/native/src/test/c++/nativeMap/test.cc
@@ -0,0 +1,134 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#include "NativeMap.h"
+#include "util.h"
+#include "Key.h"
+#include <stdio.h>
+
+void runTest(int numRows, int numCf, int numCq, int rowLen, int cfLen, int cqLen, int cvLen, int valLen, bool testOld){
+
+ map<std::string, std::string> tm;
+ tm.insert(pair<string, string>("a","b"));
+ tm.insert(pair<string, string>("a","c")).first->second="c";
+
+ cout << tm["a"] << std::endl;
+
+ size_t initialMemUsage = getMemUsage();
+
+ NativeMap nm(1<<17, 1<<11);
+
+ cout << " size pair<Key, Field> : " << sizeof(std::pair<Key, Field>) << endl;
+ cout << " size pair<Field, CMT> : " << sizeof(std::pair<Field, CMT>) << endl;
+ cout << " size pair<SubKey, Field> : " << sizeof(std::pair<SubKey, Field>) << endl;
+
+ map<Key, Field> oldMap;
+
+ int entries = 0;
+
+
+ char rowFmt[16];
+ char cfFmt[16];
+ char cqFmt[16];
+ char cvFmt[16];
+ char valFmt[16];
+
+ sprintf(rowFmt, "%s0%dd", "%", rowLen);
+ sprintf(cfFmt, "%s0%dd", "%", cfLen);
+ sprintf(cqFmt, "%s0%dd", "%", cqLen);
+ sprintf(cvFmt, "%s0%dd", "%", cvLen);
+ sprintf(valFmt, "%s0%dd", "%", valLen);
+
+ for(int r = 0; r < numRows; r++){
+ char row[rowLen+1];
+ snprintf(row, rowLen+1, rowFmt, r);
+
+ CMT *cmt = NULL;
+ if(!testOld){
+ cmt = nm.startUpdate(row);
+ }
+
+ for(int cf = 0; cf < numCf; cf++){
+ char colf[cfLen+1];
+ snprintf(colf, cfLen+1, cfFmt, cf);
+
+ for(int cq = 0; cq < numCq; cq++){
+ char colq[cqLen+1];
+ snprintf(colq, cqLen+1, cqFmt, cq);
+
+ char colv[cvLen+1];
+ snprintf(colv, cvLen+1, cvFmt, 1);
+
+ char val[valLen+1];
+ snprintf(val, valLen+1, valFmt, entries);
+
+ if(!testOld){
+ nm.update(cmt, colf, colq, colv, 5, false, val, valLen);
+ }else{
+ oldMap.insert(pair<Key, Field>(Key(row, colf, colq, colv, 5, false), Field(val)));
+ }
+
+
+ entries++;
+
+ }
+ }
+ }
+
+ int expectedRowBytes = rowLen * numRows;
+ int expectedEntryBytes = entries * (cfLen + cqLen + cvLen + 9 + valLen);
+
+ cout << "row bytes : " << expectedRowBytes << endl;
+ cout << "entry bytes : " << expectedEntryBytes << endl;
+ cout << "count : " << nm.count << " " << entries << endl;
+ size_t memUsage = getMemUsage();
+ cout << " mem delta " << memUsage - initialMemUsage << endl;
+ cout << " mem usage " << memUsage << endl;
+ cout << " simple overhead " << ( (getMemUsage() - initialMemUsage) - (entries * (rowLen + cfLen + cqLen + cvLen + 9 + valLen) ) ) / ((double)entries) << endl;
+
+ if(testOld){
+ map<Key, Field>::iterator iter = oldMap.begin();
+ while(iter != oldMap.end()){
+ delete(iter->first.keyData);
+ delete(iter->second.field);
+ iter++;
+ }
+ }
+
+}
+
+int main(int argc, char **argv){
+ int numRows,numCf, numCq, rowLen, cfLen, cqLen, cvLen, valLen, testOld;
+
+ if(argc != 10){
+ cout << "Usage : " << argv[0] << " <numRows> <numCf> <numCq> <rowLen> <cfLen> <cqLen> <cvLen> <valLen> <testOld>" << endl;
+ return -1;
+ }
+
+ sscanf(argv[1], "%d", &numRows);
+ sscanf(argv[2], "%d", &numCf);
+ sscanf(argv[3], "%d", &numCq);
+ sscanf(argv[4], "%d", &rowLen);
+ sscanf(argv[5], "%d", &cfLen);
+ sscanf(argv[6], "%d", &cqLen);
+ sscanf(argv[7], "%d", &cvLen);
+ sscanf(argv[8], "%d", &valLen);
+ sscanf(argv[9], "%d", &testOld);
+
+ for(int i = 0; i < 3; i++){
+ runTest(numRows,numCf, numCq, rowLen, cfLen, cqLen, cvLen, valLen, testOld);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/test/c++/nativeMap/util.cc
----------------------------------------------------------------------
diff --git a/server/native/src/test/c++/nativeMap/util.cc b/server/native/src/test/c++/nativeMap/util.cc
new file mode 100644
index 0000000..b9393ee
--- /dev/null
+++ b/server/native/src/test/c++/nativeMap/util.cc
@@ -0,0 +1,32 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "util.h"
+
+size_t getMemUsage(){
+ pid_t pid = getpid();
+ char cmd[1000];
+ sprintf(cmd, "cat /proc/%d/status | grep VmData | awk '{print $2}'", pid);
+ FILE *f = popen(cmd, "r");
+ int dataSize;
+ fscanf(f, "%d\n", &dataSize);
+ pclose(f);
+ return (size_t)dataSize * (size_t)1024;
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/native/src/test/c++/nativeMap/util.h
----------------------------------------------------------------------
diff --git a/server/native/src/test/c++/nativeMap/util.h b/server/native/src/test/c++/nativeMap/util.h
new file mode 100644
index 0000000..65d5721
--- /dev/null
+++ b/server/native/src/test/c++/nativeMap/util.h
@@ -0,0 +1,19 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+size_t getMemUsage();
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/server/pom.xml b/server/server/pom.xml
deleted file mode 100644
index 30b1f97..0000000
--- a/server/server/pom.xml
+++ /dev/null
@@ -1,54 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-project</artifactId>
- <version>1.6.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <artifactId>accumulo-server</artifactId>
- <name>Server</name>
- <profiles>
- <profile>
- <id>native</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>generate-native-in-memory-map</id>
- <goals>
- <goal>exec</goal>
- </goals>
- <phase>compile</phase>
- <configuration>
- <executable>make</executable>
- <workingDirectory>${basedir}/src/main/c++</workingDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/server/src/main/c++/Makefile
----------------------------------------------------------------------
diff --git a/server/server/src/main/c++/Makefile b/server/server/src/main/c++/Makefile
deleted file mode 100644
index 2ad4514..0000000
--- a/server/server/src/main/c++/Makefile
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# ya... I know, this is a bad recursive makefile...
-all: nm
-
-nm:
- cd nativeMap ; $(MAKE)
-
-clean:
- cd nativeMap ; $(MAKE) $@
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/server/src/main/c++/nativeMap/BlockAllocator.h
----------------------------------------------------------------------
diff --git a/server/server/src/main/c++/nativeMap/BlockAllocator.h b/server/server/src/main/c++/nativeMap/BlockAllocator.h
deleted file mode 100644
index 3126217..0000000
--- a/server/server/src/main/c++/nativeMap/BlockAllocator.h
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-
-#ifndef _BLOCK_ALLOCATOR_H_
-#define _BLOCK_ALLOCATOR_H_ 1
-
-#include <new>
-#include <bits/functexcept.h>
-#include <iostream>
-#include <string>
-#include <vector>
-#include <stdlib.h>
-#include <stddef.h>
-
-struct Block {
- unsigned char *data;
- unsigned char *currentPos;
- unsigned char *end;
- unsigned char *prevPos;
-
- Block(uint32_t size){
- data = new unsigned char[size];
- end = data + size;
- currentPos = data;
- prevPos = NULL;
- }
-
- ~Block(){
- }
-
- void *allocate(size_t amount){
- unsigned char *nextPos = currentPos + amount;
-
- if(nextPos > end){
- return NULL;
- }
-
- prevPos = currentPos;
- currentPos = nextPos;
- return prevPos;
- }
-
- size_t rollback(void *p){
- if(p == prevPos){
- size_t diff = currentPos - prevPos;
- currentPos = prevPos;
- return diff;
- }else{
- std::cerr << "Tried to delete something that was not previous allocation " << p << " " << prevPos << std::endl;
- exit(-1);
- }
-
- return 0;
- }
-
- size_t getMemoryFree() {
- return end - currentPos;
- }
-};
-
-struct BigBlock {
- unsigned char *ptr;
- size_t length;
-
- BigBlock(unsigned char *p, size_t len):ptr(p),length(len){}
-};
-
-struct LinkedBlockAllocator {
-
- std::vector<Block> blocks;
- std::vector<BigBlock> bigBlocks;
- int blockSize;
- int bigBlockSize;
- int64_t memused;
- void *lastAlloc;
-
- LinkedBlockAllocator(int blockSize, int bigBlockSize){
- this->blockSize = blockSize;
- this->bigBlockSize = bigBlockSize;
- lastAlloc = NULL;
- memused = 0;
- }
-
- void *allocate(size_t amount){
-
- if(amount > (size_t)bigBlockSize){
- unsigned char *p = new unsigned char[amount];
- bigBlocks.push_back(BigBlock(p, amount));
- memused += sizeof(BigBlock) + amount;
- return p;
- }else{
- if(blocks.size() == 0){
- //do lazy allocation of memory, do not allocate a block until it is used
- blocks.push_back(Block(blockSize));
- memused += sizeof(Block) + blockSize;
- }
-
- lastAlloc = blocks.back().allocate(amount);
- if(lastAlloc == NULL){
- blocks.push_back(Block(blockSize));
- lastAlloc = blocks.back().allocate(amount);
- memused += sizeof(Block) + blockSize;
- }
-
- return lastAlloc;
- }
- }
-
- void deleteLast(void *p){
- if(p != NULL){
- if(p == lastAlloc){
- blocks.back().rollback(p);
- lastAlloc = NULL;
- return;
- }else if(bigBlocks.back().ptr == p){
- memused -= (sizeof(BigBlock) + bigBlocks.back().length);
- bigBlocks.pop_back();
- delete((unsigned char *)p);
- return;
- }
- }
-
- std::cerr << "Tried to delete something that was not last allocation " << p << " " << lastAlloc << std::endl;
- exit(-1);
- }
-
- size_t getMemoryUsed(){
- if(blocks.size() == 0)
- return memused;
- else
- return memused - blocks.back().getMemoryFree();
- }
-
- ~LinkedBlockAllocator(){
- //std::cout << "Deleting " << blocks.size() << " blocks, memused : " << memused << std::endl;
- std::vector<Block>::iterator iter = blocks.begin();
- while(iter != blocks.end()){
- delete(iter->data);
- iter++;
- }
-
- std::vector<BigBlock>::iterator iter2 = bigBlocks.begin();
- while(iter2 != bigBlocks.end()){
- delete(iter2->ptr);
- iter2++;
- }
- }
-
-};
-
-
- /**
- * @brief An allocator that uses global new, as per [20.4].
- *
- * This is precisely the allocator defined in the C++ Standard.
- * - all allocation calls operator new
- * - all deallocation calls operator delete
- */
- template<typename _Tp>
- class BlockAllocator
- {
- public:
- typedef size_t size_type;
- typedef ptrdiff_t difference_type;
- typedef _Tp* pointer;
- typedef const _Tp* const_pointer;
- typedef _Tp& reference;
- typedef const _Tp& const_reference;
- typedef _Tp value_type;
-
- LinkedBlockAllocator *lba;
-
- template<typename _Tp1>
- struct rebind
- { typedef BlockAllocator<_Tp1> other; };
-
- BlockAllocator() throw() {
- lba = NULL;
- }
-
- BlockAllocator(LinkedBlockAllocator *lba) throw() {
- this->lba = lba;
- }
-
- BlockAllocator(const BlockAllocator& ba) throw() {
- lba = ba.lba;
- }
-
- template<typename _Tp1>
- BlockAllocator(const BlockAllocator<_Tp1>& ba) throw() {
- lba = ba.lba;
- }
-
- ~BlockAllocator() throw() { }
-
- pointer
- address(reference __x) const { return &__x; }
-
- const_pointer
- address(const_reference __x) const { return &__x; }
-
- // NB: __n is permitted to be 0. The C++ standard says nothing
- // about what the return value is when __n == 0.
- pointer
- allocate(size_type __n, const void* = 0)
- {
- if (__builtin_expect(__n > this->max_size(), false))
- std::__throw_bad_alloc();
-
-
- //void *p = ::operator new(__n * sizeof(_Tp));
- void *p = lba->allocate(__n * sizeof(_Tp));
-
- //std::cout << "Allocating "<< name <<" " << __n * sizeof(_Tp) << " " << ((unsigned long long)p) % 4 << " " << ((unsigned long long)p) % 8 << std::endl;
-
- return static_cast<_Tp*>(p);
- }
-
- // __p is not permitted to be a null pointer.
- void
- deallocate(pointer __p, size_type)
- {
- //::operator delete(__p);
- }
-
- size_type
- max_size() const throw()
- { return size_t(-1) / sizeof(_Tp); }
-
- // _GLIBCXX_RESOLVE_LIB_DEFECTS
- // 402. wrong new expression in [some_] allocator::construct
- void
- construct(pointer __p, const _Tp& __val)
- { ::new(__p) _Tp(__val); }
-
- void
- destroy(pointer __p) { __p->~_Tp(); }
-
-
- };
-
-
- template<typename _Tp>
- inline bool
- operator==(const BlockAllocator<_Tp>& ba1, const BlockAllocator<_Tp>& ba2)
- { return true; }
-
- template<typename _Tp>
- inline bool
- operator!=(const BlockAllocator<_Tp>& ba1, const BlockAllocator<_Tp>& ba2)
- { return false; }
-
-#endif
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/server/src/main/c++/nativeMap/Field.h
----------------------------------------------------------------------
diff --git a/server/server/src/main/c++/nativeMap/Field.h b/server/server/src/main/c++/nativeMap/Field.h
deleted file mode 100644
index 7c3884e..0000000
--- a/server/server/src/main/c++/nativeMap/Field.h
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-#include <stdint.h>
-#include <string.h>
-#include <jni.h>
-#include <string>
-#include <string.h>
-#include <iostream>
-#include <stdlib.h>
-#include "BlockAllocator.h"
-
-using namespace std;
-
-#ifndef __FIELD__
-#define __FIELD__
-
-struct Field {
- uint8_t *field;
- int32_t len;
-
- int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{
- int result = memcmp(d1, d2, len1 < len2 ? len1 : len2);
-
- if(result != 0)
- return result;
- if(len1 == len2)
- return 0;
- if(len1 < len2)
- return -1;
-
- return 1;
- }
-
- Field(){}
-
- Field(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray f, int l){
- len = l;
- field=(uint8_t *)lba->allocate(len);
- env->GetByteArrayRegion(f, 0, len, (jbyte *)field);
- }
-
- Field(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray f){
- len = env->GetArrayLength(f);
- field=(uint8_t *)lba->allocate(len);
- env->GetByteArrayRegion(f, 0, len, (jbyte *)field);
- }
-
- Field(uint8_t *f, int32_t l):field(f),len(l){
- }
-
- Field(const char *cstr){
- //constructor for testing C++
- len = strlen(cstr);
- field=new uint8_t[len];
- memcpy(field, cstr, len);
- }
-
- Field(LinkedBlockAllocator *lba, const char *cstr){
- //constructor for testing C++
- len = strlen(cstr);
- field=(uint8_t *)lba->allocate(len);
- memcpy(field, cstr, len);
- }
-
- void set(const char *d, int l){
- if(l < 0 || l > len){
- cerr << "Tried to set field with value that is too long " << l << " " << len << endl;
- }
- memcpy(field, d, l);
- len = l;
- }
-
- void set(JNIEnv *env, jbyteArray f, int l){
- if(l < 0 || l > len){
- cerr << "Tried to set field with value that is too long " << l << " " << len << endl;
- }
- len = l;
- env->GetByteArrayRegion(f, 0, len, (jbyte *)field);
- }
-
- int compare(const Field &of) const{
- return compare(field, len, of.field, of.len);
- }
-
- bool operator<(const Field &of) const{
- return compare(of) < 0;
- }
-
- int32_t length() const {
- return len;
- }
-
- void fillIn(JNIEnv *env, jbyteArray d) const {
- //TODO ensure lengths match up
- env->SetByteArrayRegion(d, 0, len, (jbyte *)field);
- }
-
- jbyteArray createJByteArray(JNIEnv *env) const{
- jbyteArray valData = env->NewByteArray(len);
- env->SetByteArrayRegion(valData, 0, len, (jbyte *)field);
- return valData;
- }
-
- string toString() const{
- return string((char *)field, len);
- }
-
- void clear(){
- //delete(field);
- }
-
- void clear(LinkedBlockAllocator *lba){
- lba->deleteLast(field);
- }
-};
-
-struct LocalField : public Field {
- LocalField(JNIEnv *env, jbyteArray f){
- len = env->GetArrayLength(f);
- field= new uint8_t[len];
- env->GetByteArrayRegion(f, 0, len, (jbyte *)field);
- }
-
- ~LocalField(){
- delete(field);
- }
-};
-#endif
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/server/src/main/c++/nativeMap/Key.h
----------------------------------------------------------------------
diff --git a/server/server/src/main/c++/nativeMap/Key.h b/server/server/src/main/c++/nativeMap/Key.h
deleted file mode 100644
index f902426..0000000
--- a/server/server/src/main/c++/nativeMap/Key.h
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-#include <algorithm>
-#include <stdint.h>
-#include <string.h>
-#include <string>
-#include <jni.h>
-
-using namespace std;
-
-class Key {
-
-public:
- int32_t colFamilyOffset;
- int32_t colQualifierOffset;
- int32_t colVisibilityOffset;
- int32_t totalLen;
-
- uint8_t *keyData;
-
- int64_t timestamp;
- bool deleted;
-
- int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{
- int result = memcmp(d1, d2, len1 < len2 ? len1 : len2);
-
- if(result != 0)
- return result;
- if(len1 == len2)
- return 0;
- if(len1 < len2)
- return -1;
-
- return 1;
- }
-
- /**
- * Constructor for testing purposes
- */
-
- Key(){}
-
- Key(const string &r, const string &cf, const string &cq, const string &cv, long ts, bool del){
-
- colFamilyOffset = r.length();
- colQualifierOffset = colFamilyOffset + cf.length();
- colVisibilityOffset = colQualifierOffset + cq.length();
- totalLen = colVisibilityOffset + cv.length();
-
- keyData = new uint8_t[totalLen];
-
- copy(r.begin(), r.end(), keyData);
- copy(cf.begin(), cf.end(), keyData+colFamilyOffset);
- copy(cq.begin(), cq.end(), keyData+colQualifierOffset);
- copy(cv.begin(), cv.end(), keyData+colVisibilityOffset);
-
- timestamp = ts;
- deleted = del;
- }
-
- /**
- * Constructor used for taking data from Java Key
- */
-
- Key(JNIEnv *env, jbyteArray kd, jint cfo, jint cqo, jint cvo, jint tl, jlong ts, jboolean del){
-
- colFamilyOffset = cfo;
- colQualifierOffset = cqo;
- colVisibilityOffset = cvo;
- totalLen = tl;
- timestamp = ts;
- deleted = del == JNI_TRUE ? true : false;
-
- keyData = new uint8_t[totalLen];
- env->GetByteArrayRegion(kd, 0, totalLen, (jbyte *)keyData);
- }
-
- bool operator<(const Key &key) const{
- int result = compare(keyData, colFamilyOffset, key.keyData, key.colFamilyOffset);
- if(result != 0) return result < 0;
-
- result = compare(keyData + colFamilyOffset, colQualifierOffset - colFamilyOffset, key.keyData + key.colFamilyOffset, key.colQualifierOffset - key.colFamilyOffset);
- if(result != 0) return result < 0;
-
- result = compare(keyData + colQualifierOffset, colVisibilityOffset - colQualifierOffset, key.keyData + key.colQualifierOffset, key.colVisibilityOffset - key.colQualifierOffset);
- if(result != 0) return result < 0;
-
- result = compare(keyData + colVisibilityOffset, totalLen - colVisibilityOffset, key.keyData + key.colVisibilityOffset, key.totalLen - key.colVisibilityOffset);
- if(result != 0) return result < 0;
-
- if(timestamp < key.timestamp){
- return false;
- }else if(timestamp > key.timestamp){
- return true;
- }
-
- return deleted && !key.deleted;
- }
-
-};
-
-
-class LocalKey : public Key {
-
-public:
-
- JNIEnv *envPtr;
- jbyteArray kd;
-
- LocalKey(JNIEnv *env, jbyteArray kd, jint cfo, jint cqo, jint cvo, jint tl, jlong ts, jboolean del){
- envPtr = env;
-
- colFamilyOffset = cfo;
- colQualifierOffset = cqo;
- colVisibilityOffset = cvo;
- totalLen = tl;
- timestamp = ts;
- deleted = del == JNI_TRUE ? true : false;
-
- this->kd = kd;
- keyData = (uint8_t *)env->GetByteArrayElements((jbyteArray)kd, NULL);
- }
-
- ~LocalKey(){
- envPtr->ReleaseByteArrayElements(kd, (jbyte *)keyData, JNI_ABORT);
- }
-
-};
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/server/src/main/c++/nativeMap/Makefile
----------------------------------------------------------------------
diff --git a/server/server/src/main/c++/nativeMap/Makefile b/server/server/src/main/c++/nativeMap/Makefile
deleted file mode 100644
index 26a8ad9..0000000
--- a/server/server/src/main/c++/nativeMap/Makefile
+++ /dev/null
@@ -1,75 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-SRCS=$(filter-out test.cc,$(wildcard *.cc))
-HDRS=$(wildcard *.h) org_apache_accumulo_server_tabletserver_NativeMap.h
-CXX=g++
-
-ifeq ($(shell uname),Linux)
- LIBS_32 := libNativeMap-Linux-i386-32.so
- LIBS_64 := libNativeMap-Linux-amd64-64.so
-
- ifneq ($(DARCH),)
- ifeq ($(DARCH),64)
- LIBS := $(LIBS_64)
- endif
- ifeq ($(DARCH),32)
- LIBS := $(LIBS_32)
- endif
- ifeq ($(DARCH),both)
- LIBS := $(LIBS_64) $(LIBS_32)
- endif
- ifeq ($(LIBS),)
- LIBS := $(LIBS_64) $(LIBS_32)
- endif
- else
- DARCH := $(shell getconf LONG_BIT)
- LIBS := $(LIBS_$(DARCH))
- endif
-
- CXXFLAGS=-g -fPIC -shared -O2 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -I$(JAVA_HOME)/include/linux -I$(JAVA_HOME)/include
-endif
-
-ifeq ($(shell uname),Darwin)
- LIBS:= libNativeMap-Mac_OS_X-x86_64-64.jnilib
- CXXFLAGS=-m64 -dynamiclib -O3 -I/System/Library/Frameworks/JavaVM.framework/Headers
-endif
-
-INSTALL_DIR=../../../../../lib/native/map
-INSTALLED_LIBS=$(patsubst %,$(INSTALL_DIR)/%,$(LIBS))
-
-all : $(INSTALLED_LIBS)
-
-clean:
- rm -f $(INSTALLED_LIBS) $(LIBS)
-
-org_apache_accumulo_server_tabletserver_NativeMap.h :
- javah -classpath ../../../../../lib/accumulo-server.jar org.apache.accumulo.server.tabletserver.NativeMap
-
-$(INSTALLED_LIBS) : $(INSTALL_DIR) $(LIBS)
- cp $(LIBS) $(INSTALL_DIR)
-
-$(INSTALL_DIR):
- mkdir -p $@
-
-libNativeMap-Linux-amd64-64.so : $(SRCS) $(HDRS)
- $(CXX) $(CXXFLAGS) -m64 -o $@ $(SRCS)
-
-libNativeMap-Linux-i386-32.so : $(SRCS) $(HDRS)
- $(CXX) $(CXXFLAGS) -m32 -o $@ $(SRCS)
-
-libNativeMap-Mac_OS_X-x86_64-64.jnilib : $(SRCS) $(HDRS)
- $(CXX) $(CXXFLAGS) -m64 -o $@ $(SRCS)
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/server/src/main/c++/nativeMap/NativeMap.h
----------------------------------------------------------------------
diff --git a/server/server/src/main/c++/nativeMap/NativeMap.h b/server/server/src/main/c++/nativeMap/NativeMap.h
deleted file mode 100644
index 85060d6..0000000
--- a/server/server/src/main/c++/nativeMap/NativeMap.h
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-#ifndef __NATIVE_MAP_A12__
-#define __NATIVE_MAP_A12__
-
-#include "SubKey.h"
-#include "Field.h"
-#include "BlockAllocator.h"
-#include <map>
-#include <vector>
-#include <iostream>
-
-using namespace std;
-
-typedef map<SubKey, Field, std::less<SubKey>, BlockAllocator<std::pair<const SubKey, Field> > > ColumnMap;
-typedef map<Field, ColumnMap, std::less<Field>, BlockAllocator<std::pair<const Field, ColumnMap> > > RowMap;
-
-
-struct NativeMapData {
- LinkedBlockAllocator *lba;
- RowMap rowmap;
- int count;
-
- NativeMapData(int blockSize, int bigBlockSize):lba(new LinkedBlockAllocator(blockSize, bigBlockSize)),
- rowmap(RowMap(std::less<Field>(), BlockAllocator<std::pair<Field, ColumnMap> >(lba))){
- this->lba = lba;
- }
-
- ~NativeMapData(){
- rowmap.clear(); //if row map is not cleared here, it will be deconstructed after lba is deleted
- delete(lba);
- }
-};
-
-
-struct Iterator {
- NativeMapData &nativeMap;
- RowMap::iterator rowIter;
- ColumnMap::iterator colIter;
-
- Iterator(NativeMapData &nm, int32_t *ia):nativeMap(nm){
- rowIter = nativeMap.rowmap.begin();
- if(rowIter == nativeMap.rowmap.end()){
- return;
- }
-
- colIter = rowIter->second.begin();
-
- skipAndFillIn(ia, true);
- }
-
-
- Iterator(NativeMapData &nm, Field &row, SubKey &sk, int32_t *ia):nativeMap(nm){
- rowIter = nativeMap.rowmap.lower_bound(row);
- if(rowIter == nativeMap.rowmap.end()){
- return;
- }
-
- //TODO use equals instead of compare
- if(rowIter->first.compare(row) == 0){
- colIter = rowIter->second.lower_bound(sk);
- }else{
- colIter = rowIter->second.begin();
- }
-
- skipAndFillIn(ia, true);
- }
-
- bool skipAndFillIn(int32_t *ia, bool firstCall)
- {
- bool rowChanged = false;
-
- while(colIter == rowIter->second.end()){
- rowIter++;
- rowChanged = true;
- if(rowIter == nativeMap.rowmap.end()){
- return false;
- }
- colIter = rowIter->second.begin();
- }
-
- ia[0] = (firstCall || rowChanged) ? rowIter->first.length() : -1;
- ia[1] = colIter->first.getCFLen();
- ia[2] = colIter->first.getCQLen();
- ia[3] = colIter->first.getCVLen();
- ia[4] = colIter->first.isDeleted() ? 1 : 0;
- ia[5] = colIter->second.length();
- ia[6] = colIter->first.getMC();
-
- return true;
- }
-
- bool atEnd(){
- return rowIter == nativeMap.rowmap.end();
- }
-
- void advance(int32_t *ia){
- colIter++;
- skipAndFillIn(ia, false);
- }
-};
-
-struct NativeMap : public NativeMapData {
-
- NativeMap(int blockSize, int bigBlockSize):NativeMapData(blockSize, bigBlockSize){
- count = 0;
- }
-
- ~NativeMap(){
-
- }
-
-
- ColumnMap *startUpdate(JNIEnv * env, jbyteArray r){
- Field row(lba, env, r);
- return startUpdate(row);
- }
-
- ColumnMap *startUpdate(const char *r){
-
- Field row(lba, r);
- return startUpdate(row);
- }
-
- ColumnMap *startUpdate(Field &row){
- //cout << "Starting update "<<row.toString()<<endl;
-
- pair<RowMap::iterator, bool> insertResult = rowmap.insert(pair<Field, ColumnMap>(row, ColumnMap(std::less<SubKey>(), BlockAllocator<std::pair<SubKey, Field> >(lba))));
-
- if(!insertResult.second)
- row.clear(lba);
-
- return &(insertResult.first->second);
-
- }
-
- void update(ColumnMap *cm, JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jbyteArray val, jint mutationCount){
-
- SubKey sk(lba, env, cf, cq, cv, ts, del, mutationCount);
- //cout << "Updating " << sk.toString() << " " << sk.getTimestamp() << " " << sk.isDeleted() << endl;
- //do not bother allocating value if not needed
- Field value(NULL, 0);
-
- pair<ColumnMap::iterator, bool> insertResult = cm->insert(pair<SubKey, Field>(sk, value));
- if(insertResult.second){
- insertResult.first->second = Field(lba, env, val);
- count++;
- }else{
- sk.clear(lba);
- int valLen = env->GetArrayLength(val);
- if(valLen <= insertResult.first->second.length()){
- insertResult.first->second.set(env, val, valLen);
- }else{
- insertResult.first->second.clear();
- insertResult.first->second = Field(lba, env, val, valLen);
- }
- }
- }
-
- void update(ColumnMap *cm, const char *cf, const char *cq, const char *cv, long ts, bool del, const char *val, int valLen, int mutationCount){
-
- SubKey sk(lba, cf, cq, cv, ts, del, mutationCount);
- //do not bother allocating value if not needed
- Field value(NULL, 0);
-
- pair<ColumnMap::iterator, bool> insertResult = cm->insert(pair<SubKey, Field>(sk, value));
- if(insertResult.second){
- insertResult.first->second = Field(lba, val);
- count++;
- }else{
- sk.clear(lba);
- if(insertResult.first->second.length() <= valLen){
- insertResult.first->second.set(val, valLen);
- }else{
- insertResult.first->second.clear();
- insertResult.first->second = Field(lba, val);
- }
- }
- }
-
- Iterator *iterator(int32_t *ia){
- return new Iterator(*this, ia);
- }
-
- Iterator *iterator(Field &row, SubKey &sk, int32_t *ia){
- return new Iterator(*this, row, sk, ia);
- }
-
- int64_t getMemoryUsed(){
- return lba->getMemoryUsed();
- }
-};
-
-#endif
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1b449d3/server/server/src/main/c++/nativeMap/SubKey.h
----------------------------------------------------------------------
diff --git a/server/server/src/main/c++/nativeMap/SubKey.h b/server/server/src/main/c++/nativeMap/SubKey.h
deleted file mode 100644
index 59e7a6d..0000000
--- a/server/server/src/main/c++/nativeMap/SubKey.h
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-#include <jni.h>
-#include <limits.h>
-#include "Field.h"
-#include "BlockAllocator.h"
-
-using namespace std;
-
-#ifndef __SUB_KEY__
-#define __SUB_KEY__
-
-class SubKey {
-
-public:
-
- int32_t colQualifierOffset;
- int32_t colVisibilityOffset;
- int32_t totalLen;
-
- uint8_t *keyData;
-
- int64_t timestamp;
-
- int32_t mutationCount;
-
- bool deleted;
-
- int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{
- int result = memcmp(d1, d2, len1 < len2 ? len1 : len2);
-
- if(result != 0)
- return result;
- if(len1 == len2)
- return 0;
- if(len1 < len2)
- return -1;
-
- return 1;
- }
-
- /**
- * Constructor for testing purposes
- */
-
- SubKey(LinkedBlockAllocator *lba, const string &cf, const string &cq, const string &cv, int64_t ts, bool del, int32_t mc){
-
- colQualifierOffset = cf.length();
- colVisibilityOffset = colQualifierOffset + cq.length();
- totalLen = colVisibilityOffset + cv.length();
-
- keyData = (uint8_t *)lba->allocate(totalLen);
-
- copy(cf.begin(), cf.end(), keyData);
- copy(cq.begin(), cq.end(), keyData+colQualifierOffset);
- copy(cv.begin(), cv.end(), keyData+colVisibilityOffset);
-
- timestamp = ts;
- deleted = del;
-
- mutationCount = mc;
- }
-
- SubKey(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, int32_t mc){
-
- int cfLen = env->GetArrayLength(cf);
- int cqLen = env->GetArrayLength(cq);
- int cvLen = env->GetArrayLength(cv);
-
- colQualifierOffset = cfLen;
- colVisibilityOffset = colQualifierOffset + cqLen;
- totalLen = colVisibilityOffset + cvLen;
-
- if(lba == NULL)
- keyData = new uint8_t[totalLen];
- else
- keyData = (uint8_t *)lba->allocate(totalLen);
-
-
- env->GetByteArrayRegion(cf, 0, cfLen, (jbyte *)keyData);
- env->GetByteArrayRegion(cq, 0, cqLen, (jbyte *)(keyData+colQualifierOffset));
- env->GetByteArrayRegion(cv, 0, cvLen, (jbyte *)(keyData+colVisibilityOffset));
-
- timestamp = ts;
- deleted = del;
-
- mutationCount = mc;
- }
-
-
- bool operator<(const SubKey &key) const{
-
- int result = compare(keyData, colQualifierOffset, key.keyData, key.colQualifierOffset);
- if(result != 0) return result < 0;
-
- result = compare(keyData + colQualifierOffset, colVisibilityOffset - colQualifierOffset, key.keyData + key.colQualifierOffset, key.colVisibilityOffset - key.colQualifierOffset);
- if(result != 0) return result < 0;
-
- result = compare(keyData + colVisibilityOffset, totalLen - colVisibilityOffset, key.keyData + key.colVisibilityOffset, key.totalLen - key.colVisibilityOffset);
- if(result != 0) return result < 0;
-
- if(timestamp < key.timestamp){
- return false;
- }else if(timestamp > key.timestamp){
- return true;
- }
-
- if(deleted != key.deleted)
- return deleted && !key.deleted;
-
- return mutationCount > key.mutationCount;
- }
-
- void clear(){
- //delete(keyData);
- }
-
- void clear(LinkedBlockAllocator *lba){
- lba->deleteLast(keyData);
- }
-
-
- int64_t bytesUsed() const{
- return totalLen + 9;
- }
-
- bool isDeleted() const{
- return deleted;
- }
-
- int32_t getCFLen() const{
- return colQualifierOffset;
- }
-
- int32_t getCQLen() const {
- return colVisibilityOffset - colQualifierOffset;
- }
-
- int32_t getCVLen() const {
- return totalLen - colVisibilityOffset;
- }
-
- const Field getCF() const{
- return Field(keyData, getCFLen());
- }
-
- const Field getCQ() const{
- return Field(keyData + colQualifierOffset, getCQLen());
- }
-
- const Field getCV() const{
- return Field(keyData + colVisibilityOffset, getCVLen());
- }
-
- string toString() const{
- return getCF().toString()+":"+getCQ().toString()+":"+getCV().toString();
- }
-
- int64_t getTimestamp() const{
- return timestamp;
- }
-
- int32_t getMC() const{
- return mutationCount;
- }
-
-};
-
-struct LocalSubKey : public SubKey {
-
- LocalSubKey(JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del):SubKey(NULL, env, cf, cq, cv, ts, del, INT_MAX){}
-
- ~LocalSubKey(){
- delete(keyData);
- }
-};
-
-#endif
-