You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@ignite.apache.org by GitBox <gi...@apache.org> on 2021/11/29 08:57:42 UTC
[GitHub] [ignite] SammyVimes commented on a change in pull request #9569: IGNITE-15922 Implement NUMA-aware allocator
SammyVimes commented on a change in pull request #9569:
URL: https://github.com/apache/ignite/pull/9569#discussion_r758123632
##########
File path: modules/numa-allocator/src/main/cpp/src/numa/numa_alloc.cpp
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <numa.h>
+#include <numa/numa_alloc.h>
+
+namespace numa {
+ class BitSet::BitSetImpl {
+ public:
+ BitSetImpl() : size_(numa_max_node() + 1) {
+ mask_ = numa_bitmask_alloc(size_);
+ }
+
+ BitSetImpl(const BitSetImpl &other) : size_(other.size_) {
+ mask_ = numa_bitmask_alloc(size_);
+ copy_bitmask_to_bitmask(other.mask_, mask_);
+ }
+
+ void SetBit(size_t idx, bool val) {
+ if (idx < size_) {
+ if (val)
+ numa_bitmask_setbit(mask_, idx);
+ else
+ numa_bitmask_clearbit(mask_, idx);
+ }
+ }
+
+ void SetAll(bool val) {
+ if (val)
+ numa_bitmask_setall(mask_);
+ else
+ numa_bitmask_clearall(mask_);
+ }
+
+ bool GetBit(size_t idx) const {
+ if (idx < size_)
+ return numa_bitmask_isbitset(mask_, idx);
+ else
+ return false;
+ }
+
+ bool Equals(const BitSetImpl &other) const {
+ return numa_bitmask_equal(mask_, other.mask_);
+ }
+
+ size_t Size() const {
+ return size_;
+ }
+
+ bitmask *Raw() {
+ return mask_;
+ }
+
+ ~BitSetImpl() {
+ numa_bitmask_free(mask_);
+ }
+
+ private:
+ bitmask *mask_;
+ size_t size_;
+ };
+
+ BitSet::BitSet() {
+ p_impl_ = new BitSetImpl();
+ }
+
+ BitSet::BitSet(const BitSet &other) {
+ p_impl_ = new BitSetImpl(*other.p_impl_);
+ }
+
+ BitSet &BitSet::operator=(const BitSet &other) {
+ if (this != &other) {
+ BitSet tmp(other);
+ std::swap(this->p_impl_, tmp.p_impl_);
+ }
+ return *this;
+ }
+
+ bool BitSet::Get(size_t pos) const {
+ return p_impl_->GetBit(pos);
+ }
+
+ void BitSet::Set(size_t pos, bool value) {
+ p_impl_->SetBit(pos, value);
+ }
+
+ void BitSet::Set() {
+ p_impl_->SetAll(true);
+ }
+
+ void BitSet::Reset(size_t pos) {
+ p_impl_->SetBit(pos, false);
+ }
+
+ void BitSet::Reset() {
+ p_impl_->SetAll(false);
+ }
+
+ size_t BitSet::Size() const {
+ return p_impl_->Size();
+ }
+
+ bool BitSet::operator==(const BitSet &other) {
+ return this->p_impl_->Equals(*other.p_impl_);
+ }
+
+ bool BitSet::operator!=(const BitSet &other) {
+ return !(*this == other);
+ }
+
+ BitSet::~BitSet() {
+ delete p_impl_;
+ }
+
+ std::ostream &operator<<(std::ostream &os, const BitSet &set) {
+ os << '{';
+ for (size_t i = 0; i < set.Size(); ++i) {
+ os << set[i];
+ if (i < set.Size() - 1) {
+ os << ", ";
+ }
+ }
+ os << '}';
+ return os;
+ }
+
+ union region_size {
+ size_t size;
+ max_align_t a;
+ };
+
+ int NumaNodesCount() {
+ return numa_max_node() + 1;
+ }
+
+ template<typename Func, typename ...Args>
+ void *NumaAllocHelper(Func f, size_t size, Args ...args) {
+ auto ptr = static_cast<region_size *>(f(size + sizeof(region_size), args...));
Review comment:
I think there should be a comment about the layout of a memory that is allocated.
##########
File path: modules/numa-allocator/pom.xml
##########
@@ -0,0 +1,178 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ POM file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-parent</artifactId>
+ <version>1</version>
+ <relativePath>../../parent</relativePath>
+ </parent>
+
+ <artifactId>ignite-numa-allocator</artifactId>
+ <version>${revision}</version>
+ <url>http://ignite.apache.org</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>ignite-tools</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.thoughtworks.xstream</groupId>
+ <artifactId>xstream</artifactId>
+ <version>1.4.8</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
Review comment:
Maybe https://github.com/cmake-maven-project/cmake-maven-project would be more suitable here. Not sure though
##########
File path: modules/numa-allocator/src/main/java/org/apache/ignite/internal/mem/NumaAllocUtil.java
##########
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.mem;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+
+/** */
+public class NumaAllocUtil {
+ /** */
+ private static final String extension = ".so";
+
+ /** */
+ private static final String libraryName = "libnuma_alloc";
+
+ /** */
+ private NumaAllocUtil() {
+ // No-op.
+ }
+
+ /** */
+ private static String getLibName() {
+ return Paths.get("/org/apache/ignite/internal/mem", getOSName(), getOSArch(), libraryName + extension)
+ .toString();
+ }
+
+ /** */
+ private static String getOSArch() {
+ return System.getProperty("os.arch", "");
+ }
+
+ /** */
+ private static String getOSName() {
+ if (System.getProperty("os.name", "").contains("Linux"))
+ return "linux";
+ else
+ throw new UnsupportedOperationException("Operating System is not supported");
+ }
+
+ static {
+ String libName = getLibName();
+ File nativeLib = null;
+
+ try (InputStream in = NumaAllocUtil.class.getResourceAsStream(libName)) {
+ if (in == null) {
+ throw new ExceptionInInitializerError("Failed to load native numa_alloc library, "
+ + libName + " not found");
+ }
+
+ nativeLib = File.createTempFile(libraryName, extension);
+
+ try (FileOutputStream out = new FileOutputStream(nativeLib)) {
Review comment:
Can be replaced with
```
nativeLib = Files.createTempFile(libraryName, extension);
Files.copy(in, nativeLib);
```
##########
File path: modules/numa-allocator/src/main/cpp/include/numa/numa_alloc.h
##########
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _NUMA_ALLOC_H
+#define _NUMA_ALLOC_H
+
+#include <ostream>
+
+namespace numa {
+ class BitSet {
+ public:
+ BitSet();
+
+ BitSet(const BitSet &other);
+
+ BitSet &operator=(const BitSet &other);
+
+ ~BitSet();
+
+ class Reference {
Review comment:
I don't quite get where this class is used. I see that it captures the position in a bitset and an object of the `Reference` is returned by the `[]` op. Is it used in the internals of the libnuma?
##########
File path: modules/numa-allocator/src/main/cpp/src/numa/numa_alloc.cpp
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <numa.h>
+#include <numa/numa_alloc.h>
+
+namespace numa {
+ class BitSet::BitSetImpl {
+ public:
+ BitSetImpl() : size_(numa_max_node() + 1) {
+ mask_ = numa_bitmask_alloc(size_);
+ }
+
+ BitSetImpl(const BitSetImpl &other) : size_(other.size_) {
+ mask_ = numa_bitmask_alloc(size_);
+ copy_bitmask_to_bitmask(other.mask_, mask_);
+ }
+
+ void SetBit(size_t idx, bool val) {
+ if (idx < size_) {
+ if (val)
+ numa_bitmask_setbit(mask_, idx);
+ else
+ numa_bitmask_clearbit(mask_, idx);
+ }
+ }
+
+ void SetAll(bool val) {
+ if (val)
+ numa_bitmask_setall(mask_);
+ else
+ numa_bitmask_clearall(mask_);
+ }
+
+ bool GetBit(size_t idx) const {
+ if (idx < size_)
+ return numa_bitmask_isbitset(mask_, idx);
+ else
+ return false;
+ }
+
+ bool Equals(const BitSetImpl &other) const {
+ return numa_bitmask_equal(mask_, other.mask_);
+ }
+
+ size_t Size() const {
+ return size_;
+ }
+
+ bitmask *Raw() {
+ return mask_;
+ }
+
+ ~BitSetImpl() {
+ numa_bitmask_free(mask_);
+ }
+
+ private:
+ bitmask *mask_;
+ size_t size_;
+ };
+
+ BitSet::BitSet() {
+ p_impl_ = new BitSetImpl();
+ }
+
+ BitSet::BitSet(const BitSet &other) {
+ p_impl_ = new BitSetImpl(*other.p_impl_);
+ }
+
+ BitSet &BitSet::operator=(const BitSet &other) {
+ if (this != &other) {
+ BitSet tmp(other);
+ std::swap(this->p_impl_, tmp.p_impl_);
+ }
+ return *this;
+ }
+
+ bool BitSet::Get(size_t pos) const {
+ return p_impl_->GetBit(pos);
+ }
+
+ void BitSet::Set(size_t pos, bool value) {
+ p_impl_->SetBit(pos, value);
+ }
+
+ void BitSet::Set() {
+ p_impl_->SetAll(true);
+ }
+
+ void BitSet::Reset(size_t pos) {
+ p_impl_->SetBit(pos, false);
+ }
+
+ void BitSet::Reset() {
+ p_impl_->SetAll(false);
+ }
+
+ size_t BitSet::Size() const {
+ return p_impl_->Size();
+ }
+
+ bool BitSet::operator==(const BitSet &other) {
+ return this->p_impl_->Equals(*other.p_impl_);
+ }
+
+ bool BitSet::operator!=(const BitSet &other) {
+ return !(*this == other);
+ }
+
+ BitSet::~BitSet() {
+ delete p_impl_;
+ }
+
+ std::ostream &operator<<(std::ostream &os, const BitSet &set) {
+ os << '{';
+ for (size_t i = 0; i < set.Size(); ++i) {
+ os << set[i];
+ if (i < set.Size() - 1) {
+ os << ", ";
+ }
+ }
+ os << '}';
+ return os;
+ }
+
+ union region_size {
+ size_t size;
+ max_align_t a;
+ };
+
+ int NumaNodesCount() {
+ return numa_max_node() + 1;
+ }
+
+ template<typename Func, typename ...Args>
+ void *NumaAllocHelper(Func f, size_t size, Args ...args) {
+ auto ptr = static_cast<region_size *>(f(size + sizeof(region_size), args...));
+ if (ptr) {
+ ptr->size = size;
+ ptr++;
+ }
+ return ptr;
+ }
+
+ void *Alloc(size_t size) {
+ return NumaAllocHelper(numa_alloc, size);
+ }
+
+ void *Alloc(size_t size, int node) {
+ return NumaAllocHelper(numa_alloc_onnode, size, node);
+ }
+
+ void *AllocLocal(size_t size) {
+ return NumaAllocHelper(numa_alloc_local, size);
+ }
+
+ void *AllocInterleaved(size_t size) {
+ return NumaAllocHelper(numa_alloc_interleaved, size);
+ }
+
+ void *AllocInterleaved(size_t size, const BitSet &node_set) {
+ return NumaAllocHelper(numa_alloc_interleaved_subset, size, node_set.p_impl_->Raw());
+ }
+
+ size_t Size(void *buf) {
+ if (buf) {
+ auto *ptr = static_cast<region_size *>(buf);
+ ptr--;
+ return ptr->size;
+ }
+ return 0;
+ }
+
+ void Free(void *buf) {
+ if (buf) {
+ auto *ptr = static_cast<region_size *>(buf);
+ ptr--;
Review comment:
I'd recommend writing a comment here and in `Size() ` about this cast and decrement, because ptr is not actually pointing to region_size before the decrement.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org