Project
Loading...
Searching...
No Matches
ShmManager.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12/*
13
14 * ShmManager.cxx
15 *
16 * Created on: Jun 17, 2018
17 * Author: swenzel
18 */
19
21#include <fairlogger/Logger.h>
22#include <sys/shm.h>
23#include <sys/ipc.h>
24#include <algorithm>
25
26#include <boost/interprocess/managed_external_buffer.hpp>
27#include <boost/interprocess/allocators/allocator.hpp>
28#include <sstream>
29#include <cstdlib>
30#include <cassert>
31
32using namespace boost::interprocess;
33
34namespace o2
35{
36namespace utils
37{
38
39// the shared mem id under which this is accessible
40const char* SHMIDNAME = "ALICEO2_SIMSHM_SHMID";
41// a common virtual address under which this should be mapped
42const char* SHMADDRNAME = "ALICEO2_SIMSHM_COMMONADDR";
43
44ShmManager::ShmManager() = default;
45
46void* ShmManager::tryAttach(bool& success)
47{
48 success = false;
49 if (auto id = getenv(SHMIDNAME)) {
50 mShmID = atoi(getenv(SHMIDNAME));
51 } else {
52 mShmID = -1;
53 return nullptr;
54 }
55 LOG(info) << "FOUND ID TO ATTACH " << mShmID;
56
57 // if the segment was not created in the first place ...
58 if (mShmID == -1) {
59 return nullptr;
60 }
61
62 void* addr_wanted;
63 if (auto addrstr = getenv(SHMADDRNAME)) {
64 addr_wanted = (void*)atoll(addrstr);
65 } else {
66 mShmID = -1;
67 return nullptr;
68 }
69 LOG(info) << "TRYING ADDRESS " << addr_wanted;
70 auto addr = shmat(mShmID, addr_wanted, 0);
71 if (addr != (void*)-1) {
72 // this means success
73 assert(addr == addr_wanted);
74 success = true;
75 mSegPtr = addr;
76 mSegInfoPtr = static_cast<o2::utils::ShmMetaInfo*>(mSegPtr);
77 return addr;
78 }
79
80 // trying without constraints
81 addr = shmat(mShmID, nullptr, 0);
82 if (addr == (void*)(-1)) {
83 LOG(fatal) << "SHOULD NOT HAPPEN";
84 }
85 // signal failure
86 auto info = static_cast<o2::utils::ShmMetaInfo*>(addr);
87 info->failures.fetch_add(1);
88 shmdt(addr);
89 mShmID = -1;
90 mSegInfoPtr = nullptr;
91 mSegPtr = nullptr;
92 // otherwise for now this means that we could not map the segment
93 // at the right place ...
94 return nullptr;
95}
96
98{
99 if (mSegInfoPtr) {
100 LOG(info) << "ATTACHED WORKERS " << mSegInfoPtr->counter;
101 LOG(info) << "CONNECTION FAILURES " << mSegInfoPtr->failures;
102 } else {
103 LOG(info) << "no segment info to print";
104 }
105}
106
108{
109 mIsMaster = true;
110 // first of all take a look if we really start from a clean state
111 if (mShmID != -1) {
112 LOG(warn) << "A SEGMENT IS ALREADY INITIALIZED";
113 return false;
114 }
115 // make sure no one else has created the shm pool
116 // TODO: this can be relaxed / generalized
117 if (getenv(SHMIDNAME) || getenv(SHMADDRNAME)) {
118 LOG(warn) << "A SEGMET IS ALREADY PRESENT ON THE SYSTEM";
119 return false;
120 }
121
122#ifdef USESHM
123 LOG(info) << "CREATING SIM SHARED MEM SEGMENT FOR " << nsegments << " WORKERS";
124 // LOG(info) << "SIZEOF ShmMetaInfo " << sizeof(ShmMetaInfo);
125 const auto totalsize = sizeof(ShmMetaInfo) + SHMPOOLSIZE * nsegments;
126 if ((mShmID = shmget(IPC_PRIVATE, totalsize, IPC_CREAT | 0600)) == -1) {
127 perror("shmget: shmget failed");
128 } else {
129 // We are attaching once to determine a common virtual address under which everyone else should attach.
130 // In this case (if it succeeds) we can also share objects with shm pointers
131 auto addr = shmat(mShmID, nullptr, 0);
132 mSegPtr = addr;
133 LOG(debug) << "COMMON ADDRESS " << addr << " AS NUMBER " << (unsigned long long)addr;
134
135 // initialize the meta information (counter)
137 std::memcpy(addr, &info, sizeof(info));
138 mSegInfoPtr = static_cast<o2::utils::ShmMetaInfo*>(mSegPtr);
139 mSegInfoPtr->allocedbytes = totalsize;
140
141 // communicating information about this segment via
142 // an environment variable
143 // TODO: consider using named posix shared memory segments to avoid this
144 setenv(SHMIDNAME, std::to_string(mShmID).c_str(), 1);
145 setenv(SHMADDRNAME, std::to_string((unsigned long long)(addr)).c_str(), 1);
146
147 // mark the segment for removal right away: Linux still lets the workers attach by id,
148 // and the kernel frees it when the last process detaches, even after a crash
149 shmctl(mShmID, IPC_RMID, nullptr);
150 return true;
151 }
152 LOG(info) << "SHARED MEM INITIALIZED AT ID " << mShmID;
153 if (mShmID == -1) {
154 LOG(warn) << "COULD NOT CREATE SHARED MEMORY ... FALLING BACK TO SIMPLE MODE";
155 setenv(SHMIDNAME, std::to_string(mShmID).c_str(), 1);
156 setenv(SHMADDRNAME, std::to_string(0).c_str(), 1);
157 }
158#endif
159 return false;
160}
161
163{
164 LOG(info) << "OCCUPYING A SEGMENT IN A SHARED REGION";
165 // get information about the global shared mem in which to occupy a region
166 if (!(getenv(SHMIDNAME) && getenv(SHMADDRNAME))) {
167 LOG(warn) << "NO INFORMATION ABOUT SHARED SEGMENT FOUND";
168 return false;
169 }
170
171 if (mShmID != -1) {
172 LOG(warn) << "REGION ALREADY OCCUPIED OR CREATED";
173 return false;
174 }
175
176 bool b;
177 tryAttach(b);
178 return b;
179}
180
181void ShmManager::occupySegment(bool wholeRegion)
182{
183 LOG(info) << "OCCUPYING A SEGMENT IN A SHARED REGION";
184 // get information about the global shared mem in which to occupy a region
185 if (!(getenv(SHMIDNAME) && getenv(SHMADDRNAME))) {
186 LOG(warn) << "NO INFORMATION ABOUT SHARED SEGMENT FOUND";
187 return;
188 }
189
190 if (mShmID != -1) {
191 LOG(warn) << "REGION ALREADY OCCUPIED OR CREATED";
192 return;
193 }
194
195 bool b;
196 auto addr = tryAttach(b);
197 if (b) {
198 // read meta information in the segment to determine the id of this segment
199 auto info = static_cast<o2::utils::ShmMetaInfo*>(addr);
200 const int npools = wholeRegion ? (info->allocedbytes - sizeof(ShmMetaInfo)) / SHMPOOLSIZE : 1;
201 const int segmentcounter = info->counter.fetch_add(npools);
202 LOG(info) << "SEGMENTCOUNT " << segmentcounter;
203
204 const auto offset_in_bytes = sizeof(ShmMetaInfo) + segmentcounter * SHMPOOLSIZE;
205 mBufferPtr = (void*)(((char*)addr) + offset_in_bytes);
206 mPoolSize = npools * SHMPOOLSIZE;
207
208 assert((unsigned long long)((char*)mBufferPtr - (char*)addr) + mPoolSize <= info->allocedbytes);
209
210 boostmanagedbuffer = new boost::interprocess::wmanaged_external_buffer(create_only, mBufferPtr, mPoolSize);
211 boostallocator = new boost::interprocess::allocator<char, wmanaged_external_buffer::segment_manager>(
212 boostmanagedbuffer->get_segment_manager());
213
214 LOG(info) << "SHARED MEM OCCUPIED AT ID " << mShmID << " AND SEGMENT COUNTER " << segmentcounter;
215 } else {
216 LOG(info) << "ATTACH NOT SUCCESSFUL";
217 }
218}
219
220ShmManager::~ShmManager()
221{
222 release();
223}
224
225// This implements a very malloc/free mechanism ...
226// ... but we are using available boost functionality
228{
229 void* addr = nullptr;
230 std::lock_guard<std::mutex> lock(mAllocMutex);
231 try {
232 addr = (void*)boostallocator->allocate(size).get();
233 } catch (const std::exception& e) {
234 LOG(fatal) << "THROW IN BOOST SHM ALLOCATION";
235 };
236 return addr;
237}
238
239void ShmManager::freememblock(void* ptr, size_t s)
240{
241 std::lock_guard<std::mutex> lock(mAllocMutex);
242 boostallocator->deallocate((char*)ptr, s);
243}
244
246{
247#ifdef USESHM
248 if (mIsMaster) {
249 LOG(debug) << "REMOVING SHARED MEM SEGMENT ID " << mShmID;
250 if (mShmID != -1) {
251 shmctl(mShmID, IPC_RMID, nullptr);
252 mShmID = -1;
253 }
254 }
255#endif
256}
257
258} // end namespace utils
259} // end namespace o2
std::ostringstream debug
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char e
TBranch * ptr
void * getmemblock(size_t size)
void printSegInfo() const
void freememblock(void *, std::size_t=1)
bool createGlobalSegment(int nsubsegments=1)
void occupySegment(bool wholeRegion=false)
GLsizeiptr size
Definition glcorearb.h:659
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
constexpr size_t SHMPOOLSIZE
Definition ShmManager.h:43
const char * SHMADDRNAME
const char * SHMIDNAME
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
std::string to_string(gsl::span< T, Size > span)
Definition common.h:52
Common utility functions.
std::atomic< int > failures
Definition ShmManager.h:50
std::atomic< int > counter
Definition ShmManager.h:48
unsigned long long allocedbytes
Definition ShmManager.h:47
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"