1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "C2AllocatorIon"
19 #include <utils/Log.h>
20
21 #include <list>
22
23 #include <ion/ion.h>
24 #include <sys/mman.h>
25 #include <unistd.h> // getpagesize, size_t, close, dup
26
27 #include <C2AllocatorIon.h>
28 #include <C2Buffer.h>
29 #include <C2Debug.h>
30 #include <C2ErrnoUtils.h>
31
32 namespace android {
33
34 namespace {
35 constexpr size_t USAGE_LRU_CACHE_SIZE = 1024;
36 }
37
38 /* size_t <=> int(lo), int(hi) conversions */
size2intLo(size_t s)39 constexpr inline int size2intLo(size_t s) {
40 return int(s & 0xFFFFFFFF);
41 }
42
size2intHi(size_t s)43 constexpr inline int size2intHi(size_t s) {
44 // cast to uint64_t as size_t may be 32 bits wide
45 return int((uint64_t(s) >> 32) & 0xFFFFFFFF);
46 }
47
ints2size(int intLo,int intHi)48 constexpr inline size_t ints2size(int intLo, int intHi) {
49 // convert in 2 stages to 64 bits as intHi may be negative
50 return size_t(unsigned(intLo)) | size_t(uint64_t(unsigned(intHi)) << 32);
51 }
52
53 /* ========================================= ION HANDLE ======================================== */
54 /**
55 * ION handle
56 *
57 * There can be only a sole ion client per process, this is captured in the ion fd that is passed
58 * to the constructor, but this should be managed by the ion buffer allocator/mapper.
59 *
60 * ion uses ion_user_handle_t for buffers. We don't store this in the native handle as
61 * it requires an ion_free to decref. Instead, we share the buffer to get an fd that also holds
62 * a refcount.
63 *
64 * This handle will not capture mapped fd-s as updating that would require a global mutex.
65 */
66
67 struct C2HandleIon : public C2Handle {
68 // ion handle owns ionFd(!) and bufferFd
C2HandleIonandroid::C2HandleIon69 C2HandleIon(int bufferFd, size_t size)
70 : C2Handle(cHeader),
71 mFds{ bufferFd },
72 mInts{ int(size & 0xFFFFFFFF), int((uint64_t(size) >> 32) & 0xFFFFFFFF), kMagic } { }
73
74 static bool isValid(const C2Handle * const o);
75
bufferFdandroid::C2HandleIon76 int bufferFd() const { return mFds.mBuffer; }
sizeandroid::C2HandleIon77 size_t size() const {
78 return size_t(unsigned(mInts.mSizeLo))
79 | size_t(uint64_t(unsigned(mInts.mSizeHi)) << 32);
80 }
81
82 protected:
83 struct {
84 int mBuffer; // shared ion buffer
85 } mFds;
86 struct {
87 int mSizeLo; // low 32-bits of size
88 int mSizeHi; // high 32-bits of size
89 int mMagic;
90 } mInts;
91
92 private:
93 typedef C2HandleIon _type;
94 enum {
95 kMagic = '\xc2io\x00',
96 numFds = sizeof(mFds) / sizeof(int),
97 numInts = sizeof(mInts) / sizeof(int),
98 version = sizeof(C2Handle)
99 };
100 //constexpr static C2Handle cHeader = { version, numFds, numInts, {} };
101 const static C2Handle cHeader;
102 };
103
104 const C2Handle C2HandleIon::cHeader = {
105 C2HandleIon::version,
106 C2HandleIon::numFds,
107 C2HandleIon::numInts,
108 {}
109 };
110
111 // static
isValid(const C2Handle * const o)112 bool C2HandleIon::isValid(const C2Handle * const o) {
113 if (!o || memcmp(o, &cHeader, sizeof(cHeader))) {
114 return false;
115 }
116 const C2HandleIon *other = static_cast<const C2HandleIon*>(o);
117 return other->mInts.mMagic == kMagic;
118 }
119
120 // TODO: is the dup of an ion fd identical to ion_share?
121
122 /* ======================================= ION ALLOCATION ====================================== */
123 class C2AllocationIon : public C2LinearAllocation {
124 public:
125 /* Interface methods */
126 virtual c2_status_t map(
127 size_t offset, size_t size, C2MemoryUsage usage, C2Fence *fence,
128 void **addr /* nonnull */) override;
129 virtual c2_status_t unmap(void *addr, size_t size, C2Fence *fenceFd) override;
130 virtual ~C2AllocationIon() override;
131 virtual const C2Handle *handle() const override;
132 virtual id_t getAllocatorId() const override;
133 virtual bool equals(const std::shared_ptr<C2LinearAllocation> &other) const override;
134
135 // internal methods
136 C2AllocationIon(int ionFd, size_t size, size_t align, unsigned heapMask, unsigned flags, C2Allocator::id_t id);
137 C2AllocationIon(int ionFd, size_t size, int shareFd, C2Allocator::id_t id);
138
139 c2_status_t status() const;
140
141 protected:
142 class Impl;
143 Impl *mImpl;
144
145 // TODO: we could make this encapsulate shared_ptr and copiable
146 C2_DO_NOT_COPY(C2AllocationIon);
147 };
148
149 class C2AllocationIon::Impl {
150 private:
151 /**
152 * Constructs an ion allocation.
153 *
154 * \note We always create an ion allocation, even if the allocation or import fails
155 * so that we can capture the error.
156 *
157 * \param ionFd ion client (ownership transferred to created object)
158 * \param capacity size of allocation
159 * \param bufferFd buffer handle (ownership transferred to created object). Must be
160 * invalid if err is not 0.
161 * \param buffer ion buffer user handle (ownership transferred to created object). Must be
162 * invalid if err is not 0.
163 * \param err errno during buffer allocation or import
164 */
Impl(int ionFd,size_t capacity,int bufferFd,ion_user_handle_t buffer,C2Allocator::id_t id,int err)165 Impl(int ionFd, size_t capacity, int bufferFd, ion_user_handle_t buffer, C2Allocator::id_t id, int err)
166 : mIonFd(ionFd),
167 mHandle(bufferFd, capacity),
168 mBuffer(buffer),
169 mId(id),
170 mInit(c2_map_errno<ENOMEM, EACCES, EINVAL>(err)),
171 mMapFd(-1) {
172 if (mInit != C2_OK) {
173 // close ionFd now on error
174 if (mIonFd >= 0) {
175 close(mIonFd);
176 mIonFd = -1;
177 }
178 // C2_CHECK(bufferFd < 0);
179 // C2_CHECK(buffer < 0);
180 }
181 }
182
183 public:
184 /**
185 * Constructs an ion allocation by importing a shared buffer fd.
186 *
187 * \param ionFd ion client (ownership transferred to created object)
188 * \param capacity size of allocation
189 * \param bufferFd buffer handle (ownership transferred to created object)
190 *
191 * \return created ion allocation (implementation) which may be invalid if the
192 * import failed.
193 */
Import(int ionFd,size_t capacity,int bufferFd,C2Allocator::id_t id)194 static Impl *Import(int ionFd, size_t capacity, int bufferFd, C2Allocator::id_t id) {
195 ion_user_handle_t buffer = -1;
196 int ret = ion_import(ionFd, bufferFd, &buffer);
197 return new Impl(ionFd, capacity, bufferFd, buffer, id, ret);
198 }
199
200 /**
201 * Constructs an ion allocation by allocating an ion buffer.
202 *
203 * \param ionFd ion client (ownership transferred to created object)
204 * \param size size of allocation
205 * \param align desired alignment of allocation
206 * \param heapMask mask of heaps considered
207 * \param flags ion allocation flags
208 *
209 * \return created ion allocation (implementation) which may be invalid if the
210 * allocation failed.
211 */
Alloc(int ionFd,size_t size,size_t align,unsigned heapMask,unsigned flags,C2Allocator::id_t id)212 static Impl *Alloc(int ionFd, size_t size, size_t align, unsigned heapMask, unsigned flags, C2Allocator::id_t id) {
213 int bufferFd = -1;
214 ion_user_handle_t buffer = -1;
215 size_t alignedSize = align == 0 ? size : (size + align - 1) & ~(align - 1);
216 int ret = ion_alloc(ionFd, alignedSize, align, heapMask, flags, &buffer);
217 ALOGV("ion_alloc(ionFd = %d, size = %zu, align = %zu, prot = %d, flags = %d) "
218 "returned (%d) ; buffer = %d",
219 ionFd, alignedSize, align, heapMask, flags, ret, buffer);
220 if (ret == 0) {
221 // get buffer fd for native handle constructor
222 ret = ion_share(ionFd, buffer, &bufferFd);
223 if (ret != 0) {
224 ion_free(ionFd, buffer);
225 buffer = -1;
226 }
227 }
228 return new Impl(ionFd, alignedSize, bufferFd, buffer, id, ret);
229 }
230
map(size_t offset,size_t size,C2MemoryUsage usage,C2Fence * fence,void ** addr)231 c2_status_t map(size_t offset, size_t size, C2MemoryUsage usage, C2Fence *fence, void **addr) {
232 (void)fence; // TODO: wait for fence
233 *addr = nullptr;
234 if (!mMappings.empty()) {
235 ALOGV("multiple map");
236 // TODO: technically we should return DUPLICATE here, but our block views don't
237 // actually unmap, so we end up remapping an ion buffer multiple times.
238 //
239 // return C2_DUPLICATE;
240 }
241 if (size == 0) {
242 return C2_BAD_VALUE;
243 }
244
245 int prot = PROT_NONE;
246 int flags = MAP_SHARED;
247 if (usage.expected & C2MemoryUsage::CPU_READ) {
248 prot |= PROT_READ;
249 }
250 if (usage.expected & C2MemoryUsage::CPU_WRITE) {
251 prot |= PROT_WRITE;
252 }
253
254 size_t alignmentBytes = offset % PAGE_SIZE;
255 size_t mapOffset = offset - alignmentBytes;
256 size_t mapSize = size + alignmentBytes;
257 Mapping map = { nullptr, alignmentBytes, mapSize };
258
259 c2_status_t err = C2_OK;
260 if (mMapFd == -1) {
261 int ret = ion_map(mIonFd, mBuffer, mapSize, prot,
262 flags, mapOffset, (unsigned char**)&map.addr, &mMapFd);
263 ALOGV("ion_map(ionFd = %d, handle = %d, size = %zu, prot = %d, flags = %d, "
264 "offset = %zu) returned (%d)",
265 mIonFd, mBuffer, mapSize, prot, flags, mapOffset, ret);
266 if (ret) {
267 mMapFd = -1;
268 map.addr = *addr = nullptr;
269 err = c2_map_errno<EINVAL>(-ret);
270 } else {
271 *addr = (uint8_t *)map.addr + alignmentBytes;
272 }
273 } else {
274 map.addr = mmap(nullptr, mapSize, prot, flags, mMapFd, mapOffset);
275 ALOGV("mmap(size = %zu, prot = %d, flags = %d, mapFd = %d, offset = %zu) "
276 "returned (%d)",
277 mapSize, prot, flags, mMapFd, mapOffset, errno);
278 if (map.addr == MAP_FAILED) {
279 map.addr = *addr = nullptr;
280 err = c2_map_errno<EINVAL>(errno);
281 } else {
282 *addr = (uint8_t *)map.addr + alignmentBytes;
283 }
284 }
285 if (map.addr) {
286 mMappings.push_back(map);
287 }
288 return err;
289 }
290
unmap(void * addr,size_t size,C2Fence * fence)291 c2_status_t unmap(void *addr, size_t size, C2Fence *fence) {
292 if (mMapFd < 0 || mMappings.empty()) {
293 ALOGD("tried to unmap unmapped buffer");
294 return C2_NOT_FOUND;
295 }
296 for (auto it = mMappings.begin(); it != mMappings.end(); ++it) {
297 if (addr != (uint8_t *)it->addr + it->alignmentBytes ||
298 size + it->alignmentBytes != it->size) {
299 continue;
300 }
301 int err = munmap(it->addr, it->size);
302 if (err != 0) {
303 ALOGD("munmap failed");
304 return c2_map_errno<EINVAL>(errno);
305 }
306 if (fence) {
307 *fence = C2Fence(); // not using fences
308 }
309 (void)mMappings.erase(it);
310 ALOGV("successfully unmapped: %d", mBuffer);
311 return C2_OK;
312 }
313 ALOGD("unmap failed to find specified map");
314 return C2_BAD_VALUE;
315 }
316
~Impl()317 ~Impl() {
318 if (!mMappings.empty()) {
319 ALOGD("Dangling mappings!");
320 for (const Mapping &map : mMappings) {
321 (void)munmap(map.addr, map.size);
322 }
323 }
324 if (mMapFd >= 0) {
325 close(mMapFd);
326 mMapFd = -1;
327 }
328 if (mInit == C2_OK) {
329 (void)ion_free(mIonFd, mBuffer);
330 native_handle_close(&mHandle);
331 }
332 if (mIonFd >= 0) {
333 close(mIonFd);
334 }
335 }
336
status() const337 c2_status_t status() const {
338 return mInit;
339 }
340
handle() const341 const C2Handle *handle() const {
342 return &mHandle;
343 }
344
getAllocatorId() const345 C2Allocator::id_t getAllocatorId() const {
346 return mId;
347 }
348
ionHandle() const349 ion_user_handle_t ionHandle() const {
350 return mBuffer;
351 }
352
353 private:
354 int mIonFd;
355 C2HandleIon mHandle;
356 ion_user_handle_t mBuffer;
357 C2Allocator::id_t mId;
358 c2_status_t mInit;
359 int mMapFd; // only one for now
360 struct Mapping {
361 void *addr;
362 size_t alignmentBytes;
363 size_t size;
364 };
365 std::list<Mapping> mMappings;
366 };
367
map(size_t offset,size_t size,C2MemoryUsage usage,C2Fence * fence,void ** addr)368 c2_status_t C2AllocationIon::map(
369 size_t offset, size_t size, C2MemoryUsage usage, C2Fence *fence, void **addr) {
370 return mImpl->map(offset, size, usage, fence, addr);
371 }
372
unmap(void * addr,size_t size,C2Fence * fence)373 c2_status_t C2AllocationIon::unmap(void *addr, size_t size, C2Fence *fence) {
374 return mImpl->unmap(addr, size, fence);
375 }
376
status() const377 c2_status_t C2AllocationIon::status() const {
378 return mImpl->status();
379 }
380
getAllocatorId() const381 C2Allocator::id_t C2AllocationIon::getAllocatorId() const {
382 return mImpl->getAllocatorId();
383 }
384
equals(const std::shared_ptr<C2LinearAllocation> & other) const385 bool C2AllocationIon::equals(const std::shared_ptr<C2LinearAllocation> &other) const {
386 if (!other || other->getAllocatorId() != getAllocatorId()) {
387 return false;
388 }
389 // get user handle to compare objects
390 std::shared_ptr<C2AllocationIon> otherAsIon = std::static_pointer_cast<C2AllocationIon>(other);
391 return mImpl->ionHandle() == otherAsIon->mImpl->ionHandle();
392 }
393
handle() const394 const C2Handle *C2AllocationIon::handle() const {
395 return mImpl->handle();
396 }
397
~C2AllocationIon()398 C2AllocationIon::~C2AllocationIon() {
399 delete mImpl;
400 }
401
C2AllocationIon(int ionFd,size_t size,size_t align,unsigned heapMask,unsigned flags,C2Allocator::id_t id)402 C2AllocationIon::C2AllocationIon(int ionFd, size_t size, size_t align,
403 unsigned heapMask, unsigned flags, C2Allocator::id_t id)
404 : C2LinearAllocation(size),
405 mImpl(Impl::Alloc(ionFd, size, align, heapMask, flags, id)) { }
406
C2AllocationIon(int ionFd,size_t size,int shareFd,C2Allocator::id_t id)407 C2AllocationIon::C2AllocationIon(int ionFd, size_t size, int shareFd, C2Allocator::id_t id)
408 : C2LinearAllocation(size),
409 mImpl(Impl::Import(ionFd, size, shareFd, id)) { }
410
411 /* ======================================= ION ALLOCATOR ====================================== */
C2AllocatorIon(id_t id)412 C2AllocatorIon::C2AllocatorIon(id_t id)
413 : mInit(C2_OK),
414 mIonFd(ion_open()) {
415 if (mIonFd < 0) {
416 switch (errno) {
417 case ENOENT: mInit = C2_OMITTED; break;
418 default: mInit = c2_map_errno<EACCES>(errno); break;
419 }
420 } else {
421 C2MemoryUsage minUsage = { 0, 0 };
422 C2MemoryUsage maxUsage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
423 Traits traits = { "android.allocator.ion", id, LINEAR, minUsage, maxUsage };
424 mTraits = std::make_shared<Traits>(traits);
425 mBlockSize = ::getpagesize();
426 }
427 }
428
~C2AllocatorIon()429 C2AllocatorIon::~C2AllocatorIon() {
430 if (mInit == C2_OK) {
431 ion_close(mIonFd);
432 }
433 }
434
getId() const435 C2Allocator::id_t C2AllocatorIon::getId() const {
436 std::lock_guard<std::mutex> lock(mUsageMapperLock);
437 return mTraits->id;
438 }
439
getName() const440 C2String C2AllocatorIon::getName() const {
441 std::lock_guard<std::mutex> lock(mUsageMapperLock);
442 return mTraits->name;
443 }
444
getTraits() const445 std::shared_ptr<const C2Allocator::Traits> C2AllocatorIon::getTraits() const {
446 std::lock_guard<std::mutex> lock(mUsageMapperLock);
447 return mTraits;
448 }
449
setUsageMapper(const UsageMapperFn & mapper,uint64_t minUsage,uint64_t maxUsage,uint64_t blockSize)450 void C2AllocatorIon::setUsageMapper(
451 const UsageMapperFn &mapper, uint64_t minUsage, uint64_t maxUsage, uint64_t blockSize) {
452 std::lock_guard<std::mutex> lock(mUsageMapperLock);
453 mUsageMapperCache.clear();
454 mUsageMapperLru.clear();
455 mUsageMapper = mapper;
456 Traits traits = {
457 mTraits->name, mTraits->id, LINEAR,
458 C2MemoryUsage(minUsage), C2MemoryUsage(maxUsage)
459 };
460 mTraits = std::make_shared<Traits>(traits);
461 mBlockSize = blockSize;
462 }
463
operator ()(const MapperKey & k) const464 std::size_t C2AllocatorIon::MapperKeyHash::operator()(const MapperKey &k) const {
465 return std::hash<uint64_t>{}(k.first) ^ std::hash<size_t>{}(k.second);
466 }
467
mapUsage(C2MemoryUsage usage,size_t capacity,size_t * align,unsigned * heapMask,unsigned * flags)468 c2_status_t C2AllocatorIon::mapUsage(
469 C2MemoryUsage usage, size_t capacity, size_t *align, unsigned *heapMask, unsigned *flags) {
470 std::lock_guard<std::mutex> lock(mUsageMapperLock);
471 c2_status_t res = C2_OK;
472 // align capacity
473 capacity = (capacity + mBlockSize - 1) & ~(mBlockSize - 1);
474 MapperKey key = std::make_pair(usage.expected, capacity);
475 auto entry = mUsageMapperCache.find(key);
476 if (entry == mUsageMapperCache.end()) {
477 if (mUsageMapper) {
478 res = mUsageMapper(usage, capacity, align, heapMask, flags);
479 } else {
480 *align = 0; // TODO make this 1
481 *heapMask = ~0; // default mask
482 *flags = 0; // default flags
483 res = C2_NO_INIT;
484 }
485 // add usage to cache
486 MapperValue value = std::make_tuple(*align, *heapMask, *flags, res);
487 mUsageMapperLru.emplace_front(key, value);
488 mUsageMapperCache.emplace(std::make_pair(key, mUsageMapperLru.begin()));
489 if (mUsageMapperCache.size() > USAGE_LRU_CACHE_SIZE) {
490 // remove LRU entry
491 MapperKey lruKey = mUsageMapperLru.front().first;
492 mUsageMapperCache.erase(lruKey);
493 mUsageMapperLru.pop_back();
494 }
495 } else {
496 // move entry to MRU
497 mUsageMapperLru.splice(mUsageMapperLru.begin(), mUsageMapperLru, entry->second);
498 const MapperValue &value = entry->second->second;
499 std::tie(*align, *heapMask, *flags, res) = value;
500 }
501 return res;
502 }
503
newLinearAllocation(uint32_t capacity,C2MemoryUsage usage,std::shared_ptr<C2LinearAllocation> * allocation)504 c2_status_t C2AllocatorIon::newLinearAllocation(
505 uint32_t capacity, C2MemoryUsage usage, std::shared_ptr<C2LinearAllocation> *allocation) {
506 if (allocation == nullptr) {
507 return C2_BAD_VALUE;
508 }
509
510 allocation->reset();
511 if (mInit != C2_OK) {
512 return mInit;
513 }
514
515 size_t align = 0;
516 unsigned heapMask = ~0;
517 unsigned flags = 0;
518 c2_status_t ret = mapUsage(usage, capacity, &align, &heapMask, &flags);
519 if (ret && ret != C2_NO_INIT) {
520 return ret;
521 }
522
523 std::shared_ptr<C2AllocationIon> alloc
524 = std::make_shared<C2AllocationIon>(dup(mIonFd), capacity, align, heapMask, flags, mTraits->id);
525 ret = alloc->status();
526 if (ret == C2_OK) {
527 *allocation = alloc;
528 }
529 return ret;
530 }
531
priorLinearAllocation(const C2Handle * handle,std::shared_ptr<C2LinearAllocation> * allocation)532 c2_status_t C2AllocatorIon::priorLinearAllocation(
533 const C2Handle *handle, std::shared_ptr<C2LinearAllocation> *allocation) {
534 *allocation = nullptr;
535 if (mInit != C2_OK) {
536 return mInit;
537 }
538
539 if (!C2HandleIon::isValid(handle)) {
540 return C2_BAD_VALUE;
541 }
542
543 // TODO: get capacity and validate it
544 const C2HandleIon *h = static_cast<const C2HandleIon*>(handle);
545 std::shared_ptr<C2AllocationIon> alloc
546 = std::make_shared<C2AllocationIon>(dup(mIonFd), h->size(), h->bufferFd(), mTraits->id);
547 c2_status_t ret = alloc->status();
548 if (ret == C2_OK) {
549 *allocation = alloc;
550 native_handle_delete(const_cast<native_handle_t*>(
551 reinterpret_cast<const native_handle_t*>(handle)));
552 }
553 return ret;
554 }
555
isValid(const C2Handle * const o)556 bool C2AllocatorIon::isValid(const C2Handle* const o) {
557 return C2HandleIon::isValid(o);
558 }
559
560 } // namespace android
561
562