1
0

array.rs 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282
  1. // SPDX-License-Identifier: GPL-2.0
  2. // Copyright (C) 2025 Google LLC.
  3. use kernel::{
  4. page::{PAGE_MASK, PAGE_SIZE},
  5. prelude::*,
  6. seq_file::SeqFile,
  7. seq_print,
  8. task::Pid,
  9. };
  10. use crate::range_alloc::{DescriptorState, FreedRange, Range};
  11. /// Keeps track of allocations in a process' mmap.
  12. ///
  13. /// Each process has an mmap where the data for incoming transactions will be placed. This struct
  14. /// keeps track of allocations made in the mmap. For each allocation, we store a descriptor that
  15. /// has metadata related to the allocation. We also keep track of available free space.
  16. pub(super) struct ArrayRangeAllocator<T> {
  17. /// This stores all ranges that are allocated. Unlike the tree based allocator, we do *not*
  18. /// store the free ranges.
  19. ///
  20. /// Sorted by offset.
  21. pub(super) ranges: KVec<Range<T>>,
  22. size: usize,
  23. free_oneway_space: usize,
  24. }
  25. struct FindEmptyRes {
  26. /// Which index in `ranges` should we insert the new range at?
  27. ///
  28. /// Inserting the new range at this index keeps `ranges` sorted.
  29. insert_at_idx: usize,
  30. /// Which offset should we insert the new range at?
  31. insert_at_offset: usize,
  32. }
  33. impl<T> ArrayRangeAllocator<T> {
  34. pub(crate) fn new(size: usize, alloc: EmptyArrayAlloc<T>) -> Self {
  35. Self {
  36. ranges: alloc.ranges,
  37. size,
  38. free_oneway_space: size / 2,
  39. }
  40. }
  41. pub(crate) fn free_oneway_space(&self) -> usize {
  42. self.free_oneway_space
  43. }
  44. pub(crate) fn count_buffers(&self) -> usize {
  45. self.ranges.len()
  46. }
  47. pub(crate) fn total_size(&self) -> usize {
  48. self.size
  49. }
  50. pub(crate) fn is_full(&self) -> bool {
  51. self.ranges.len() == self.ranges.capacity()
  52. }
  53. pub(crate) fn debug_print(&self, m: &SeqFile) -> Result<()> {
  54. for range in &self.ranges {
  55. seq_print!(
  56. m,
  57. " buffer {}: {} size {} pid {} oneway {}",
  58. 0,
  59. range.offset,
  60. range.size,
  61. range.state.pid(),
  62. range.state.is_oneway(),
  63. );
  64. if let DescriptorState::Reserved(_) = range.state {
  65. seq_print!(m, " reserved\n");
  66. } else {
  67. seq_print!(m, " allocated\n");
  68. }
  69. }
  70. Ok(())
  71. }
  72. /// Find somewhere to put a new range.
  73. ///
  74. /// Unlike the tree implementation, we do not bother to find the smallest gap. The idea is that
  75. /// fragmentation isn't a big issue when we don't have many ranges.
  76. ///
  77. /// Returns the index that the new range should have in `self.ranges` after insertion.
  78. fn find_empty_range(&self, size: usize) -> Option<FindEmptyRes> {
  79. let after_last_range = self.ranges.last().map(Range::endpoint).unwrap_or(0);
  80. if size <= self.total_size() - after_last_range {
  81. // We can put the range at the end, so just do that.
  82. Some(FindEmptyRes {
  83. insert_at_idx: self.ranges.len(),
  84. insert_at_offset: after_last_range,
  85. })
  86. } else {
  87. let mut end_of_prev = 0;
  88. for (i, range) in self.ranges.iter().enumerate() {
  89. // Does it fit before the i'th range?
  90. if size <= range.offset - end_of_prev {
  91. return Some(FindEmptyRes {
  92. insert_at_idx: i,
  93. insert_at_offset: end_of_prev,
  94. });
  95. }
  96. end_of_prev = range.endpoint();
  97. }
  98. None
  99. }
  100. }
  101. pub(crate) fn reserve_new(
  102. &mut self,
  103. debug_id: usize,
  104. size: usize,
  105. is_oneway: bool,
  106. pid: Pid,
  107. ) -> Result<(usize, bool)> {
  108. // Compute new value of free_oneway_space, which is set only on success.
  109. let new_oneway_space = if is_oneway {
  110. match self.free_oneway_space.checked_sub(size) {
  111. Some(new_oneway_space) => new_oneway_space,
  112. None => return Err(ENOSPC),
  113. }
  114. } else {
  115. self.free_oneway_space
  116. };
  117. let FindEmptyRes {
  118. insert_at_idx,
  119. insert_at_offset,
  120. } = self.find_empty_range(size).ok_or(ENOSPC)?;
  121. self.free_oneway_space = new_oneway_space;
  122. let new_range = Range {
  123. offset: insert_at_offset,
  124. size,
  125. state: DescriptorState::new(is_oneway, debug_id, pid),
  126. };
  127. // Insert the value at the given index to keep the array sorted.
  128. self.ranges
  129. .insert_within_capacity(insert_at_idx, new_range)
  130. .ok()
  131. .unwrap();
  132. // Start detecting spammers once we have less than 20%
  133. // of async space left (which is less than 10% of total
  134. // buffer size).
  135. //
  136. // (This will short-circuit, so `low_oneway_space` is
  137. // only called when necessary.)
  138. let oneway_spam_detected =
  139. is_oneway && new_oneway_space < self.size / 10 && self.low_oneway_space(pid);
  140. Ok((insert_at_offset, oneway_spam_detected))
  141. }
  142. /// Find the amount and size of buffers allocated by the current caller.
  143. ///
  144. /// The idea is that once we cross the threshold, whoever is responsible
  145. /// for the low async space is likely to try to send another async transaction,
  146. /// and at some point we'll catch them in the act. This is more efficient
  147. /// than keeping a map per pid.
  148. fn low_oneway_space(&self, calling_pid: Pid) -> bool {
  149. let mut total_alloc_size = 0;
  150. let mut num_buffers = 0;
  151. // Warn if this pid has more than 50 transactions, or more than 50% of
  152. // async space (which is 25% of total buffer size). Oneway spam is only
  153. // detected when the threshold is exceeded.
  154. for range in &self.ranges {
  155. if range.state.is_oneway() && range.state.pid() == calling_pid {
  156. total_alloc_size += range.size;
  157. num_buffers += 1;
  158. }
  159. }
  160. num_buffers > 50 || total_alloc_size > self.size / 4
  161. }
  162. pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result<FreedRange> {
  163. // This could use a binary search, but linear scans are usually faster for small arrays.
  164. let i = self
  165. .ranges
  166. .iter()
  167. .position(|range| range.offset == offset)
  168. .ok_or(EINVAL)?;
  169. let range = &self.ranges[i];
  170. if let DescriptorState::Allocated(_) = range.state {
  171. return Err(EPERM);
  172. }
  173. let size = range.size;
  174. let offset = range.offset;
  175. if range.state.is_oneway() {
  176. self.free_oneway_space += size;
  177. }
  178. // This computes the range of pages that are no longer used by *any* allocated range. The
  179. // caller will mark them as unused, which means that they can be freed if the system comes
  180. // under memory pressure.
  181. let mut freed_range = FreedRange::interior_pages(offset, size);
  182. #[expect(clippy::collapsible_if)] // reads better like this
  183. if offset % PAGE_SIZE != 0 {
  184. if i == 0 || self.ranges[i - 1].endpoint() <= (offset & PAGE_MASK) {
  185. freed_range.start_page_idx -= 1;
  186. }
  187. }
  188. if range.endpoint() % PAGE_SIZE != 0 {
  189. let page_after = (range.endpoint() & PAGE_MASK) + PAGE_SIZE;
  190. if i + 1 == self.ranges.len() || page_after <= self.ranges[i + 1].offset {
  191. freed_range.end_page_idx += 1;
  192. }
  193. }
  194. self.ranges.remove(i)?;
  195. Ok(freed_range)
  196. }
  197. pub(crate) fn reservation_commit(&mut self, offset: usize, data: &mut Option<T>) -> Result {
  198. // This could use a binary search, but linear scans are usually faster for small arrays.
  199. let range = self
  200. .ranges
  201. .iter_mut()
  202. .find(|range| range.offset == offset)
  203. .ok_or(ENOENT)?;
  204. let DescriptorState::Reserved(reservation) = &range.state else {
  205. return Err(ENOENT);
  206. };
  207. range.state = DescriptorState::Allocated(reservation.clone().allocate(data.take()));
  208. Ok(())
  209. }
  210. pub(crate) fn reserve_existing(&mut self, offset: usize) -> Result<(usize, usize, Option<T>)> {
  211. // This could use a binary search, but linear scans are usually faster for small arrays.
  212. let range = self
  213. .ranges
  214. .iter_mut()
  215. .find(|range| range.offset == offset)
  216. .ok_or(ENOENT)?;
  217. let DescriptorState::Allocated(allocation) = &mut range.state else {
  218. return Err(ENOENT);
  219. };
  220. let data = allocation.take();
  221. let debug_id = allocation.reservation.debug_id;
  222. range.state = DescriptorState::Reserved(allocation.reservation.clone());
  223. Ok((range.size, debug_id, data))
  224. }
  225. pub(crate) fn take_for_each<F: Fn(usize, usize, usize, Option<T>)>(&mut self, callback: F) {
  226. for range in self.ranges.iter_mut() {
  227. if let DescriptorState::Allocated(allocation) = &mut range.state {
  228. callback(
  229. range.offset,
  230. range.size,
  231. allocation.reservation.debug_id,
  232. allocation.data.take(),
  233. );
  234. }
  235. }
  236. }
  237. }
  238. pub(crate) struct EmptyArrayAlloc<T> {
  239. ranges: KVec<Range<T>>,
  240. }
  241. impl<T> EmptyArrayAlloc<T> {
  242. pub(crate) fn try_new(capacity: usize) -> Result<Self> {
  243. Ok(Self {
  244. ranges: KVec::with_capacity(capacity, GFP_KERNEL)?,
  245. })
  246. }
  247. }