mod.rs 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329
  1. // SPDX-License-Identifier: GPL-2.0
  2. // Copyright (C) 2025 Google LLC.
  3. use kernel::{page::PAGE_SIZE, prelude::*, seq_file::SeqFile, task::Pid};
  4. mod tree;
  5. use self::tree::{FromArrayAllocs, ReserveNewTreeAlloc, TreeRangeAllocator};
  6. mod array;
  7. use self::array::{ArrayRangeAllocator, EmptyArrayAlloc};
  8. enum DescriptorState<T> {
  9. Reserved(Reservation),
  10. Allocated(Allocation<T>),
  11. }
  12. impl<T> DescriptorState<T> {
  13. fn new(is_oneway: bool, debug_id: usize, pid: Pid) -> Self {
  14. DescriptorState::Reserved(Reservation {
  15. debug_id,
  16. is_oneway,
  17. pid,
  18. })
  19. }
  20. fn pid(&self) -> Pid {
  21. match self {
  22. DescriptorState::Reserved(inner) => inner.pid,
  23. DescriptorState::Allocated(inner) => inner.reservation.pid,
  24. }
  25. }
  26. fn is_oneway(&self) -> bool {
  27. match self {
  28. DescriptorState::Reserved(inner) => inner.is_oneway,
  29. DescriptorState::Allocated(inner) => inner.reservation.is_oneway,
  30. }
  31. }
  32. }
  33. #[derive(Clone)]
  34. struct Reservation {
  35. debug_id: usize,
  36. is_oneway: bool,
  37. pid: Pid,
  38. }
  39. impl Reservation {
  40. fn allocate<T>(self, data: Option<T>) -> Allocation<T> {
  41. Allocation {
  42. data,
  43. reservation: self,
  44. }
  45. }
  46. }
  47. struct Allocation<T> {
  48. reservation: Reservation,
  49. data: Option<T>,
  50. }
  51. impl<T> Allocation<T> {
  52. fn deallocate(self) -> (Reservation, Option<T>) {
  53. (self.reservation, self.data)
  54. }
  55. fn debug_id(&self) -> usize {
  56. self.reservation.debug_id
  57. }
  58. fn take(&mut self) -> Option<T> {
  59. self.data.take()
  60. }
  61. }
  62. /// The array implementation must switch to the tree if it wants to go beyond this number of
  63. /// ranges.
  64. const TREE_THRESHOLD: usize = 8;
  65. /// Represents a range of pages that have just become completely free.
  66. #[derive(Copy, Clone)]
  67. pub(crate) struct FreedRange {
  68. pub(crate) start_page_idx: usize,
  69. pub(crate) end_page_idx: usize,
  70. }
  71. impl FreedRange {
  72. fn interior_pages(offset: usize, size: usize) -> FreedRange {
  73. FreedRange {
  74. // Divide round up
  75. start_page_idx: offset.div_ceil(PAGE_SIZE),
  76. // Divide round down
  77. end_page_idx: (offset + size) / PAGE_SIZE,
  78. }
  79. }
  80. }
  81. struct Range<T> {
  82. offset: usize,
  83. size: usize,
  84. state: DescriptorState<T>,
  85. }
  86. impl<T> Range<T> {
  87. fn endpoint(&self) -> usize {
  88. self.offset + self.size
  89. }
  90. }
  91. pub(crate) struct RangeAllocator<T> {
  92. inner: Impl<T>,
  93. }
  94. enum Impl<T> {
  95. Empty(usize),
  96. Array(ArrayRangeAllocator<T>),
  97. Tree(TreeRangeAllocator<T>),
  98. }
  99. impl<T> RangeAllocator<T> {
  100. pub(crate) fn new(size: usize) -> Self {
  101. Self {
  102. inner: Impl::Empty(size),
  103. }
  104. }
  105. pub(crate) fn free_oneway_space(&self) -> usize {
  106. match &self.inner {
  107. Impl::Empty(size) => size / 2,
  108. Impl::Array(array) => array.free_oneway_space(),
  109. Impl::Tree(tree) => tree.free_oneway_space(),
  110. }
  111. }
  112. pub(crate) fn count_buffers(&self) -> usize {
  113. match &self.inner {
  114. Impl::Empty(_size) => 0,
  115. Impl::Array(array) => array.count_buffers(),
  116. Impl::Tree(tree) => tree.count_buffers(),
  117. }
  118. }
  119. pub(crate) fn debug_print(&self, m: &SeqFile) -> Result<()> {
  120. match &self.inner {
  121. Impl::Empty(_size) => Ok(()),
  122. Impl::Array(array) => array.debug_print(m),
  123. Impl::Tree(tree) => tree.debug_print(m),
  124. }
  125. }
  126. /// Try to reserve a new buffer, using the provided allocation if necessary.
  127. pub(crate) fn reserve_new(&mut self, mut args: ReserveNewArgs<T>) -> Result<ReserveNew<T>> {
  128. match &mut self.inner {
  129. Impl::Empty(size) => {
  130. let empty_array = match args.empty_array_alloc.take() {
  131. Some(empty_array) => ArrayRangeAllocator::new(*size, empty_array),
  132. None => {
  133. return Ok(ReserveNew::NeedAlloc(ReserveNewNeedAlloc {
  134. args,
  135. need_empty_array_alloc: true,
  136. need_new_tree_alloc: false,
  137. need_tree_alloc: false,
  138. }))
  139. }
  140. };
  141. self.inner = Impl::Array(empty_array);
  142. self.reserve_new(args)
  143. }
  144. Impl::Array(array) if array.is_full() => {
  145. let allocs = match args.new_tree_alloc {
  146. Some(ref mut allocs) => allocs,
  147. None => {
  148. return Ok(ReserveNew::NeedAlloc(ReserveNewNeedAlloc {
  149. args,
  150. need_empty_array_alloc: false,
  151. need_new_tree_alloc: true,
  152. need_tree_alloc: true,
  153. }))
  154. }
  155. };
  156. let new_tree =
  157. TreeRangeAllocator::from_array(array.total_size(), &mut array.ranges, allocs);
  158. self.inner = Impl::Tree(new_tree);
  159. self.reserve_new(args)
  160. }
  161. Impl::Array(array) => {
  162. let (offset, oneway_spam_detected) =
  163. array.reserve_new(args.debug_id, args.size, args.is_oneway, args.pid)?;
  164. Ok(ReserveNew::Success(ReserveNewSuccess {
  165. offset,
  166. oneway_spam_detected,
  167. _empty_array_alloc: args.empty_array_alloc,
  168. _new_tree_alloc: args.new_tree_alloc,
  169. _tree_alloc: args.tree_alloc,
  170. }))
  171. }
  172. Impl::Tree(tree) => {
  173. let alloc = match args.tree_alloc {
  174. Some(alloc) => alloc,
  175. None => {
  176. return Ok(ReserveNew::NeedAlloc(ReserveNewNeedAlloc {
  177. args,
  178. need_empty_array_alloc: false,
  179. need_new_tree_alloc: false,
  180. need_tree_alloc: true,
  181. }));
  182. }
  183. };
  184. let (offset, oneway_spam_detected) =
  185. tree.reserve_new(args.debug_id, args.size, args.is_oneway, args.pid, alloc)?;
  186. Ok(ReserveNew::Success(ReserveNewSuccess {
  187. offset,
  188. oneway_spam_detected,
  189. _empty_array_alloc: args.empty_array_alloc,
  190. _new_tree_alloc: args.new_tree_alloc,
  191. _tree_alloc: None,
  192. }))
  193. }
  194. }
  195. }
  196. /// Deletes the allocations at `offset`.
  197. pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result<FreedRange> {
  198. match &mut self.inner {
  199. Impl::Empty(_size) => Err(EINVAL),
  200. Impl::Array(array) => array.reservation_abort(offset),
  201. Impl::Tree(tree) => {
  202. let freed_range = tree.reservation_abort(offset)?;
  203. if tree.is_empty() {
  204. self.inner = Impl::Empty(tree.total_size());
  205. }
  206. Ok(freed_range)
  207. }
  208. }
  209. }
  210. /// Called when an allocation is no longer in use by the kernel.
  211. ///
  212. /// The value in `data` will be stored, if any. A mutable reference is used to avoid dropping
  213. /// the `T` when an error is returned.
  214. pub(crate) fn reservation_commit(&mut self, offset: usize, data: &mut Option<T>) -> Result {
  215. match &mut self.inner {
  216. Impl::Empty(_size) => Err(EINVAL),
  217. Impl::Array(array) => array.reservation_commit(offset, data),
  218. Impl::Tree(tree) => tree.reservation_commit(offset, data),
  219. }
  220. }
  221. /// Called when the kernel starts using an allocation.
  222. ///
  223. /// Returns the size of the existing entry and the data associated with it.
  224. pub(crate) fn reserve_existing(&mut self, offset: usize) -> Result<(usize, usize, Option<T>)> {
  225. match &mut self.inner {
  226. Impl::Empty(_size) => Err(EINVAL),
  227. Impl::Array(array) => array.reserve_existing(offset),
  228. Impl::Tree(tree) => tree.reserve_existing(offset),
  229. }
  230. }
  231. /// Call the provided callback at every allocated region.
  232. ///
  233. /// This destroys the range allocator. Used only during shutdown.
  234. pub(crate) fn take_for_each<F: Fn(usize, usize, usize, Option<T>)>(&mut self, callback: F) {
  235. match &mut self.inner {
  236. Impl::Empty(_size) => {}
  237. Impl::Array(array) => array.take_for_each(callback),
  238. Impl::Tree(tree) => tree.take_for_each(callback),
  239. }
  240. }
  241. }
  242. /// The arguments for `reserve_new`.
  243. #[derive(Default)]
  244. pub(crate) struct ReserveNewArgs<T> {
  245. pub(crate) size: usize,
  246. pub(crate) is_oneway: bool,
  247. pub(crate) debug_id: usize,
  248. pub(crate) pid: Pid,
  249. pub(crate) empty_array_alloc: Option<EmptyArrayAlloc<T>>,
  250. pub(crate) new_tree_alloc: Option<FromArrayAllocs<T>>,
  251. pub(crate) tree_alloc: Option<ReserveNewTreeAlloc<T>>,
  252. }
  253. /// The return type of `ReserveNew`.
  254. pub(crate) enum ReserveNew<T> {
  255. Success(ReserveNewSuccess<T>),
  256. NeedAlloc(ReserveNewNeedAlloc<T>),
  257. }
  258. /// Returned by `reserve_new` when the reservation was successul.
  259. pub(crate) struct ReserveNewSuccess<T> {
  260. pub(crate) offset: usize,
  261. pub(crate) oneway_spam_detected: bool,
  262. // If the user supplied an allocation that we did not end up using, then we return it here.
  263. // The caller will kfree it outside of the lock.
  264. _empty_array_alloc: Option<EmptyArrayAlloc<T>>,
  265. _new_tree_alloc: Option<FromArrayAllocs<T>>,
  266. _tree_alloc: Option<ReserveNewTreeAlloc<T>>,
  267. }
  268. /// Returned by `reserve_new` to request the caller to make an allocation before calling the method
  269. /// again.
  270. pub(crate) struct ReserveNewNeedAlloc<T> {
  271. args: ReserveNewArgs<T>,
  272. need_empty_array_alloc: bool,
  273. need_new_tree_alloc: bool,
  274. need_tree_alloc: bool,
  275. }
  276. impl<T> ReserveNewNeedAlloc<T> {
  277. /// Make the necessary allocations for another call to `reserve_new`.
  278. pub(crate) fn make_alloc(mut self) -> Result<ReserveNewArgs<T>> {
  279. if self.need_empty_array_alloc && self.args.empty_array_alloc.is_none() {
  280. self.args.empty_array_alloc = Some(EmptyArrayAlloc::try_new(TREE_THRESHOLD)?);
  281. }
  282. if self.need_new_tree_alloc && self.args.new_tree_alloc.is_none() {
  283. self.args.new_tree_alloc = Some(FromArrayAllocs::try_new(TREE_THRESHOLD)?);
  284. }
  285. if self.need_tree_alloc && self.args.tree_alloc.is_none() {
  286. self.args.tree_alloc = Some(ReserveNewTreeAlloc::try_new()?);
  287. }
  288. Ok(self.args)
  289. }
  290. }