summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-06 01:02:30 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-06 01:02:30 +0000
commit76cb841cb886eef6b3bee341a2266c76578724ad (patch)
treef5892e5ba6cc11949952a6ce4ecbe6d516d6ce58 /drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
parentInitial commit. (diff)
downloadlinux-upstream.tar.xz
linux-upstream.zip
Adding upstream version 4.19.249.upstream/4.19.249upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c')
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c537
1 files changed, 537 insertions, 0 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
new file mode 100644
index 000000000..e55508b39
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -0,0 +1,537 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ */
+/*
+ * Authors:
+ * Christian König <christian.koenig@amd.com>
+ */
+
+/**
+ * DOC: MMU Notifier
+ *
+ * For coherent userptr handling registers an MMU notifier to inform the driver
+ * about updates on the page tables of a process.
+ *
+ * When somebody tries to invalidate the page tables we block the update until
+ * all operations on the pages in question are completed, then those pages are
+ * marked as accessed and also dirty if it wasn't a read only access.
+ *
+ * New command submissions using the userptrs in question are delayed until all
+ * page table invalidation are completed and we once more see a coherent process
+ * address space.
+ */
+
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/mmu_notifier.h>
+#include <linux/interval_tree.h>
+#include <drm/drmP.h>
+#include <drm/drm.h>
+
+#include "amdgpu.h"
+#include "amdgpu_amdkfd.h"
+
+/**
+ * struct amdgpu_mn
+ *
+ * @adev: amdgpu device pointer
+ * @mm: process address space
+ * @mn: MMU notifier structure
+ * @type: type of MMU notifier
+ * @work: destruction work item
+ * @node: hash table node to find structure by adev and mn
+ * @lock: rw semaphore protecting the notifier nodes
+ * @objects: interval tree containing amdgpu_mn_nodes
+ * @read_lock: mutex for recursive locking of @lock
+ * @recursion: depth of recursion
+ *
+ * Data for each amdgpu device and process address space.
+ */
+struct amdgpu_mn {
+ /* constant after initialisation */
+ struct amdgpu_device *adev;
+ struct mm_struct *mm;
+ struct mmu_notifier mn;
+ enum amdgpu_mn_type type;
+
+ /* only used on destruction */
+ struct work_struct work;
+
+ /* protected by adev->mn_lock */
+ struct hlist_node node;
+
+ /* objects protected by lock */
+ struct rw_semaphore lock;
+ struct rb_root_cached objects;
+ struct mutex read_lock;
+ atomic_t recursion;
+};
+
+/**
+ * struct amdgpu_mn_node
+ *
+ * @it: interval node defining start-last of the affected address range
+ * @bos: list of all BOs in the affected address range
+ *
+ * Manages all BOs which are affected of a certain range of address space.
+ */
+struct amdgpu_mn_node {
+ struct interval_tree_node it;
+ struct list_head bos;
+};
+
+/**
+ * amdgpu_mn_destroy - destroy the MMU notifier
+ *
+ * @work: previously sheduled work item
+ *
+ * Lazy destroys the notifier from a work item
+ */
+static void amdgpu_mn_destroy(struct work_struct *work)
+{
+ struct amdgpu_mn *amn = container_of(work, struct amdgpu_mn, work);
+ struct amdgpu_device *adev = amn->adev;
+ struct amdgpu_mn_node *node, *next_node;
+ struct amdgpu_bo *bo, *next_bo;
+
+ mutex_lock(&adev->mn_lock);
+ down_write(&amn->lock);
+ hash_del(&amn->node);
+ rbtree_postorder_for_each_entry_safe(node, next_node,
+ &amn->objects.rb_root, it.rb) {
+ list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
+ bo->mn = NULL;
+ list_del_init(&bo->mn_list);
+ }
+ kfree(node);
+ }
+ up_write(&amn->lock);
+ mutex_unlock(&adev->mn_lock);
+ mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
+ kfree(amn);
+}
+
+/**
+ * amdgpu_mn_release - callback to notify about mm destruction
+ *
+ * @mn: our notifier
+ * @mm: the mm this callback is about
+ *
+ * Shedule a work item to lazy destroy our notifier.
+ */
+static void amdgpu_mn_release(struct mmu_notifier *mn,
+ struct mm_struct *mm)
+{
+ struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+
+ INIT_WORK(&amn->work, amdgpu_mn_destroy);
+ schedule_work(&amn->work);
+}
+
+
+/**
+ * amdgpu_mn_lock - take the write side lock for this notifier
+ *
+ * @mn: our notifier
+ */
+void amdgpu_mn_lock(struct amdgpu_mn *mn)
+{
+ if (mn)
+ down_write(&mn->lock);
+}
+
+/**
+ * amdgpu_mn_unlock - drop the write side lock for this notifier
+ *
+ * @mn: our notifier
+ */
+void amdgpu_mn_unlock(struct amdgpu_mn *mn)
+{
+ if (mn)
+ up_write(&mn->lock);
+}
+
+/**
+ * amdgpu_mn_read_lock - take the read side lock for this notifier
+ *
+ * @amn: our notifier
+ */
+static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable)
+{
+ if (blockable)
+ mutex_lock(&amn->read_lock);
+ else if (!mutex_trylock(&amn->read_lock))
+ return -EAGAIN;
+
+ if (atomic_inc_return(&amn->recursion) == 1)
+ down_read_non_owner(&amn->lock);
+ mutex_unlock(&amn->read_lock);
+
+ return 0;
+}
+
+/**
+ * amdgpu_mn_read_unlock - drop the read side lock for this notifier
+ *
+ * @amn: our notifier
+ */
+static void amdgpu_mn_read_unlock(struct amdgpu_mn *amn)
+{
+ if (atomic_dec_return(&amn->recursion) == 0)
+ up_read_non_owner(&amn->lock);
+}
+
+/**
+ * amdgpu_mn_invalidate_node - unmap all BOs of a node
+ *
+ * @node: the node with the BOs to unmap
+ * @start: start of address range affected
+ * @end: end of address range affected
+ *
+ * Block for operations on BOs to finish and mark pages as accessed and
+ * potentially dirty.
+ */
+static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
+ unsigned long start,
+ unsigned long end)
+{
+ struct amdgpu_bo *bo;
+ long r;
+
+ list_for_each_entry(bo, &node->bos, mn_list) {
+
+ if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end))
+ continue;
+
+ r = reservation_object_wait_timeout_rcu(bo->tbo.resv,
+ true, false, MAX_SCHEDULE_TIMEOUT);
+ if (r <= 0)
+ DRM_ERROR("(%ld) failed to wait for user bo\n", r);
+
+ amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
+ }
+}
+
+/**
+ * amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change
+ *
+ * @mn: our notifier
+ * @mm: the mm this callback is about
+ * @start: start of updated range
+ * @end: end of updated range
+ *
+ * Block for operations on BOs to finish and mark pages as accessed and
+ * potentially dirty.
+ */
+static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
+ struct mm_struct *mm,
+ unsigned long start,
+ unsigned long end,
+ bool blockable)
+{
+ struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+ struct interval_tree_node *it;
+
+ /* notification is exclusive, but interval is inclusive */
+ end -= 1;
+
+ /* TODO we should be able to split locking for interval tree and
+ * amdgpu_mn_invalidate_node
+ */
+ if (amdgpu_mn_read_lock(amn, blockable))
+ return -EAGAIN;
+
+ it = interval_tree_iter_first(&amn->objects, start, end);
+ while (it) {
+ struct amdgpu_mn_node *node;
+
+ if (!blockable) {
+ amdgpu_mn_read_unlock(amn);
+ return -EAGAIN;
+ }
+
+ node = container_of(it, struct amdgpu_mn_node, it);
+ it = interval_tree_iter_next(it, start, end);
+
+ amdgpu_mn_invalidate_node(node, start, end);
+ }
+
+ return 0;
+}
+
+/**
+ * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change
+ *
+ * @mn: our notifier
+ * @mm: the mm this callback is about
+ * @start: start of updated range
+ * @end: end of updated range
+ *
+ * We temporarily evict all BOs between start and end. This
+ * necessitates evicting all user-mode queues of the process. The BOs
+ * are restorted in amdgpu_mn_invalidate_range_end_hsa.
+ */
+static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
+ struct mm_struct *mm,
+ unsigned long start,
+ unsigned long end,
+ bool blockable)
+{
+ struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+ struct interval_tree_node *it;
+
+ /* notification is exclusive, but interval is inclusive */
+ end -= 1;
+
+ if (amdgpu_mn_read_lock(amn, blockable))
+ return -EAGAIN;
+
+ it = interval_tree_iter_first(&amn->objects, start, end);
+ while (it) {
+ struct amdgpu_mn_node *node;
+ struct amdgpu_bo *bo;
+
+ if (!blockable) {
+ amdgpu_mn_read_unlock(amn);
+ return -EAGAIN;
+ }
+
+ node = container_of(it, struct amdgpu_mn_node, it);
+ it = interval_tree_iter_next(it, start, end);
+
+ list_for_each_entry(bo, &node->bos, mn_list) {
+ struct kgd_mem *mem = bo->kfd_bo;
+
+ if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
+ start, end))
+ amdgpu_amdkfd_evict_userptr(mem, mm);
+ }
+ }
+
+ return 0;
+}
+
+/**
+ * amdgpu_mn_invalidate_range_end - callback to notify about mm change
+ *
+ * @mn: our notifier
+ * @mm: the mm this callback is about
+ * @start: start of updated range
+ * @end: end of updated range
+ *
+ * Release the lock again to allow new command submissions.
+ */
+static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
+ struct mm_struct *mm,
+ unsigned long start,
+ unsigned long end)
+{
+ struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+
+ amdgpu_mn_read_unlock(amn);
+}
+
+static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
+ [AMDGPU_MN_TYPE_GFX] = {
+ .release = amdgpu_mn_release,
+ .invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx,
+ .invalidate_range_end = amdgpu_mn_invalidate_range_end,
+ },
+ [AMDGPU_MN_TYPE_HSA] = {
+ .release = amdgpu_mn_release,
+ .invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa,
+ .invalidate_range_end = amdgpu_mn_invalidate_range_end,
+ },
+};
+
+/* Low bits of any reasonable mm pointer will be unused due to struct
+ * alignment. Use these bits to make a unique key from the mm pointer
+ * and notifier type.
+ */
+#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
+
+/**
+ * amdgpu_mn_get - create notifier context
+ *
+ * @adev: amdgpu device pointer
+ * @type: type of MMU notifier context
+ *
+ * Creates a notifier context for current->mm.
+ */
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+ enum amdgpu_mn_type type)
+{
+ struct mm_struct *mm = current->mm;
+ struct amdgpu_mn *amn;
+ unsigned long key = AMDGPU_MN_KEY(mm, type);
+ int r;
+
+ mutex_lock(&adev->mn_lock);
+ if (down_write_killable(&mm->mmap_sem)) {
+ mutex_unlock(&adev->mn_lock);
+ return ERR_PTR(-EINTR);
+ }
+
+ hash_for_each_possible(adev->mn_hash, amn, node, key)
+ if (AMDGPU_MN_KEY(amn->mm, amn->type) == key)
+ goto release_locks;
+
+ amn = kzalloc(sizeof(*amn), GFP_KERNEL);
+ if (!amn) {
+ amn = ERR_PTR(-ENOMEM);
+ goto release_locks;
+ }
+
+ amn->adev = adev;
+ amn->mm = mm;
+ init_rwsem(&amn->lock);
+ amn->type = type;
+ amn->mn.ops = &amdgpu_mn_ops[type];
+ amn->objects = RB_ROOT_CACHED;
+ mutex_init(&amn->read_lock);
+ atomic_set(&amn->recursion, 0);
+
+ r = __mmu_notifier_register(&amn->mn, mm);
+ if (r)
+ goto free_amn;
+
+ hash_add(adev->mn_hash, &amn->node, AMDGPU_MN_KEY(mm, type));
+
+release_locks:
+ up_write(&mm->mmap_sem);
+ mutex_unlock(&adev->mn_lock);
+
+ return amn;
+
+free_amn:
+ up_write(&mm->mmap_sem);
+ mutex_unlock(&adev->mn_lock);
+ kfree(amn);
+
+ return ERR_PTR(r);
+}
+
+/**
+ * amdgpu_mn_register - register a BO for notifier updates
+ *
+ * @bo: amdgpu buffer object
+ * @addr: userptr addr we should monitor
+ *
+ * Registers an MMU notifier for the given BO at the specified address.
+ * Returns 0 on success, -ERRNO if anything goes wrong.
+ */
+int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
+{
+ unsigned long end = addr + amdgpu_bo_size(bo) - 1;
+ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+ enum amdgpu_mn_type type =
+ bo->kfd_bo ? AMDGPU_MN_TYPE_HSA : AMDGPU_MN_TYPE_GFX;
+ struct amdgpu_mn *amn;
+ struct amdgpu_mn_node *node = NULL, *new_node;
+ struct list_head bos;
+ struct interval_tree_node *it;
+
+ amn = amdgpu_mn_get(adev, type);
+ if (IS_ERR(amn))
+ return PTR_ERR(amn);
+
+ new_node = kmalloc(sizeof(*new_node), GFP_KERNEL);
+ if (!new_node)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&bos);
+
+ down_write(&amn->lock);
+
+ while ((it = interval_tree_iter_first(&amn->objects, addr, end))) {
+ kfree(node);
+ node = container_of(it, struct amdgpu_mn_node, it);
+ interval_tree_remove(&node->it, &amn->objects);
+ addr = min(it->start, addr);
+ end = max(it->last, end);
+ list_splice(&node->bos, &bos);
+ }
+
+ if (!node)
+ node = new_node;
+ else
+ kfree(new_node);
+
+ bo->mn = amn;
+
+ node->it.start = addr;
+ node->it.last = end;
+ INIT_LIST_HEAD(&node->bos);
+ list_splice(&bos, &node->bos);
+ list_add(&bo->mn_list, &node->bos);
+
+ interval_tree_insert(&node->it, &amn->objects);
+
+ up_write(&amn->lock);
+
+ return 0;
+}
+
+/**
+ * amdgpu_mn_unregister - unregister a BO for notifier updates
+ *
+ * @bo: amdgpu buffer object
+ *
+ * Remove any registration of MMU notifier updates from the buffer object.
+ */
+void amdgpu_mn_unregister(struct amdgpu_bo *bo)
+{
+ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+ struct amdgpu_mn *amn;
+ struct list_head *head;
+
+ mutex_lock(&adev->mn_lock);
+
+ amn = bo->mn;
+ if (amn == NULL) {
+ mutex_unlock(&adev->mn_lock);
+ return;
+ }
+
+ down_write(&amn->lock);
+
+ /* save the next list entry for later */
+ head = bo->mn_list.next;
+
+ bo->mn = NULL;
+ list_del_init(&bo->mn_list);
+
+ if (list_empty(head)) {
+ struct amdgpu_mn_node *node;
+
+ node = container_of(head, struct amdgpu_mn_node, bos);
+ interval_tree_remove(&node->it, &amn->objects);
+ kfree(node);
+ }
+
+ up_write(&amn->lock);
+ mutex_unlock(&adev->mn_lock);
+}
+