ARK-OS/arkos/kernel/drivers/gpu/drm/panthor/panthor_device.c

// SPDX-License-Identifier: GPL-2.0 or MIT
/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
/* Copyright 2023 Collabora ltd. */

#include <linux/clk.h>
#include <linux/mm.h>
#include <linux/platform_device.h>
#include <linux/pm_domain.h>
#include <linux/pm_runtime.h>
#include <linux/regulator/consumer.h>
#include <linux/reset.h>

#include <drm/drm_drv.h>
#include <drm/drm_managed.h>
#include <drm/drm_print.h>

#include "panthor_devfreq.h"
#include "panthor_device.h"
#include "panthor_fw.h"
#include "panthor_gem.h"
#include "panthor_gpu.h"
#include "panthor_hw.h"
#include "panthor_mmu.h"
#include "panthor_pwr.h"
#include "panthor_regs.h"
#include "panthor_sched.h"

static int panthor_gpu_coherency_init(struct panthor_device *ptdev)
{
	BUILD_BUG_ON(GPU_COHERENCY_NONE != DRM_PANTHOR_GPU_COHERENCY_NONE);
	BUILD_BUG_ON(GPU_COHERENCY_ACE_LITE != DRM_PANTHOR_GPU_COHERENCY_ACE_LITE);
	BUILD_BUG_ON(GPU_COHERENCY_ACE != DRM_PANTHOR_GPU_COHERENCY_ACE);

	/* Start with no coherency, and update it if the device is flagged coherent. */
	ptdev->gpu_info.selected_coherency = GPU_COHERENCY_NONE;
	ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;

	if (!ptdev->coherent)
		return 0;

	/* Check if the ACE-Lite coherency protocol is actually supported by the GPU.
	 * ACE protocol has never been supported for command stream frontend GPUs.
	 */
	if ((gpu_read(ptdev, GPU_COHERENCY_FEATURES) &
		      GPU_COHERENCY_PROT_BIT(ACE_LITE))) {
		ptdev->gpu_info.selected_coherency = GPU_COHERENCY_ACE_LITE;
		return 0;
	}

	drm_err(&ptdev->base, "Coherency not supported by the device");
	return -ENOTSUPP;
}

static int panthor_clk_init(struct panthor_device *ptdev)
{
	ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL);
	if (IS_ERR(ptdev->clks.core))
		return dev_err_probe(ptdev->base.dev,
				     PTR_ERR(ptdev->clks.core),
				     "get 'core' clock failed");

	ptdev->clks.stacks = devm_clk_get_optional(ptdev->base.dev, "stacks");
	if (IS_ERR(ptdev->clks.stacks))
		return dev_err_probe(ptdev->base.dev,
				     PTR_ERR(ptdev->clks.stacks),
				     "get 'stacks' clock failed");

	ptdev->clks.coregroup = devm_clk_get_optional(ptdev->base.dev, "coregroup");
	if (IS_ERR(ptdev->clks.coregroup))
		return dev_err_probe(ptdev->base.dev,
				     PTR_ERR(ptdev->clks.coregroup),
				     "get 'coregroup' clock failed");

	drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core));
	return 0;
}

static int panthor_init_power(struct device *dev)
{
	struct dev_pm_domain_list  *pd_list = NULL;

	if (dev->pm_domain)
		return 0;

	return devm_pm_domain_attach_list(dev, NULL, &pd_list);
}

void panthor_device_unplug(struct panthor_device *ptdev)
{
	/* This function can be called from two different path: the reset work
	 * and the platform device remove callback. drm_dev_unplug() doesn't
	 * deal with concurrent callers, so we have to protect drm_dev_unplug()
	 * calls with our own lock, and bail out if the device is already
	 * unplugged.
	 */
	mutex_lock(&ptdev->unplug.lock);
	if (drm_dev_is_unplugged(&ptdev->base)) {
		/* Someone beat us, release the lock and wait for the unplug
		 * operation to be reported as done.
		 **/
		mutex_unlock(&ptdev->unplug.lock);
		wait_for_completion(&ptdev->unplug.done);
		return;
	}

	drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0);

	/* Call drm_dev_unplug() so any access to HW blocks happening after
	 * that point get rejected.
	 */
	drm_dev_unplug(&ptdev->base);

	/* We do the rest of the unplug with the unplug lock released,
	 * future callers will wait on ptdev->unplug.done anyway.
	 */
	mutex_unlock(&ptdev->unplug.lock);

	/* Now, try to cleanly shutdown the GPU before the device resources
	 * get reclaimed.
	 */
	panthor_sched_unplug(ptdev);
	panthor_fw_unplug(ptdev);
	panthor_mmu_unplug(ptdev);
	panthor_gpu_unplug(ptdev);
	panthor_pwr_unplug(ptdev);

	pm_runtime_dont_use_autosuspend(ptdev->base.dev);
	pm_runtime_put_sync_suspend(ptdev->base.dev);

	/* If PM is disabled, we need to call the suspend handler manually. */
	if (!IS_ENABLED(CONFIG_PM))
		panthor_device_suspend(ptdev->base.dev);

	/* Report the unplug operation as done to unblock concurrent
	 * panthor_device_unplug() callers.
	 */
	complete_all(&ptdev->unplug.done);
}

static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data)
{
	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);

	disable_work_sync(&ptdev->reset.work);
	destroy_workqueue(ptdev->reset.wq);
}

static void panthor_device_reset_work(struct work_struct *work)
{
	struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work);
	int ret = 0, cookie;

	/* If the device is entering suspend, we don't reset. A slow reset will
	 * be forced at resume time instead.
	 */
	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
		return;

	if (!drm_dev_enter(&ptdev->base, &cookie))
		return;

	panthor_sched_pre_reset(ptdev);
	panthor_fw_pre_reset(ptdev, true);
	panthor_mmu_pre_reset(ptdev);
	panthor_hw_soft_reset(ptdev);
	panthor_hw_l2_power_on(ptdev);
	panthor_mmu_post_reset(ptdev);
	ret = panthor_fw_post_reset(ptdev);
	atomic_set(&ptdev->reset.pending, 0);
	panthor_sched_post_reset(ptdev, ret != 0);
	drm_dev_exit(cookie);

	if (ret) {
		panthor_device_unplug(ptdev);
		drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable.");
	}
}

static bool panthor_device_is_initialized(struct panthor_device *ptdev)
{
	return !!ptdev->scheduler;
}

static void panthor_device_free_page(struct drm_device *ddev, void *data)
{
	__free_page(data);
}

int panthor_device_init(struct panthor_device *ptdev)
{
	u32 *dummy_page_virt;
	struct resource *res;
	struct page *p;
	int ret;

	ptdev->soc_data = of_device_get_match_data(ptdev->base.dev);

	init_completion(&ptdev->unplug.done);
	ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock);
	if (ret)
		return ret;

	ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock);
	if (ret)
		return ret;

#ifdef CONFIG_DEBUG_FS
	drmm_mutex_init(&ptdev->base, &ptdev->gems.lock);
	INIT_LIST_HEAD(&ptdev->gems.node);
#endif

	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
	p = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!p)
		return -ENOMEM;

	ptdev->pm.dummy_latest_flush = p;
	dummy_page_virt = page_address(p);
	ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page,
				       ptdev->pm.dummy_latest_flush);
	if (ret)
		return ret;

	/*
	 * Set the dummy page holding the latest flush to 1. This will cause the
	 * flush to avoided as we know it isn't necessary if the submission
	 * happens while the dummy page is mapped. Zero cannot be used because
	 * that means 'always flush'.
	 */
	*dummy_page_virt = 1;

	INIT_WORK(&ptdev->reset.work, panthor_device_reset_work);
	ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0);
	if (!ptdev->reset.wq)
		return -ENOMEM;

	ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL);
	if (ret)
		return ret;

	ret = panthor_clk_init(ptdev);
	if (ret)
		return ret;

	ret = panthor_init_power(ptdev->base.dev);
	if (ret < 0) {
		drm_err(&ptdev->base, "init power domains failed, ret=%d", ret);
		return ret;
	}

	ret = panthor_devfreq_init(ptdev);
	if (ret)
		return ret;

	ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev),
							      0, &res);
	if (IS_ERR(ptdev->iomem))
		return PTR_ERR(ptdev->iomem);

	ptdev->phys_addr = res->start;

	ret = devm_pm_runtime_enable(ptdev->base.dev);
	if (ret)
		return ret;

	ret = pm_runtime_resume_and_get(ptdev->base.dev);
	if (ret)
		return ret;

	/* If PM is disabled, we need to call panthor_device_resume() manually. */
	if (!IS_ENABLED(CONFIG_PM)) {
		ret = panthor_device_resume(ptdev->base.dev);
		if (ret)
			return ret;
	}

	ret = panthor_hw_init(ptdev);
	if (ret)
		goto err_rpm_put;

	ret = panthor_pwr_init(ptdev);
	if (ret)
		goto err_rpm_put;

	ret = panthor_gpu_init(ptdev);
	if (ret)
		goto err_unplug_pwr;

	ret = panthor_gpu_coherency_init(ptdev);
	if (ret)
		goto err_unplug_gpu;

	ret = panthor_mmu_init(ptdev);
	if (ret)
		goto err_unplug_gpu;

	ret = panthor_fw_init(ptdev);
	if (ret)
		goto err_unplug_mmu;

	ret = panthor_sched_init(ptdev);
	if (ret)
		goto err_unplug_fw;

	panthor_gem_init(ptdev);

	/* ~3 frames */
	pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50);
	pm_runtime_use_autosuspend(ptdev->base.dev);

	ret = drm_dev_register(&ptdev->base, 0);
	if (ret)
		goto err_disable_autosuspend;

	pm_runtime_put_autosuspend(ptdev->base.dev);
	return 0;

err_disable_autosuspend:
	pm_runtime_dont_use_autosuspend(ptdev->base.dev);
	panthor_sched_unplug(ptdev);

err_unplug_fw:
	panthor_fw_unplug(ptdev);

err_unplug_mmu:
	panthor_mmu_unplug(ptdev);

err_unplug_gpu:
	panthor_gpu_unplug(ptdev);

err_unplug_pwr:
	panthor_pwr_unplug(ptdev);

err_rpm_put:
	pm_runtime_put_sync_suspend(ptdev->base.dev);
	return ret;
}

#define PANTHOR_EXCEPTION(id) \
	[DRM_PANTHOR_EXCEPTION_ ## id] = { \
		.name = #id, \
	}

struct panthor_exception_info {
	const char *name;
};

static const struct panthor_exception_info panthor_exception_infos[] = {
	PANTHOR_EXCEPTION(OK),
	PANTHOR_EXCEPTION(TERMINATED),
	PANTHOR_EXCEPTION(KABOOM),
	PANTHOR_EXCEPTION(EUREKA),
	PANTHOR_EXCEPTION(ACTIVE),
	PANTHOR_EXCEPTION(CS_RES_TERM),
	PANTHOR_EXCEPTION(CS_CONFIG_FAULT),
	PANTHOR_EXCEPTION(CS_UNRECOVERABLE),
	PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT),
	PANTHOR_EXCEPTION(CS_BUS_FAULT),
	PANTHOR_EXCEPTION(CS_INSTR_INVALID),
	PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW),
	PANTHOR_EXCEPTION(CS_INHERIT_FAULT),
	PANTHOR_EXCEPTION(INSTR_INVALID_PC),
	PANTHOR_EXCEPTION(INSTR_INVALID_ENC),
	PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT),
	PANTHOR_EXCEPTION(DATA_INVALID_FAULT),
	PANTHOR_EXCEPTION(TILE_RANGE_FAULT),
	PANTHOR_EXCEPTION(ADDR_RANGE_FAULT),
	PANTHOR_EXCEPTION(IMPRECISE_FAULT),
	PANTHOR_EXCEPTION(OOM),
	PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR),
	PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT),
	PANTHOR_EXCEPTION(GPU_BUS_FAULT),
	PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT),
	PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT),
	PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT),
	PANTHOR_EXCEPTION(TRANSLATION_FAULT_0),
	PANTHOR_EXCEPTION(TRANSLATION_FAULT_1),
	PANTHOR_EXCEPTION(TRANSLATION_FAULT_2),
	PANTHOR_EXCEPTION(TRANSLATION_FAULT_3),
	PANTHOR_EXCEPTION(TRANSLATION_FAULT_4),
	PANTHOR_EXCEPTION(PERM_FAULT_0),
	PANTHOR_EXCEPTION(PERM_FAULT_1),
	PANTHOR_EXCEPTION(PERM_FAULT_2),
	PANTHOR_EXCEPTION(PERM_FAULT_3),
	PANTHOR_EXCEPTION(ACCESS_FLAG_1),
	PANTHOR_EXCEPTION(ACCESS_FLAG_2),
	PANTHOR_EXCEPTION(ACCESS_FLAG_3),
	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN),
	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0),
	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1),
	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2),
	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3),
};

const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code)
{
	if (exception_code >= ARRAY_SIZE(panthor_exception_infos) ||
	    !panthor_exception_infos[exception_code].name)
		return "Unknown exception type";

	return panthor_exception_infos[exception_code].name;
}

static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf)
{
	struct vm_area_struct *vma = vmf->vma;
	struct panthor_device *ptdev = vma->vm_private_data;
	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
	unsigned long pfn;
	pgprot_t pgprot;
	vm_fault_t ret;
	bool active;
	int cookie;

	if (!drm_dev_enter(&ptdev->base, &cookie))
		return VM_FAULT_SIGBUS;

	mutex_lock(&ptdev->pm.mmio_lock);
	active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE;

	switch (offset) {
	case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
		if (active)
			pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID);
		else
			pfn = page_to_pfn(ptdev->pm.dummy_latest_flush);
		break;

	default:
		ret = VM_FAULT_SIGBUS;
		goto out_unlock;
	}

	pgprot = vma->vm_page_prot;
	if (active)
		pgprot = pgprot_noncached(pgprot);

	ret = vmf_insert_pfn_prot(vma, vmf->address, pfn, pgprot);

out_unlock:
	mutex_unlock(&ptdev->pm.mmio_lock);
	drm_dev_exit(cookie);
	return ret;
}

static const struct vm_operations_struct panthor_mmio_vm_ops = {
	.fault = panthor_mmio_vm_fault,
};

int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma)
{
	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;

	if ((vma->vm_flags & VM_SHARED) == 0)
		return -EINVAL;

	switch (offset) {
	case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
		if (vma->vm_end - vma->vm_start != PAGE_SIZE ||
		    (vma->vm_flags & (VM_WRITE | VM_EXEC)))
			return -EINVAL;
		vm_flags_clear(vma, VM_MAYWRITE);

		break;

	default:
		return -EINVAL;
	}

	/* Defer actual mapping to the fault handler. */
	vma->vm_private_data = ptdev;
	vma->vm_ops = &panthor_mmio_vm_ops;
	vm_flags_set(vma,
		     VM_IO | VM_DONTCOPY | VM_DONTEXPAND |
		     VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
	return 0;
}

static int panthor_device_resume_hw_components(struct panthor_device *ptdev)
{
	int ret;

	panthor_pwr_resume(ptdev);
	panthor_gpu_resume(ptdev);
	panthor_mmu_resume(ptdev);

	ret = panthor_fw_resume(ptdev);
	if (!ret)
		return 0;

	panthor_mmu_suspend(ptdev);
	panthor_gpu_suspend(ptdev);
	panthor_pwr_suspend(ptdev);
	return ret;
}

int panthor_device_resume(struct device *dev)
{
	struct panthor_device *ptdev = dev_get_drvdata(dev);
	int ret, cookie;

	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED)
		return -EINVAL;

	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_RESUMING);

	ret = clk_prepare_enable(ptdev->clks.core);
	if (ret)
		goto err_set_suspended;

	ret = clk_prepare_enable(ptdev->clks.stacks);
	if (ret)
		goto err_disable_core_clk;

	ret = clk_prepare_enable(ptdev->clks.coregroup);
	if (ret)
		goto err_disable_stacks_clk;

	panthor_devfreq_resume(ptdev);

	if (panthor_device_is_initialized(ptdev) &&
	    drm_dev_enter(&ptdev->base, &cookie)) {
		/* If there was a reset pending at the time we suspended the
		 * device, we force a slow reset.
		 */
		if (atomic_read(&ptdev->reset.pending)) {
			ptdev->reset.fast = false;
			atomic_set(&ptdev->reset.pending, 0);
		}

		ret = panthor_device_resume_hw_components(ptdev);
		if (ret && ptdev->reset.fast) {
			drm_err(&ptdev->base, "Fast reset failed, trying a slow reset");
			ptdev->reset.fast = false;
			ret = panthor_device_resume_hw_components(ptdev);
		}

		if (!ret)
			panthor_sched_resume(ptdev);

		drm_dev_exit(cookie);

		if (ret)
			goto err_suspend_devfreq;
	}

	/* Clear all IOMEM mappings pointing to this device after we've
	 * resumed. This way the fake mappings pointing to the dummy pages
	 * are removed and the real iomem mapping will be restored on next
	 * access.
	 */
	mutex_lock(&ptdev->pm.mmio_lock);
	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);
	mutex_unlock(&ptdev->pm.mmio_lock);
	return 0;

err_suspend_devfreq:
	panthor_devfreq_suspend(ptdev);
	clk_disable_unprepare(ptdev->clks.coregroup);

err_disable_stacks_clk:
	clk_disable_unprepare(ptdev->clks.stacks);

err_disable_core_clk:
	clk_disable_unprepare(ptdev->clks.core);

err_set_suspended:
	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
	atomic_set(&ptdev->pm.recovery_needed, 1);
	return ret;
}

int panthor_device_suspend(struct device *dev)
{
	struct panthor_device *ptdev = dev_get_drvdata(dev);
	int cookie;

	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
		return -EINVAL;

	/* Clear all IOMEM mappings pointing to this device before we
	 * shutdown the power-domain and clocks. Failing to do that results
	 * in external aborts when the process accesses the iomem region.
	 * We change the state and call unmap_mapping_range() with the
	 * mmio_lock held to make sure the vm_fault handler won't set up
	 * invalid mappings.
	 */
	mutex_lock(&ptdev->pm.mmio_lock);
	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDING);
	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
	mutex_unlock(&ptdev->pm.mmio_lock);

	if (panthor_device_is_initialized(ptdev) &&
	    drm_dev_enter(&ptdev->base, &cookie)) {
		cancel_work_sync(&ptdev->reset.work);

		/* We prepare everything as if we were resetting the GPU.
		 * The end of the reset will happen in the resume path though.
		 */
		panthor_sched_suspend(ptdev);
		panthor_fw_suspend(ptdev);
		panthor_mmu_suspend(ptdev);
		panthor_gpu_suspend(ptdev);
		panthor_pwr_suspend(ptdev);
		drm_dev_exit(cookie);
	}

	panthor_devfreq_suspend(ptdev);

	clk_disable_unprepare(ptdev->clks.coregroup);
	clk_disable_unprepare(ptdev->clks.stacks);
	clk_disable_unprepare(ptdev->clks.core);
	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
	return 0;
}