drivers/edgetpu/edgetpu-core.c - kernel/google-modules/edgetpu/abrolhos - Gitiles

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Common support functions for Edge TPU ML accelerator host-side ops.
  *
  * Copyright (C) 2019 Google, Inc.
  */

 #include <asm/current.h>
 #include <asm/page.h>
 #include <linux/atomic.h>
 #include <linux/compiler.h>
 #include <linux/cred.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
 #include <linux/uidgid.h>

 #include "edgetpu-config.h"
 #include "edgetpu-debug-dump.h"
 #include "edgetpu-device-group.h"
 #include "edgetpu-dram.h"
 #include "edgetpu-internal.h"
 #include "edgetpu-kci.h"
 #include "edgetpu-mailbox.h"
 #include "edgetpu-mcp.h"
 #include "edgetpu-mmu.h"
 #include "edgetpu-sw-watchdog.h"
 #include "edgetpu-telemetry.h"
 #include "edgetpu-usage-stats.h"
 #include "edgetpu-wakelock.h"
 #include "edgetpu.h"

 /* Bits higher than VMA_TYPE_WIDTH are used to carry type specific data, e.g., core id. */
 #define VMA_TYPE_WIDTH 16
 #define VMA_TYPE(x) ((x) & (BIT_MASK(VMA_TYPE_WIDTH) - 1))
 #define VMA_DATA_GET(x) ((x) >> VMA_TYPE_WIDTH)
 #define VMA_DATA_SET(x, y) (VMA_TYPE(x) | ((y) << VMA_TYPE_WIDTH))

 enum edgetpu_vma_type {
 	VMA_INVALID,

 	VMA_FULL_CSR,
 	VMA_VII_CSR,
 	VMA_VII_CMDQ,
 	VMA_VII_RESPQ,
 	VMA_EXT_CSR,
 	VMA_EXT_CMDQ,
 	VMA_EXT_RESPQ,
 	/* For VMA_LOG and VMA_TRACE, core id is stored in bits higher than VMA_TYPE_WIDTH. */
 	VMA_LOG,
 	VMA_TRACE,
 };

 /* type that combines enum edgetpu_vma_type and data in higher bits. */
 typedef u32 edgetpu_vma_flags_t;

 /* structure to be set to vma->vm_private_data on mmap */
 struct edgetpu_vma_private {
 	struct edgetpu_client *client;
 	edgetpu_vma_flags_t flag;
 	/*
 	 * vm_private_data is copied when a VMA is split, using this reference
 	 * counter to know when should this object be freed.
 	 */
 	refcount_t count;
 };

 static atomic_t single_dev_count = ATOMIC_INIT(-1);

 static int edgetpu_mmap_full_csr(struct edgetpu_client *client,
 				 struct vm_area_struct *vma)
 {
 	int ret;
 	ulong phys_base, vma_size, map_size;

 	if (!uid_eq(current_euid(), GLOBAL_ROOT_UID))
 		return -EPERM;
 	vma_size = vma->vm_end - vma->vm_start;
 	map_size = min_t(ulong, vma_size, client->etdev->regs.size);
 	phys_base = client->etdev->regs.phys;
 	ret = io_remap_pfn_range(vma, vma->vm_start, phys_base >> PAGE_SHIFT,
 				 map_size, vma->vm_page_prot);
 	if (ret)
 		etdev_dbg(client->etdev,
 			  "Error remapping PFN range: %d\n", ret);
 	return ret;
 }

 static edgetpu_vma_flags_t mmap_vma_flag(unsigned long pgoff)
 {
 	const unsigned long off = pgoff << PAGE_SHIFT;

 	switch (off) {
 	case 0:
 		return VMA_FULL_CSR;
 	case EDGETPU_MMAP_CSR_OFFSET:
 		return VMA_VII_CSR;
 	case EDGETPU_MMAP_CMD_QUEUE_OFFSET:
 		return VMA_VII_CMDQ;
 	case EDGETPU_MMAP_RESP_QUEUE_OFFSET:
 		return VMA_VII_RESPQ;
 #ifdef EDGETPU_FEATURE_INTEROP
 	case EDGETPU_MMAP_EXT_CSR_OFFSET:
 		return VMA_EXT_CSR;
 	case EDGETPU_MMAP_EXT_CMD_QUEUE_OFFSET:
 		return VMA_EXT_CMDQ;
 	case EDGETPU_MMAP_EXT_RESP_QUEUE_OFFSET:
 		return VMA_EXT_RESPQ;
 #endif /* EDGETPU_FEATURE_INTEROP */
 	case EDGETPU_MMAP_LOG_BUFFER_OFFSET:
 		return VMA_DATA_SET(VMA_LOG, 0);
 	case EDGETPU_MMAP_TRACE_BUFFER_OFFSET:
 		return VMA_DATA_SET(VMA_TRACE, 0);
 #if EDGETPU_NUM_CORES > 1
 	case EDGETPU_MMAP_LOG1_BUFFER_OFFSET:
 		return VMA_DATA_SET(VMA_LOG, 1);
 	case EDGETPU_MMAP_TRACE1_BUFFER_OFFSET:
 		return VMA_DATA_SET(VMA_TRACE, 1);
 #endif /* EDGETPU_NUM_CORES > 1 */
 	default:
 		return VMA_INVALID;
 	}
 }

 /*
  * Returns the wakelock event by VMA type. Returns EDGETPU_WAKELOCK_EVENT_END
  * if the type does not correspond to a wakelock event.
  */
 static enum edgetpu_wakelock_event
 vma_type_to_wakelock_event(enum edgetpu_vma_type type)
 {
 	switch (type) {
 	case VMA_FULL_CSR:
 		return EDGETPU_WAKELOCK_EVENT_FULL_CSR;
 	case VMA_VII_CSR:
 		return EDGETPU_WAKELOCK_EVENT_MBOX_CSR;
 	case VMA_VII_CMDQ:
 		return EDGETPU_WAKELOCK_EVENT_CMD_QUEUE;
 	case VMA_VII_RESPQ:
 		return EDGETPU_WAKELOCK_EVENT_RESP_QUEUE;
 	case VMA_EXT_CSR:
 		return EDGETPU_WAKELOCK_EVENT_MBOX_CSR;
 	case VMA_EXT_CMDQ:
 		return EDGETPU_WAKELOCK_EVENT_CMD_QUEUE;
 	case VMA_EXT_RESPQ:
 		return EDGETPU_WAKELOCK_EVENT_RESP_QUEUE;
 	default:
 		return EDGETPU_WAKELOCK_EVENT_END;
 	}
 }

 static struct edgetpu_vma_private *
 edgetpu_vma_private_alloc(struct edgetpu_client *client,
 			  edgetpu_vma_flags_t flag)
 {
 	struct edgetpu_vma_private *pvt = kmalloc(sizeof(*pvt), GFP_KERNEL);

 	if (!pvt)
 		return NULL;
 	pvt->client = edgetpu_client_get(client);
 	pvt->flag = flag;
 	refcount_set(&pvt->count, 1);

 	return pvt;
 }

 static void edgetpu_vma_private_get(struct edgetpu_vma_private *pvt)
 {
 	WARN_ON_ONCE(!refcount_inc_not_zero(&pvt->count));
 }

 static void edgetpu_vma_private_put(struct edgetpu_vma_private *pvt)
 {
 	if (!pvt)
 		return;
 	if (refcount_dec_and_test(&pvt->count)) {
 		edgetpu_client_put(pvt->client);
 		kfree(pvt);
 	}
 }

 static void edgetpu_vma_open(struct vm_area_struct *vma)
 {
 	struct edgetpu_vma_private *pvt = vma->vm_private_data;
 	enum edgetpu_wakelock_event evt;
 	struct edgetpu_client *client;
 	struct edgetpu_dev *etdev;
 	enum edgetpu_vma_type type = VMA_TYPE(pvt->flag);

 	edgetpu_vma_private_get(pvt);
 	client = pvt->client;
 	etdev = client->etdev;

 	evt = vma_type_to_wakelock_event(type);
 	if (evt != EDGETPU_WAKELOCK_EVENT_END)
 		edgetpu_wakelock_inc_event(client->wakelock, evt);

 	/* handle telemetry types */
 	switch (type) {
 	case VMA_LOG:
 		edgetpu_telemetry_inc_mmap_count(etdev, EDGETPU_TELEMETRY_LOG,
 						 VMA_DATA_GET(pvt->flag));
 		break;
 	case VMA_TRACE:
 		edgetpu_telemetry_inc_mmap_count(etdev, EDGETPU_TELEMETRY_TRACE,
 						 VMA_DATA_GET(pvt->flag));
 		break;
 	default:
 		break;
 	}
 }

 /* Records previously mmapped addresses were unmapped. */
 static void edgetpu_vma_close(struct vm_area_struct *vma)
 {
 	struct edgetpu_vma_private *pvt = vma->vm_private_data;
 	struct edgetpu_client *client = pvt->client;
 	enum edgetpu_vma_type type = VMA_TYPE(pvt->flag);
 	enum edgetpu_wakelock_event evt = vma_type_to_wakelock_event(type);
 	struct edgetpu_dev *etdev = client->etdev;

 	if (evt != EDGETPU_WAKELOCK_EVENT_END)
 		edgetpu_wakelock_dec_event(client->wakelock, evt);

 	/* handle telemetry types */
 	switch (type) {
 	case VMA_LOG:
 		edgetpu_telemetry_dec_mmap_count(etdev, EDGETPU_TELEMETRY_LOG,
 						 VMA_DATA_GET(pvt->flag));
 		break;
 	case VMA_TRACE:
 		edgetpu_telemetry_dec_mmap_count(etdev, EDGETPU_TELEMETRY_TRACE,
 						 VMA_DATA_GET(pvt->flag));
 		break;
 	default:
 		break;
 	}

 	edgetpu_vma_private_put(pvt);
 }

 static const struct vm_operations_struct edgetpu_vma_ops = {
 	.open = edgetpu_vma_open,
 	.close = edgetpu_vma_close,
 };

 /* Map exported device CSRs or queue into user space. */
 int edgetpu_mmap(struct edgetpu_client *client, struct vm_area_struct *vma)
 {
 	int ret = 0;
 	edgetpu_vma_flags_t flag;
 	enum edgetpu_vma_type type;
 	enum edgetpu_wakelock_event evt;
 	struct edgetpu_vma_private *pvt;

 	if (vma->vm_start & ~PAGE_MASK) {
 		etdev_dbg(client->etdev,
 			  "Base address not page-aligned: %#lx\n",
 			  vma->vm_start);
 		return -EINVAL;
 	}

 	etdev_dbg(client->etdev, "%s: mmap pgoff = %#lX\n", __func__,
 		  vma->vm_pgoff);

 	flag = mmap_vma_flag(vma->vm_pgoff);
 	type = VMA_TYPE(flag);
 	if (type == VMA_INVALID)
 		return -EINVAL;
 	pvt = edgetpu_vma_private_alloc(client, flag);
 	if (!pvt)
 		return -ENOMEM;

 	/* Mark the VMA's pages as uncacheable. */
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	/* Disable fancy things to ensure our event counters work. */
 	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;

 	/* map all CSRs for debug purpose */
 	if (type == VMA_FULL_CSR) {
 		evt = EDGETPU_WAKELOCK_EVENT_FULL_CSR;
 		if (edgetpu_wakelock_inc_event(client->wakelock, evt)) {
 			ret = edgetpu_mmap_full_csr(client, vma);
 			if (ret)
 				goto out_dec_evt;
 		} else {
 			ret = -EAGAIN;
 		}
 		goto out_set_op;
 	}

 	/* Allow mapping log and telemetry buffers without a group */
 	if (type == VMA_LOG) {
 		ret = edgetpu_mmap_telemetry_buffer(client->etdev, EDGETPU_TELEMETRY_LOG, vma,
 						    VMA_DATA_GET(flag));
 		goto out_set_op;
 	}
 	if (type == VMA_TRACE) {
 		ret = edgetpu_mmap_telemetry_buffer(client->etdev, EDGETPU_TELEMETRY_TRACE, vma,
 						    VMA_DATA_GET(flag));
 		goto out_set_op;
 	}

 	evt = vma_type_to_wakelock_event(type);
 	/*
 	 * VMA_TYPE(@flag) should always correspond to a valid event since we handled
 	 * telemetry mmaps above, still check evt != END in case new types are
 	 * added in the future.
 	 */
 	if (unlikely(evt == EDGETPU_WAKELOCK_EVENT_END)) {
 		ret = -EINVAL;
 		goto err_release_pvt;
 	}
 	if (!edgetpu_wakelock_inc_event(client->wakelock, evt)) {
 		ret = -EAGAIN;
 		goto err_release_pvt;
 	}

 	mutex_lock(&client->group_lock);
 	if (!client->group) {
 		ret = -EINVAL;
 		goto out_unlock;
 	}
 	switch (type) {
 	case VMA_VII_CSR:
 		ret = edgetpu_mmap_csr(client->group, vma, false);
 		break;
 	case VMA_VII_CMDQ:
 		ret = edgetpu_mmap_queue(client->group, MAILBOX_CMD_QUEUE, vma, false);
 		break;
 	case VMA_VII_RESPQ:
 		ret = edgetpu_mmap_queue(client->group, MAILBOX_RESP_QUEUE, vma, false);
 		break;
 	case VMA_EXT_CSR:
 		ret = edgetpu_mmap_csr(client->group, vma, true);
 		break;
 	case VMA_EXT_CMDQ:
 		ret = edgetpu_mmap_queue(client->group, MAILBOX_CMD_QUEUE, vma, true);
 		break;
 	case VMA_EXT_RESPQ:
 		ret = edgetpu_mmap_queue(client->group, MAILBOX_RESP_QUEUE, vma, true);
 		break;
 	default: /* to appease compiler */
 		break;
 	}
 out_unlock:
 	mutex_unlock(&client->group_lock);
 out_dec_evt:
 	if (ret)
 		edgetpu_wakelock_dec_event(client->wakelock, evt);
 out_set_op:
 	if (!ret) {
 		vma->vm_private_data = pvt;
 		vma->vm_ops = &edgetpu_vma_ops;
 		return 0;
 	}
 err_release_pvt:
 	edgetpu_vma_private_put(pvt);
 	return ret;
 }

 static struct edgetpu_mailbox_manager_desc mailbox_manager_desc = {
 	.num_mailbox = EDGETPU_NUM_MAILBOXES,
 	.num_vii_mailbox = EDGETPU_NUM_VII_MAILBOXES,
 	.num_p2p_mailbox = EDGETPU_NUM_P2P_MAILBOXES,
 	.num_ext_mailbox = EDGETPU_NUM_EXT_MAILBOXES,
 	.get_context_csr_base = edgetpu_mailbox_get_context_csr_base,
 	.get_cmd_queue_csr_base = edgetpu_mailbox_get_cmd_queue_csr_base,
 	.get_resp_queue_csr_base = edgetpu_mailbox_get_resp_queue_csr_base,
 };

 int edgetpu_get_state_errno_locked(struct edgetpu_dev *etdev)
 {
 	switch (etdev->state) {
 	case ETDEV_STATE_BAD:
 		return -ENODEV;
 	case ETDEV_STATE_FWLOADING:
 		return -EAGAIN;
 	case ETDEV_STATE_NOFW:
 		return -EINVAL;
 	default:
 		break;
 	}
 	return 0;
 }

 int edgetpu_device_add(struct edgetpu_dev *etdev,
 		       const struct edgetpu_mapped_resource *regs,
 		       const struct edgetpu_iface_params *iface_params,
 		       uint num_ifaces)
 {
 	int ret;

 	etdev->regs = *regs;

 	etdev->etiface = devm_kzalloc(
 		etdev->dev, sizeof(*etdev->etiface) * num_ifaces, GFP_KERNEL);

 	if (!etdev->etiface) {
 		dev_err(etdev->dev,
 			"Failed to allocate memory for interfaces\n");
 		return -ENOMEM;
 	}

 	/* mcp_id and mcp_die_index fields set by caller */
 	if (etdev->mcp_id < 0) {
 		uint ordinal_id = atomic_add_return(1, &single_dev_count);

 		if (!ordinal_id)
 			snprintf(etdev->dev_name, EDGETPU_DEVICE_NAME_MAX, "%s",
 				 DRIVER_NAME);
 		else
 			snprintf(etdev->dev_name, EDGETPU_DEVICE_NAME_MAX,
 				 "%s.%u", DRIVER_NAME, ordinal_id);
 	} else {
 		snprintf(etdev->dev_name, EDGETPU_DEVICE_NAME_MAX,
 			 "%s.%u.%u", DRIVER_NAME, etdev->mcp_id,
 			 etdev->mcp_die_index);
 	}

 	mutex_init(&etdev->groups_lock);
 	INIT_LIST_HEAD(&etdev->groups);
 	etdev->n_groups = 0;
 	etdev->group_join_lockout = false;
 	mutex_init(&etdev->clients_lock);
 	INIT_LIST_HEAD(&etdev->clients);
 	etdev->vcid_pool = (1u << EDGETPU_NUM_VCIDS) - 1;
 	mutex_init(&etdev->state_lock);
 	etdev->state = ETDEV_STATE_NOFW;
 	etdev->freq_count = 0;
 	mutex_init(&etdev->freq_lock);

 	ret = edgetpu_fs_add(etdev, iface_params, num_ifaces);
 	if (ret) {
 		dev_err(etdev->dev, "%s: edgetpu_fs_add returns %d\n", etdev->dev_name, ret);
 		goto remove_dev;
 	}

 	etdev->mailbox_manager =
 		edgetpu_mailbox_create_mgr(etdev, &mailbox_manager_desc);
 	if (IS_ERR(etdev->mailbox_manager)) {
 		ret = PTR_ERR(etdev->mailbox_manager);
 		dev_err(etdev->dev,
 			"%s: edgetpu_mailbox_create_mgr returns %d\n",
 			etdev->dev_name, ret);
 		goto remove_dev;
 	}
 	ret = edgetpu_chip_setup_mmu(etdev);
 	if (ret)
 		goto remove_dev;

 	edgetpu_usage_stats_init(etdev);

 	etdev->kci = devm_kzalloc(etdev->dev, sizeof(*etdev->kci), GFP_KERNEL);
 	if (!etdev->kci) {
 		ret = -ENOMEM;
 		goto remove_usage_stats;
 	}

 	etdev->telemetry =
 		devm_kcalloc(etdev->dev, etdev->num_cores, sizeof(*etdev->telemetry), GFP_KERNEL);
 	if (!etdev->telemetry) {
 		ret = -ENOMEM;
 		goto remove_usage_stats;
 	}

 	ret = edgetpu_kci_init(etdev->mailbox_manager, etdev->kci);
 	if (ret) {
 		etdev_err(etdev, "edgetpu_kci_init returns %d\n", ret);
 		goto remove_usage_stats;
 	}

 	ret = edgetpu_device_dram_init(etdev);
 	if (ret) {
 		etdev_err(etdev,
 			  "failed to init on-device DRAM management: %d\n",
 			  ret);
 		goto remove_kci;
 	}

 	ret = edgetpu_debug_dump_init(etdev);
 	if (ret)
 		etdev_warn(etdev, "debug dump init fail: %d", ret);

 	edgetpu_chip_init(etdev);
 	return 0;

 remove_kci:
 	/* releases the resources of KCI */
 	edgetpu_mailbox_remove_all(etdev->mailbox_manager);
 remove_usage_stats:
 	edgetpu_usage_stats_exit(etdev);
 	edgetpu_chip_remove_mmu(etdev);
 remove_dev:
 	edgetpu_mark_probe_fail(etdev);
 	edgetpu_fs_remove(etdev);
 	return ret;
 }

 void edgetpu_device_remove(struct edgetpu_dev *etdev)
 {
 	edgetpu_chip_exit(etdev);
 	edgetpu_debug_dump_exit(etdev);
 	edgetpu_device_dram_exit(etdev);
 	edgetpu_mailbox_remove_all(etdev->mailbox_manager);
 	edgetpu_usage_stats_exit(etdev);
 	edgetpu_chip_remove_mmu(etdev);
 	edgetpu_fs_remove(etdev);
 }

 struct edgetpu_client *edgetpu_client_add(struct edgetpu_dev_iface *etiface)
 {
 	struct edgetpu_client *client;
 	struct edgetpu_list_device_client *l = kmalloc(sizeof(*l), GFP_KERNEL);
 	struct edgetpu_dev *etdev = etiface->etdev;

 	if (!l)
 		return ERR_PTR(-ENOMEM);
 	client = kzalloc(sizeof(*client), GFP_KERNEL);
 	if (!client) {
 		kfree(l);
 		return ERR_PTR(-ENOMEM);
 	}
 	client->wakelock = edgetpu_wakelock_alloc(etdev);
 	if (!client->wakelock) {
 		kfree(client);
 		kfree(l);
 		return ERR_PTR(-ENOMEM);
 	}

 	client->pid = current->pid;
 	client->tgid = current->tgid;
 	client->etdev = etdev;
 	client->etiface = etiface;
 	mutex_init(&client->group_lock);
 	/* equivalent to edgetpu_client_get() */
 	refcount_set(&client->count, 1);
 	client->perdie_events = 0;
 	mutex_lock(&etdev->clients_lock);
 	l->client = client;
 	list_add_tail(&l->list, &etdev->clients);
 	mutex_unlock(&etdev->clients_lock);
 	return client;
 }

 struct edgetpu_client *edgetpu_client_get(struct edgetpu_client *client)
 {
 	WARN_ON_ONCE(!refcount_inc_not_zero(&client->count));
 	return client;
 }

 void edgetpu_client_put(struct edgetpu_client *client)
 {
 	if (!client)
 		return;
 	if (refcount_dec_and_test(&client->count)) {
 		edgetpu_wakelock_free(client->wakelock);
 		kfree(client);
 	}
 }

 void edgetpu_client_remove(struct edgetpu_client *client)
 {
 	struct edgetpu_dev *etdev;
 	struct edgetpu_list_device_client *lc;

 	if (IS_ERR_OR_NULL(client))
 		return;
 	etdev = client->etdev;
 	mutex_lock(&etdev->clients_lock);
 	/* remove the client from the device list */
 	for_each_list_device_client(etdev, lc) {
 		if (lc->client == client) {
 			list_del(&lc->list);
 			kfree(lc);
 			break;
 		}
 	}
 	mutex_unlock(&etdev->clients_lock);
 	/*
 	 * A quick check without holding client->group_lock.
 	 *
 	 * If client doesn't belong to a group then we are fine to not remove
 	 * from groups.
 	 *
 	 * If there is a race that the client belongs to a group but is removing
 	 * by another process - this will be detected by the check with holding
 	 * client->group_lock later.
 	 */
 	if (client->group)
 		edgetpu_device_group_leave(client);
 	/* invoke chip-dependent removal handler before releasing resources */
 	edgetpu_chip_client_remove(client);

 	/* Clean up all the per die event fds registered by the client */
 	if (client->perdie_events &
 	    1 << perdie_event_id_to_num(EDGETPU_PERDIE_EVENT_LOGS_AVAILABLE))
 		edgetpu_telemetry_unset_event(etdev, EDGETPU_TELEMETRY_LOG);
 	if (client->perdie_events &
 	    1 << perdie_event_id_to_num(EDGETPU_PERDIE_EVENT_TRACES_AVAILABLE))
 		edgetpu_telemetry_unset_event(etdev, EDGETPU_TELEMETRY_TRACE);

 	edgetpu_client_put(client);
 }

 int edgetpu_register_irq(struct edgetpu_dev *etdev, int irq)
 {
 	int ret;

 	ret = devm_request_irq(etdev->dev, irq, edgetpu_chip_irq_handler,
 			       IRQF_ONESHOT, etdev->dev_name, etdev);
 	if (ret)
 		dev_err(etdev->dev, "%s: failed to request irq %d: %d\n",
 			etdev->dev_name, irq, ret);
 	return ret;
 }

 void edgetpu_unregister_irq(struct edgetpu_dev *etdev, int irq)
 {
 	devm_free_irq(etdev->dev, irq, etdev);
 }

 int edgetpu_alloc_coherent(struct edgetpu_dev *etdev, size_t size,
 			   struct edgetpu_coherent_mem *mem,
 			   enum edgetpu_context_id context_id)
 {
 	const u32 flags = EDGETPU_MMU_DIE | EDGETPU_MMU_32 | EDGETPU_MMU_HOST |
 			  EDGETPU_MMU_COHERENT;

 	mem->vaddr = dma_alloc_coherent(etdev->dev, size, &mem->dma_addr,
 					GFP_KERNEL);
 	if (!mem->vaddr)
 		return -ENOMEM;
 	edgetpu_x86_coherent_mem_init(mem);
 	mem->tpu_addr =
 		edgetpu_mmu_tpu_map(etdev, mem->dma_addr, size,
 				    DMA_BIDIRECTIONAL, context_id, flags);
 	if (!mem->tpu_addr) {
 		dma_free_coherent(etdev->dev, size, mem->vaddr, mem->dma_addr);
 		mem->vaddr = NULL;
 		return -EINVAL;
 	}
 	mem->size = size;
 	return 0;
 }

 void edgetpu_free_coherent(struct edgetpu_dev *etdev,
 			   struct edgetpu_coherent_mem *mem,
 			   enum edgetpu_context_id context_id)
 {
 	edgetpu_mmu_tpu_unmap(etdev, mem->tpu_addr, mem->size, context_id);
 	edgetpu_x86_coherent_mem_set_wb(mem);
 	dma_free_coherent(etdev->dev, mem->size, mem->vaddr, mem->dma_addr);
 	mem->vaddr = NULL;
 }

 void edgetpu_handle_firmware_crash(struct edgetpu_dev *etdev,
 				   enum edgetpu_fw_crash_type crash_type)
 {
 	if (crash_type == EDGETPU_FW_CRASH_UNRECOV_FAULT) {
 		etdev_err(etdev, "firmware unrecoverable crash");
 		etdev->firmware_crash_count++;
 		edgetpu_fatal_error_notify(etdev, EDGETPU_ERROR_FW_CRASH);
 		/* Restart firmware without chip reset */
 		edgetpu_watchdog_bite(etdev, false);
 	} else {
 		etdev_err(etdev, "firmware non-fatal crash event: %u",
 			  crash_type);
 	}
 }

 int __init edgetpu_init(void)
 {
 	int ret;

 	ret = edgetpu_fs_init();
 	if (ret)
 		return ret;
 	edgetpu_mcp_init();
 	return 0;
 }

 void __exit edgetpu_exit(void)
 {
 	edgetpu_mcp_exit();
 	edgetpu_fs_exit();
 }
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Common support functions for Edge TPU ML accelerator host-side ops.
	*
	* Copyright (C) 2019 Google, Inc.
	*/

	#include <asm/current.h>
	#include <asm/page.h>
	#include <linux/atomic.h>
	#include <linux/compiler.h>
	#include <linux/cred.h>
	#include <linux/delay.h>
	#include <linux/device.h>
	#include <linux/dma-mapping.h>
	#include <linux/interrupt.h>
	#include <linux/kernel.h>
	#include <linux/mm.h>
	#include <linux/mutex.h>
	#include <linux/slab.h>
	#include <linux/spinlock.h>
	#include <linux/types.h>
	#include <linux/uidgid.h>

	#include "edgetpu-config.h"
	#include "edgetpu-debug-dump.h"
	#include "edgetpu-device-group.h"
	#include "edgetpu-dram.h"
	#include "edgetpu-internal.h"
	#include "edgetpu-kci.h"
	#include "edgetpu-mailbox.h"
	#include "edgetpu-mcp.h"
	#include "edgetpu-mmu.h"
	#include "edgetpu-sw-watchdog.h"
	#include "edgetpu-telemetry.h"
	#include "edgetpu-usage-stats.h"
	#include "edgetpu-wakelock.h"
	#include "edgetpu.h"

	/* Bits higher than VMA_TYPE_WIDTH are used to carry type specific data, e.g., core id. */
	#define VMA_TYPE_WIDTH 16
	#define VMA_TYPE(x) ((x) & (BIT_MASK(VMA_TYPE_WIDTH) - 1))
	#define VMA_DATA_GET(x) ((x) >> VMA_TYPE_WIDTH)
	#define VMA_DATA_SET(x, y) (VMA_TYPE(x) \| ((y) << VMA_TYPE_WIDTH))

	enum edgetpu_vma_type {
	VMA_INVALID,

	VMA_FULL_CSR,
	VMA_VII_CSR,
	VMA_VII_CMDQ,
	VMA_VII_RESPQ,
	VMA_EXT_CSR,
	VMA_EXT_CMDQ,
	VMA_EXT_RESPQ,
	/* For VMA_LOG and VMA_TRACE, core id is stored in bits higher than VMA_TYPE_WIDTH. */
	VMA_LOG,
	VMA_TRACE,
	};

	/* type that combines enum edgetpu_vma_type and data in higher bits. */
	typedef u32 edgetpu_vma_flags_t;

	/* structure to be set to vma->vm_private_data on mmap */
	struct edgetpu_vma_private {
	struct edgetpu_client *client;
	edgetpu_vma_flags_t flag;
	/*
	* vm_private_data is copied when a VMA is split, using this reference
	* counter to know when should this object be freed.
	*/
	refcount_t count;
	};

	static atomic_t single_dev_count = ATOMIC_INIT(-1);

	static int edgetpu_mmap_full_csr(struct edgetpu_client *client,
	struct vm_area_struct *vma)
	{
	int ret;
	ulong phys_base, vma_size, map_size;

	if (!uid_eq(current_euid(), GLOBAL_ROOT_UID))
	return -EPERM;
	vma_size = vma->vm_end - vma->vm_start;
	map_size = min_t(ulong, vma_size, client->etdev->regs.size);
	phys_base = client->etdev->regs.phys;
	ret = io_remap_pfn_range(vma, vma->vm_start, phys_base >> PAGE_SHIFT,
	map_size, vma->vm_page_prot);
	if (ret)
	etdev_dbg(client->etdev,
	"Error remapping PFN range: %d\n", ret);
	return ret;
	}

	static edgetpu_vma_flags_t mmap_vma_flag(unsigned long pgoff)
	{
	const unsigned long off = pgoff << PAGE_SHIFT;

	switch (off) {
	case 0:
	return VMA_FULL_CSR;
	case EDGETPU_MMAP_CSR_OFFSET:
	return VMA_VII_CSR;
	case EDGETPU_MMAP_CMD_QUEUE_OFFSET:
	return VMA_VII_CMDQ;
	case EDGETPU_MMAP_RESP_QUEUE_OFFSET:
	return VMA_VII_RESPQ;
	#ifdef EDGETPU_FEATURE_INTEROP
	case EDGETPU_MMAP_EXT_CSR_OFFSET:
	return VMA_EXT_CSR;
	case EDGETPU_MMAP_EXT_CMD_QUEUE_OFFSET:
	return VMA_EXT_CMDQ;
	case EDGETPU_MMAP_EXT_RESP_QUEUE_OFFSET:
	return VMA_EXT_RESPQ;
	#endif /* EDGETPU_FEATURE_INTEROP */
	case EDGETPU_MMAP_LOG_BUFFER_OFFSET:
	return VMA_DATA_SET(VMA_LOG, 0);
	case EDGETPU_MMAP_TRACE_BUFFER_OFFSET:
	return VMA_DATA_SET(VMA_TRACE, 0);
	#if EDGETPU_NUM_CORES > 1
	case EDGETPU_MMAP_LOG1_BUFFER_OFFSET:
	return VMA_DATA_SET(VMA_LOG, 1);
	case EDGETPU_MMAP_TRACE1_BUFFER_OFFSET:
	return VMA_DATA_SET(VMA_TRACE, 1);
	#endif /* EDGETPU_NUM_CORES > 1 */
	default:
	return VMA_INVALID;
	}
	}

	/*
	* Returns the wakelock event by VMA type. Returns EDGETPU_WAKELOCK_EVENT_END
	* if the type does not correspond to a wakelock event.
	*/
	static enum edgetpu_wakelock_event
	vma_type_to_wakelock_event(enum edgetpu_vma_type type)
	{
	switch (type) {
	case VMA_FULL_CSR:
	return EDGETPU_WAKELOCK_EVENT_FULL_CSR;
	case VMA_VII_CSR:
	return EDGETPU_WAKELOCK_EVENT_MBOX_CSR;
	case VMA_VII_CMDQ:
	return EDGETPU_WAKELOCK_EVENT_CMD_QUEUE;
	case VMA_VII_RESPQ:
	return EDGETPU_WAKELOCK_EVENT_RESP_QUEUE;
	case VMA_EXT_CSR:
	return EDGETPU_WAKELOCK_EVENT_MBOX_CSR;
	case VMA_EXT_CMDQ:
	return EDGETPU_WAKELOCK_EVENT_CMD_QUEUE;
	case VMA_EXT_RESPQ:
	return EDGETPU_WAKELOCK_EVENT_RESP_QUEUE;
	default:
	return EDGETPU_WAKELOCK_EVENT_END;
	}
	}

	static struct edgetpu_vma_private *
	edgetpu_vma_private_alloc(struct edgetpu_client *client,
	edgetpu_vma_flags_t flag)
	{
	struct edgetpu_vma_private pvt = kmalloc(sizeof(pvt), GFP_KERNEL);

	if (!pvt)
	return NULL;
	pvt->client = edgetpu_client_get(client);
	pvt->flag = flag;
	refcount_set(&pvt->count, 1);

	return pvt;
	}

	static void edgetpu_vma_private_get(struct edgetpu_vma_private *pvt)
	{
	WARN_ON_ONCE(!refcount_inc_not_zero(&pvt->count));
	}

	static void edgetpu_vma_private_put(struct edgetpu_vma_private *pvt)
	{
	if (!pvt)
	return;
	if (refcount_dec_and_test(&pvt->count)) {
	edgetpu_client_put(pvt->client);
	kfree(pvt);
	}
	}

	static void edgetpu_vma_open(struct vm_area_struct *vma)
	{
	struct edgetpu_vma_private *pvt = vma->vm_private_data;
	enum edgetpu_wakelock_event evt;
	struct edgetpu_client *client;
	struct edgetpu_dev *etdev;
	enum edgetpu_vma_type type = VMA_TYPE(pvt->flag);

	edgetpu_vma_private_get(pvt);
	client = pvt->client;
	etdev = client->etdev;

	evt = vma_type_to_wakelock_event(type);
	if (evt != EDGETPU_WAKELOCK_EVENT_END)
	edgetpu_wakelock_inc_event(client->wakelock, evt);

	/* handle telemetry types */
	switch (type) {
	case VMA_LOG:
	edgetpu_telemetry_inc_mmap_count(etdev, EDGETPU_TELEMETRY_LOG,
	VMA_DATA_GET(pvt->flag));
	break;
	case VMA_TRACE:
	edgetpu_telemetry_inc_mmap_count(etdev, EDGETPU_TELEMETRY_TRACE,
	VMA_DATA_GET(pvt->flag));
	break;
	default:
	break;
	}
	}

	/* Records previously mmapped addresses were unmapped. */
	static void edgetpu_vma_close(struct vm_area_struct *vma)
	{
	struct edgetpu_vma_private *pvt = vma->vm_private_data;
	struct edgetpu_client *client = pvt->client;
	enum edgetpu_vma_type type = VMA_TYPE(pvt->flag);
	enum edgetpu_wakelock_event evt = vma_type_to_wakelock_event(type);
	struct edgetpu_dev *etdev = client->etdev;

	if (evt != EDGETPU_WAKELOCK_EVENT_END)
	edgetpu_wakelock_dec_event(client->wakelock, evt);

	/* handle telemetry types */
	switch (type) {
	case VMA_LOG:
	edgetpu_telemetry_dec_mmap_count(etdev, EDGETPU_TELEMETRY_LOG,
	VMA_DATA_GET(pvt->flag));
	break;
	case VMA_TRACE:
	edgetpu_telemetry_dec_mmap_count(etdev, EDGETPU_TELEMETRY_TRACE,
	VMA_DATA_GET(pvt->flag));
	break;
	default:
	break;
	}

	edgetpu_vma_private_put(pvt);
	}

	static const struct vm_operations_struct edgetpu_vma_ops = {
	.open = edgetpu_vma_open,
	.close = edgetpu_vma_close,
	};

	/* Map exported device CSRs or queue into user space. */
	int edgetpu_mmap(struct edgetpu_client client, struct vm_area_struct vma)
	{
	int ret = 0;
	edgetpu_vma_flags_t flag;
	enum edgetpu_vma_type type;
	enum edgetpu_wakelock_event evt;
	struct edgetpu_vma_private *pvt;

	if (vma->vm_start & ~PAGE_MASK) {
	etdev_dbg(client->etdev,
	"Base address not page-aligned: %#lx\n",
	vma->vm_start);
	return -EINVAL;
	}

	etdev_dbg(client->etdev, "%s: mmap pgoff = %#lX\n", __func__,
	vma->vm_pgoff);

	flag = mmap_vma_flag(vma->vm_pgoff);
	type = VMA_TYPE(flag);
	if (type == VMA_INVALID)
	return -EINVAL;
	pvt = edgetpu_vma_private_alloc(client, flag);
	if (!pvt)
	return -ENOMEM;

	/* Mark the VMA's pages as uncacheable. */
	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	/* Disable fancy things to ensure our event counters work. */
	vma->vm_flags \|= VM_DONTCOPY \| VM_DONTEXPAND \| VM_DONTDUMP;

	/* map all CSRs for debug purpose */
	if (type == VMA_FULL_CSR) {
	evt = EDGETPU_WAKELOCK_EVENT_FULL_CSR;
	if (edgetpu_wakelock_inc_event(client->wakelock, evt)) {
	ret = edgetpu_mmap_full_csr(client, vma);
	if (ret)
	goto out_dec_evt;
	} else {
	ret = -EAGAIN;
	}
	goto out_set_op;
	}

	/* Allow mapping log and telemetry buffers without a group */
	if (type == VMA_LOG) {
	ret = edgetpu_mmap_telemetry_buffer(client->etdev, EDGETPU_TELEMETRY_LOG, vma,
	VMA_DATA_GET(flag));
	goto out_set_op;
	}
	if (type == VMA_TRACE) {
	ret = edgetpu_mmap_telemetry_buffer(client->etdev, EDGETPU_TELEMETRY_TRACE, vma,
	VMA_DATA_GET(flag));
	goto out_set_op;
	}

	evt = vma_type_to_wakelock_event(type);
	/*
	* VMA_TYPE(@flag) should always correspond to a valid event since we handled
	* telemetry mmaps above, still check evt != END in case new types are
	* added in the future.
	*/
	if (unlikely(evt == EDGETPU_WAKELOCK_EVENT_END)) {
	ret = -EINVAL;
	goto err_release_pvt;
	}
	if (!edgetpu_wakelock_inc_event(client->wakelock, evt)) {
	ret = -EAGAIN;
	goto err_release_pvt;
	}

	mutex_lock(&client->group_lock);
	if (!client->group) {
	ret = -EINVAL;
	goto out_unlock;
	}
	switch (type) {
	case VMA_VII_CSR:
	ret = edgetpu_mmap_csr(client->group, vma, false);
	break;
	case VMA_VII_CMDQ:
	ret = edgetpu_mmap_queue(client->group, MAILBOX_CMD_QUEUE, vma, false);
	break;
	case VMA_VII_RESPQ:
	ret = edgetpu_mmap_queue(client->group, MAILBOX_RESP_QUEUE, vma, false);
	break;
	case VMA_EXT_CSR:
	ret = edgetpu_mmap_csr(client->group, vma, true);
	break;
	case VMA_EXT_CMDQ:
	ret = edgetpu_mmap_queue(client->group, MAILBOX_CMD_QUEUE, vma, true);
	break;
	case VMA_EXT_RESPQ:
	ret = edgetpu_mmap_queue(client->group, MAILBOX_RESP_QUEUE, vma, true);
	break;
	default: /* to appease compiler */
	break;
	}
	out_unlock:
	mutex_unlock(&client->group_lock);
	out_dec_evt:
	if (ret)
	edgetpu_wakelock_dec_event(client->wakelock, evt);
	out_set_op:
	if (!ret) {
	vma->vm_private_data = pvt;
	vma->vm_ops = &edgetpu_vma_ops;
	return 0;
	}
	err_release_pvt:
	edgetpu_vma_private_put(pvt);
	return ret;
	}

	static struct edgetpu_mailbox_manager_desc mailbox_manager_desc = {
	.num_mailbox = EDGETPU_NUM_MAILBOXES,
	.num_vii_mailbox = EDGETPU_NUM_VII_MAILBOXES,
	.num_p2p_mailbox = EDGETPU_NUM_P2P_MAILBOXES,
	.num_ext_mailbox = EDGETPU_NUM_EXT_MAILBOXES,
	.get_context_csr_base = edgetpu_mailbox_get_context_csr_base,
	.get_cmd_queue_csr_base = edgetpu_mailbox_get_cmd_queue_csr_base,
	.get_resp_queue_csr_base = edgetpu_mailbox_get_resp_queue_csr_base,
	};

	int edgetpu_get_state_errno_locked(struct edgetpu_dev *etdev)
	{
	switch (etdev->state) {
	case ETDEV_STATE_BAD:
	return -ENODEV;
	case ETDEV_STATE_FWLOADING:
	return -EAGAIN;
	case ETDEV_STATE_NOFW:
	return -EINVAL;
	default:
	break;
	}
	return 0;
	}

	int edgetpu_device_add(struct edgetpu_dev *etdev,
	const struct edgetpu_mapped_resource *regs,
	const struct edgetpu_iface_params *iface_params,
	uint num_ifaces)
	{
	int ret;

	etdev->regs = *regs;

	etdev->etiface = devm_kzalloc(
	etdev->dev, sizeof(etdev->etiface) num_ifaces, GFP_KERNEL);

	if (!etdev->etiface) {
	dev_err(etdev->dev,
	"Failed to allocate memory for interfaces\n");
	return -ENOMEM;
	}

	/* mcp_id and mcp_die_index fields set by caller */
	if (etdev->mcp_id < 0) {
	uint ordinal_id = atomic_add_return(1, &single_dev_count);

	if (!ordinal_id)
	snprintf(etdev->dev_name, EDGETPU_DEVICE_NAME_MAX, "%s",
	DRIVER_NAME);
	else
	snprintf(etdev->dev_name, EDGETPU_DEVICE_NAME_MAX,
	"%s.%u", DRIVER_NAME, ordinal_id);
	} else {
	snprintf(etdev->dev_name, EDGETPU_DEVICE_NAME_MAX,
	"%s.%u.%u", DRIVER_NAME, etdev->mcp_id,
	etdev->mcp_die_index);
	}

	mutex_init(&etdev->groups_lock);
	INIT_LIST_HEAD(&etdev->groups);
	etdev->n_groups = 0;
	etdev->group_join_lockout = false;
	mutex_init(&etdev->clients_lock);
	INIT_LIST_HEAD(&etdev->clients);
	etdev->vcid_pool = (1u << EDGETPU_NUM_VCIDS) - 1;
	mutex_init(&etdev->state_lock);
	etdev->state = ETDEV_STATE_NOFW;
	etdev->freq_count = 0;
	mutex_init(&etdev->freq_lock);

	ret = edgetpu_fs_add(etdev, iface_params, num_ifaces);
	if (ret) {
	dev_err(etdev->dev, "%s: edgetpu_fs_add returns %d\n", etdev->dev_name, ret);
	goto remove_dev;
	}

	etdev->mailbox_manager =
	edgetpu_mailbox_create_mgr(etdev, &mailbox_manager_desc);
	if (IS_ERR(etdev->mailbox_manager)) {
	ret = PTR_ERR(etdev->mailbox_manager);
	dev_err(etdev->dev,
	"%s: edgetpu_mailbox_create_mgr returns %d\n",
	etdev->dev_name, ret);
	goto remove_dev;
	}
	ret = edgetpu_chip_setup_mmu(etdev);
	if (ret)
	goto remove_dev;

	edgetpu_usage_stats_init(etdev);

	etdev->kci = devm_kzalloc(etdev->dev, sizeof(*etdev->kci), GFP_KERNEL);
	if (!etdev->kci) {
	ret = -ENOMEM;
	goto remove_usage_stats;
	}

	etdev->telemetry =
	devm_kcalloc(etdev->dev, etdev->num_cores, sizeof(*etdev->telemetry), GFP_KERNEL);
	if (!etdev->telemetry) {
	ret = -ENOMEM;
	goto remove_usage_stats;
	}

	ret = edgetpu_kci_init(etdev->mailbox_manager, etdev->kci);
	if (ret) {
	etdev_err(etdev, "edgetpu_kci_init returns %d\n", ret);
	goto remove_usage_stats;
	}

	ret = edgetpu_device_dram_init(etdev);
	if (ret) {
	etdev_err(etdev,
	"failed to init on-device DRAM management: %d\n",
	ret);
	goto remove_kci;
	}

	ret = edgetpu_debug_dump_init(etdev);
	if (ret)
	etdev_warn(etdev, "debug dump init fail: %d", ret);

	edgetpu_chip_init(etdev);
	return 0;

	remove_kci:
	/* releases the resources of KCI */
	edgetpu_mailbox_remove_all(etdev->mailbox_manager);
	remove_usage_stats:
	edgetpu_usage_stats_exit(etdev);
	edgetpu_chip_remove_mmu(etdev);
	remove_dev:
	edgetpu_mark_probe_fail(etdev);
	edgetpu_fs_remove(etdev);
	return ret;
	}

	void edgetpu_device_remove(struct edgetpu_dev *etdev)
	{
	edgetpu_chip_exit(etdev);
	edgetpu_debug_dump_exit(etdev);
	edgetpu_device_dram_exit(etdev);
	edgetpu_mailbox_remove_all(etdev->mailbox_manager);
	edgetpu_usage_stats_exit(etdev);
	edgetpu_chip_remove_mmu(etdev);
	edgetpu_fs_remove(etdev);
	}

	struct edgetpu_client edgetpu_client_add(struct edgetpu_dev_iface etiface)
	{
	struct edgetpu_client *client;
	struct edgetpu_list_device_client l = kmalloc(sizeof(l), GFP_KERNEL);
	struct edgetpu_dev *etdev = etiface->etdev;

	if (!l)
	return ERR_PTR(-ENOMEM);
	client = kzalloc(sizeof(*client), GFP_KERNEL);
	if (!client) {
	kfree(l);
	return ERR_PTR(-ENOMEM);
	}
	client->wakelock = edgetpu_wakelock_alloc(etdev);
	if (!client->wakelock) {
	kfree(client);
	kfree(l);
	return ERR_PTR(-ENOMEM);
	}

	client->pid = current->pid;
	client->tgid = current->tgid;
	client->etdev = etdev;
	client->etiface = etiface;
	mutex_init(&client->group_lock);
	/* equivalent to edgetpu_client_get() */
	refcount_set(&client->count, 1);
	client->perdie_events = 0;
	mutex_lock(&etdev->clients_lock);
	l->client = client;
	list_add_tail(&l->list, &etdev->clients);
	mutex_unlock(&etdev->clients_lock);
	return client;
	}

	struct edgetpu_client edgetpu_client_get(struct edgetpu_client client)
	{
	WARN_ON_ONCE(!refcount_inc_not_zero(&client->count));
	return client;
	}

	void edgetpu_client_put(struct edgetpu_client *client)
	{
	if (!client)
	return;
	if (refcount_dec_and_test(&client->count)) {
	edgetpu_wakelock_free(client->wakelock);
	kfree(client);
	}
	}

	void edgetpu_client_remove(struct edgetpu_client *client)
	{
	struct edgetpu_dev *etdev;
	struct edgetpu_list_device_client *lc;

	if (IS_ERR_OR_NULL(client))
	return;
	etdev = client->etdev;
	mutex_lock(&etdev->clients_lock);
	/* remove the client from the device list */
	for_each_list_device_client(etdev, lc) {
	if (lc->client == client) {
	list_del(&lc->list);
	kfree(lc);
	break;
	}
	}
	mutex_unlock(&etdev->clients_lock);
	/*
	* A quick check without holding client->group_lock.
	*
	* If client doesn't belong to a group then we are fine to not remove
	* from groups.
	*
	* If there is a race that the client belongs to a group but is removing
	* by another process - this will be detected by the check with holding
	* client->group_lock later.
	*/
	if (client->group)
	edgetpu_device_group_leave(client);
	/* invoke chip-dependent removal handler before releasing resources */
	edgetpu_chip_client_remove(client);

	/* Clean up all the per die event fds registered by the client */
	if (client->perdie_events &
	1 << perdie_event_id_to_num(EDGETPU_PERDIE_EVENT_LOGS_AVAILABLE))
	edgetpu_telemetry_unset_event(etdev, EDGETPU_TELEMETRY_LOG);
	if (client->perdie_events &
	1 << perdie_event_id_to_num(EDGETPU_PERDIE_EVENT_TRACES_AVAILABLE))
	edgetpu_telemetry_unset_event(etdev, EDGETPU_TELEMETRY_TRACE);

	edgetpu_client_put(client);
	}

	int edgetpu_register_irq(struct edgetpu_dev *etdev, int irq)
	{
	int ret;

	ret = devm_request_irq(etdev->dev, irq, edgetpu_chip_irq_handler,
	IRQF_ONESHOT, etdev->dev_name, etdev);
	if (ret)
	dev_err(etdev->dev, "%s: failed to request irq %d: %d\n",
	etdev->dev_name, irq, ret);
	return ret;
	}

	void edgetpu_unregister_irq(struct edgetpu_dev *etdev, int irq)
	{
	devm_free_irq(etdev->dev, irq, etdev);
	}

	int edgetpu_alloc_coherent(struct edgetpu_dev *etdev, size_t size,
	struct edgetpu_coherent_mem *mem,
	enum edgetpu_context_id context_id)
	{
	const u32 flags = EDGETPU_MMU_DIE \| EDGETPU_MMU_32 \| EDGETPU_MMU_HOST \|
	EDGETPU_MMU_COHERENT;

	mem->vaddr = dma_alloc_coherent(etdev->dev, size, &mem->dma_addr,
	GFP_KERNEL);
	if (!mem->vaddr)
	return -ENOMEM;
	edgetpu_x86_coherent_mem_init(mem);
	mem->tpu_addr =
	edgetpu_mmu_tpu_map(etdev, mem->dma_addr, size,
	DMA_BIDIRECTIONAL, context_id, flags);
	if (!mem->tpu_addr) {
	dma_free_coherent(etdev->dev, size, mem->vaddr, mem->dma_addr);
	mem->vaddr = NULL;
	return -EINVAL;
	}
	mem->size = size;
	return 0;
	}

	void edgetpu_free_coherent(struct edgetpu_dev *etdev,
	struct edgetpu_coherent_mem *mem,
	enum edgetpu_context_id context_id)
	{
	edgetpu_mmu_tpu_unmap(etdev, mem->tpu_addr, mem->size, context_id);
	edgetpu_x86_coherent_mem_set_wb(mem);
	dma_free_coherent(etdev->dev, mem->size, mem->vaddr, mem->dma_addr);
	mem->vaddr = NULL;
	}

	void edgetpu_handle_firmware_crash(struct edgetpu_dev *etdev,
	enum edgetpu_fw_crash_type crash_type)
	{
	if (crash_type == EDGETPU_FW_CRASH_UNRECOV_FAULT) {
	etdev_err(etdev, "firmware unrecoverable crash");
	etdev->firmware_crash_count++;
	edgetpu_fatal_error_notify(etdev, EDGETPU_ERROR_FW_CRASH);
	/* Restart firmware without chip reset */
	edgetpu_watchdog_bite(etdev, false);
	} else {
	etdev_err(etdev, "firmware non-fatal crash event: %u",
	crash_type);
	}
	}

	int __init edgetpu_init(void)
	{
	int ret;

	ret = edgetpu_fs_init();
	if (ret)
	return ret;
	edgetpu_mcp_init();
	return 0;
	}

	void __exit edgetpu_exit(void)
	{
	edgetpu_mcp_exit();
	edgetpu_fs_exit();
	}