| /* |
| * Broadcom Dongle Host Driver (DHD), Linux-specific network interface |
| * Basically selected code segments from usb-cdc.c and usb-rndis.c |
| * |
| * Copyright (C) 2021, Broadcom. |
| * |
| * Unless you and Broadcom execute a separate written software license |
| * agreement governing use of this software, this software is licensed to you |
| * under the terms of the GNU General Public License version 2 (the "GPL"), |
| * available at http://www.broadcom.com/licenses/GPLv2.php, with the |
| * following added to such license: |
| * |
| * As a special exception, the copyright holders of this software give you |
| * permission to link this software with independent modules, and to copy and |
| * distribute the resulting executable under terms of your choice, provided that |
| * you also meet, for each linked independent module, the terms and conditions of |
| * the license of that module. An independent module is a module which is not |
| * derived from this software. The special exception does not apply to any |
| * modifications of the software. |
| * |
| * |
| * <<Broadcom-WL-IPTag/Open:>> |
| * |
| * $Id$ |
| */ |
| |
| #include <dhd_linux_priv.h> |
| |
| extern dhd_pub_t* g_dhd_pub; |
| |
| #if defined(DHD_LB) |
| |
| #ifdef DHD_LB_STATS |
| #define DHD_NUM_NAPI_LATENCY_ROWS (17u) |
| #define DHD_NAPI_LATENCY_SIZE (sizeof(uint64) * DHD_NUM_NAPI_LATENCY_ROWS) |
| #endif /* DHD_LB_STATS */ |
| |
| #ifdef DHD_REPLACE_LOG_INFO_TO_TRACE |
| #define DHD_LB_INFO DHD_TRACE |
| #else |
| #define DHD_LB_INFO DHD_INFO |
| #endif /* DHD_REPLACE_LOG_INFO_TO_TRACE */ |
| |
| void |
| dhd_lb_set_default_cpus(dhd_info_t *dhd) |
| { |
| /* Default CPU allocation for the jobs */ |
| atomic_set(&dhd->rx_napi_cpu, 1); |
| atomic_set(&dhd->tx_cpu, 2); |
| atomic_set(&dhd->net_tx_cpu, 0); |
| atomic_set(&dhd->dpc_cpu, 0); |
| } |
| |
| void |
| dhd_cpumasks_deinit(dhd_info_t *dhd) |
| { |
| free_cpumask_var(dhd->cpumask_curr_avail); |
| free_cpumask_var(dhd->cpumask_primary); |
| free_cpumask_var(dhd->cpumask_primary_new); |
| free_cpumask_var(dhd->cpumask_secondary); |
| free_cpumask_var(dhd->cpumask_secondary_new); |
| } |
| |
| int |
| dhd_cpumasks_init(dhd_info_t *dhd) |
| { |
| int id; |
| uint32 cpus, num_cpus = num_possible_cpus(); |
| int ret = 0; |
| |
| DHD_ERROR(("%s CPU masks primary(big)=0x%x secondary(little)=0x%x\n", __FUNCTION__, |
| DHD_LB_PRIMARY_CPUS, DHD_LB_SECONDARY_CPUS)); |
| |
| /* FIXME: If one alloc fails we must free_cpumask_var the previous */ |
| if (!alloc_cpumask_var(&dhd->cpumask_curr_avail, GFP_KERNEL) || |
| !alloc_cpumask_var(&dhd->cpumask_primary, GFP_KERNEL) || |
| !alloc_cpumask_var(&dhd->cpumask_primary_new, GFP_KERNEL) || |
| !alloc_cpumask_var(&dhd->cpumask_secondary, GFP_KERNEL) || |
| !alloc_cpumask_var(&dhd->cpumask_secondary_new, GFP_KERNEL)) { |
| DHD_ERROR(("%s Failed to init cpumasks\n", __FUNCTION__)); |
| ret = -ENOMEM; |
| goto fail; |
| } |
| |
| cpumask_copy(dhd->cpumask_curr_avail, cpu_online_mask); |
| cpumask_clear(dhd->cpumask_primary); |
| cpumask_clear(dhd->cpumask_secondary); |
| |
| if (num_cpus > 32) { |
| DHD_ERROR(("%s max cpus must be 32, %d too big\n", __FUNCTION__, num_cpus)); |
| ASSERT(0); |
| } |
| |
| cpus = DHD_LB_PRIMARY_CPUS; |
| for (id = 0; id < num_cpus; id++) { |
| if (isset(&cpus, id)) |
| cpumask_set_cpu(id, dhd->cpumask_primary); |
| } |
| |
| cpus = DHD_LB_SECONDARY_CPUS; |
| for (id = 0; id < num_cpus; id++) { |
| if (isset(&cpus, id)) |
| cpumask_set_cpu(id, dhd->cpumask_secondary); |
| } |
| |
| return ret; |
| fail: |
| dhd_cpumasks_deinit(dhd); |
| return ret; |
| } |
| |
| /* |
| * The CPU Candidacy Algorithm |
| * ~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| * The available CPUs for selection are divided into two groups |
| * Primary Set - A CPU mask that carries the First Choice CPUs |
| * Secondary Set - A CPU mask that carries the Second Choice CPUs. |
| * |
| * There are two types of Job, that needs to be assigned to |
| * the CPUs, from one of the above mentioned CPU group. The Jobs are |
| * 1) Rx Packet Processing - napi_cpu |
| * |
| * To begin with napi_cpu is on CPU0. Whenever a CPU goes |
| * on-line/off-line the CPU candidacy algorithm is triggerd. The candidacy |
| * algo tries to pickup the first available non boot CPU (CPU0) for napi_cpu. |
| * |
| */ |
| void dhd_select_cpu_candidacy(dhd_info_t *dhd) |
| { |
| uint32 primary_available_cpus; /* count of primary available cpus */ |
| uint32 secondary_available_cpus; /* count of secondary available cpus */ |
| uint32 napi_cpu = 0; /* cpu selected for napi rx processing */ |
| uint32 tx_cpu = 0; /* cpu selected for tx processing job */ |
| uint32 dpc_cpu = atomic_read(&dhd->dpc_cpu); |
| uint32 net_tx_cpu = atomic_read(&dhd->net_tx_cpu); |
| |
| cpumask_clear(dhd->cpumask_primary_new); |
| cpumask_clear(dhd->cpumask_secondary_new); |
| |
| /* |
| * Now select from the primary mask. Even if a Job is |
| * already running on a CPU in secondary group, we still move |
| * to primary CPU. So no conditional checks. |
| */ |
| cpumask_and(dhd->cpumask_primary_new, dhd->cpumask_primary, |
| dhd->cpumask_curr_avail); |
| |
| cpumask_and(dhd->cpumask_secondary_new, dhd->cpumask_secondary, |
| dhd->cpumask_curr_avail); |
| |
| /* Clear DPC cpu from new masks so that dpc cpu is not chosen for LB */ |
| cpumask_clear_cpu(dpc_cpu, dhd->cpumask_primary_new); |
| cpumask_clear_cpu(dpc_cpu, dhd->cpumask_secondary_new); |
| |
| /* Clear net_tx_cpu from new masks so that same is not chosen for LB */ |
| cpumask_clear_cpu(net_tx_cpu, dhd->cpumask_primary_new); |
| cpumask_clear_cpu(net_tx_cpu, dhd->cpumask_secondary_new); |
| |
| primary_available_cpus = cpumask_weight(dhd->cpumask_primary_new); |
| |
| #if defined(DHD_LB_HOST_CTRL) |
| /* Does not use promary cpus if DHD received affinity off cmd |
| * from framework |
| */ |
| if (primary_available_cpus > 0 && dhd->permitted_primary_cpu) { |
| #else |
| if (primary_available_cpus > 0) { |
| #endif /* DHD_LB_HOST_CTRL */ |
| napi_cpu = cpumask_first(dhd->cpumask_primary_new); |
| |
| /* If no further CPU is available, |
| * cpumask_next returns >= nr_cpu_ids |
| */ |
| tx_cpu = cpumask_next(napi_cpu, dhd->cpumask_primary_new); |
| if (tx_cpu >= nr_cpu_ids) |
| tx_cpu = 0; |
| } |
| |
| DHD_INFO(("%s After primary CPU check napi_cpu %d tx_cpu %d\n", |
| __FUNCTION__, napi_cpu, tx_cpu)); |
| |
| /* -- Now check for the CPUs from the secondary mask -- */ |
| secondary_available_cpus = cpumask_weight(dhd->cpumask_secondary_new); |
| |
| DHD_INFO(("%s Available secondary cpus %d nr_cpu_ids %d\n", |
| __FUNCTION__, secondary_available_cpus, nr_cpu_ids)); |
| |
| if (secondary_available_cpus > 0) { |
| /* At this point if napi_cpu is unassigned it means no CPU |
| * is online from Primary Group |
| */ |
| #if defined(DHD_LB_TXP_LITTLE_CORE_CTRL) |
| /* Clear tx_cpu, so that it can be picked from little core */ |
| tx_cpu = 0; |
| #endif /* DHD_LB_TXP_LITTLE_CORE_CTRL */ |
| if (napi_cpu == 0) { |
| napi_cpu = cpumask_first(dhd->cpumask_secondary_new); |
| tx_cpu = cpumask_next(napi_cpu, dhd->cpumask_secondary_new); |
| } else if (tx_cpu == 0) { |
| tx_cpu = cpumask_first(dhd->cpumask_secondary_new); |
| } |
| |
| /* If no CPU was available for tx processing, choose CPU 0 */ |
| if (tx_cpu >= nr_cpu_ids) |
| tx_cpu = 0; |
| } |
| |
| if ((primary_available_cpus == 0) && |
| (secondary_available_cpus == 0)) { |
| /* No CPUs available from primary or secondary mask */ |
| napi_cpu = 1; |
| tx_cpu = 2; |
| } |
| |
| DHD_INFO(("%s After secondary CPU check napi_cpu %d tx_cpu %d\n", |
| __FUNCTION__, napi_cpu, tx_cpu)); |
| |
| ASSERT(napi_cpu < nr_cpu_ids); |
| ASSERT(tx_cpu < nr_cpu_ids); |
| |
| atomic_set(&dhd->rx_napi_cpu, napi_cpu); |
| atomic_set(&dhd->tx_cpu, tx_cpu); |
| |
| return; |
| } |
| |
| /* |
| * Function to handle CPU Hotplug notifications. |
| * One of the task it does is to trigger the CPU Candidacy algorithm |
| * for load balancing. |
| */ |
| |
| #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) |
| |
| int dhd_cpu_startup_callback(unsigned int cpu) |
| { |
| dhd_info_t *dhd = g_dhd_pub->info; |
| |
| DHD_INFO(("%s(): \r\n cpu:%d", __FUNCTION__, cpu)); |
| DHD_LB_STATS_INCR(dhd->cpu_online_cnt[cpu]); |
| cpumask_set_cpu(cpu, dhd->cpumask_curr_avail); |
| dhd_select_cpu_candidacy(dhd); |
| |
| return 0; |
| } |
| |
| int dhd_cpu_teardown_callback(unsigned int cpu) |
| { |
| dhd_info_t *dhd = g_dhd_pub->info; |
| |
| DHD_INFO(("%s(): \r\n cpu:%d", __FUNCTION__, cpu)); |
| DHD_LB_STATS_INCR(dhd->cpu_offline_cnt[cpu]); |
| cpumask_clear_cpu(cpu, dhd->cpumask_curr_avail); |
| dhd_select_cpu_candidacy(dhd); |
| |
| return 0; |
| } |
| #else |
| int |
| dhd_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) |
| { |
| unsigned long int cpu = (unsigned long int)hcpu; |
| dhd_info_t *dhd; |
| |
| GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST(); |
| dhd = container_of(nfb, dhd_info_t, cpu_notifier); |
| GCC_DIAGNOSTIC_POP(); |
| |
| if (!dhd || !(dhd->dhd_state & DHD_ATTACH_STATE_LB_ATTACH_DONE)) { |
| DHD_INFO(("%s(): LB data is not initialized yet.\n", |
| __FUNCTION__)); |
| return NOTIFY_BAD; |
| } |
| |
| /* XXX: Do we need other action types ? */ |
| switch (action) |
| { |
| case CPU_ONLINE: |
| case CPU_ONLINE_FROZEN: |
| DHD_LB_STATS_INCR(dhd->cpu_online_cnt[cpu]); |
| cpumask_set_cpu(cpu, dhd->cpumask_curr_avail); |
| dhd_select_cpu_candidacy(dhd); |
| break; |
| |
| case CPU_DOWN_PREPARE: |
| case CPU_DOWN_PREPARE_FROZEN: |
| DHD_LB_STATS_INCR(dhd->cpu_offline_cnt[cpu]); |
| cpumask_clear_cpu(cpu, dhd->cpumask_curr_avail); |
| dhd_select_cpu_candidacy(dhd); |
| break; |
| default: |
| break; |
| } |
| |
| return NOTIFY_OK; |
| } |
| #endif /* LINUX_VERSION_CODE < 4.10.0 */ |
| |
| int dhd_register_cpuhp_callback(dhd_info_t *dhd) |
| { |
| int cpuhp_ret = 0; |
| #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) |
| cpuhp_ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "dhd", |
| dhd_cpu_startup_callback, dhd_cpu_teardown_callback); |
| |
| if (cpuhp_ret < 0) { |
| DHD_ERROR(("%s(): cpuhp_setup_state failed %d RX LB won't happen \r\n", |
| __FUNCTION__, cpuhp_ret)); |
| } |
| #else |
| /* |
| * If we are able to initialize CPU masks, lets register to the |
| * CPU Hotplug framework to change the CPU for each job dynamically |
| * using candidacy algorithm. |
| */ |
| dhd->cpu_notifier.notifier_call = dhd_cpu_callback; |
| register_hotcpu_notifier(&dhd->cpu_notifier); /* Register a callback */ |
| #endif /* LINUX_VERSION_CODE < 4.10.0 */ |
| return cpuhp_ret; |
| } |
| |
| int dhd_unregister_cpuhp_callback(dhd_info_t *dhd) |
| { |
| int ret = 0; |
| #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) |
| /* Don't want to call tear down while unregistering */ |
| cpuhp_remove_state_nocalls(CPUHP_AP_ONLINE_DYN); |
| #else |
| if (dhd->cpu_notifier.notifier_call != NULL) { |
| unregister_cpu_notifier(&dhd->cpu_notifier); |
| } |
| #endif |
| return ret; |
| } |
| |
| #if defined(DHD_LB_STATS) |
| void dhd_lb_stats_reset(dhd_pub_t *dhdp) |
| { |
| dhd_info_t *dhd; |
| int i, j, num_cpus = num_possible_cpus(); |
| |
| if (dhdp == NULL) { |
| DHD_ERROR(("%s dhd pub pointer is NULL \n", |
| __FUNCTION__)); |
| return; |
| } |
| |
| dhd = dhdp->info; |
| if (dhd == NULL) { |
| DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__)); |
| return; |
| } |
| |
| DHD_LB_STATS_CLR(dhd->dhd_dpc_cnt); |
| DHD_LB_STATS_CLR(dhd->napi_sched_cnt); |
| |
| /* reset NAPI latency stats */ |
| if (dhd->napi_latency) { |
| bzero(dhd->napi_latency, DHD_NAPI_LATENCY_SIZE); |
| } |
| /* reset NAPI per cpu stats */ |
| if (dhd->napi_percpu_run_cnt) { |
| for (i = 0; i < num_cpus; i++) { |
| DHD_LB_STATS_CLR(dhd->napi_percpu_run_cnt[i]); |
| } |
| } |
| |
| DHD_LB_STATS_CLR(dhd->rxc_sched_cnt); |
| |
| if (dhd->rxc_percpu_run_cnt) { |
| for (i = 0; i < num_cpus; i++) { |
| DHD_LB_STATS_CLR(dhd->rxc_percpu_run_cnt[i]); |
| } |
| } |
| |
| DHD_LB_STATS_CLR(dhd->txc_sched_cnt); |
| |
| if (dhd->txc_percpu_run_cnt) { |
| for (i = 0; i < num_cpus; i++) { |
| DHD_LB_STATS_CLR(dhd->txc_percpu_run_cnt[i]); |
| } |
| } |
| |
| if (dhd->txp_percpu_run_cnt) { |
| for (i = 0; i < num_cpus; i++) { |
| DHD_LB_STATS_CLR(dhd->txp_percpu_run_cnt[i]); |
| } |
| } |
| |
| if (dhd->tx_start_percpu_run_cnt) { |
| for (i = 0; i < num_cpus; i++) { |
| DHD_LB_STATS_CLR(dhd->tx_start_percpu_run_cnt[i]); |
| } |
| } |
| |
| for (j = 0; j < HIST_BIN_SIZE; j++) { |
| for (i = 0; i < num_cpus; i++) { |
| DHD_LB_STATS_CLR(dhd->napi_rx_hist[j][i]); |
| } |
| } |
| |
| dhd->pub.lb_rxp_strt_thr_hitcnt = 0; |
| dhd->pub.lb_rxp_stop_thr_hitcnt = 0; |
| |
| dhd->pub.lb_rxp_napi_sched_cnt = 0; |
| dhd->pub.lb_rxp_napi_complete_cnt = 0; |
| return; |
| } |
| |
| void dhd_lb_stats_init(dhd_pub_t *dhdp) |
| { |
| dhd_info_t *dhd; |
| int i, j, num_cpus = num_possible_cpus(); |
| int alloc_size = sizeof(uint32) * num_cpus; |
| |
| if (dhdp == NULL) { |
| DHD_ERROR(("%s(): Invalid argument dhd pubb pointer is NULL \n", |
| __FUNCTION__)); |
| return; |
| } |
| |
| dhd = dhdp->info; |
| if (dhd == NULL) { |
| DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__)); |
| return; |
| } |
| |
| DHD_LB_STATS_CLR(dhd->dhd_dpc_cnt); |
| DHD_LB_STATS_CLR(dhd->napi_sched_cnt); |
| |
| /* NAPI latency stats */ |
| dhd->napi_latency = (uint64 *)MALLOCZ(dhdp->osh, DHD_NAPI_LATENCY_SIZE); |
| /* NAPI per cpu stats */ |
| dhd->napi_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size); |
| if (!dhd->napi_percpu_run_cnt) { |
| DHD_ERROR(("%s(): napi_percpu_run_cnt malloc failed \n", |
| __FUNCTION__)); |
| return; |
| } |
| for (i = 0; i < num_cpus; i++) |
| DHD_LB_STATS_CLR(dhd->napi_percpu_run_cnt[i]); |
| |
| DHD_LB_STATS_CLR(dhd->rxc_sched_cnt); |
| |
| dhd->rxc_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size); |
| if (!dhd->rxc_percpu_run_cnt) { |
| DHD_ERROR(("%s(): rxc_percpu_run_cnt malloc failed \n", |
| __FUNCTION__)); |
| return; |
| } |
| for (i = 0; i < num_cpus; i++) |
| DHD_LB_STATS_CLR(dhd->rxc_percpu_run_cnt[i]); |
| |
| DHD_LB_STATS_CLR(dhd->txc_sched_cnt); |
| |
| dhd->txc_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size); |
| if (!dhd->txc_percpu_run_cnt) { |
| DHD_ERROR(("%s(): txc_percpu_run_cnt malloc failed \n", |
| __FUNCTION__)); |
| return; |
| } |
| for (i = 0; i < num_cpus; i++) |
| DHD_LB_STATS_CLR(dhd->txc_percpu_run_cnt[i]); |
| |
| dhd->cpu_online_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size); |
| if (!dhd->cpu_online_cnt) { |
| DHD_ERROR(("%s(): cpu_online_cnt malloc failed \n", |
| __FUNCTION__)); |
| return; |
| } |
| for (i = 0; i < num_cpus; i++) |
| DHD_LB_STATS_CLR(dhd->cpu_online_cnt[i]); |
| |
| dhd->cpu_offline_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size); |
| if (!dhd->cpu_offline_cnt) { |
| DHD_ERROR(("%s(): cpu_offline_cnt malloc failed \n", |
| __FUNCTION__)); |
| return; |
| } |
| for (i = 0; i < num_cpus; i++) |
| DHD_LB_STATS_CLR(dhd->cpu_offline_cnt[i]); |
| |
| dhd->txp_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size); |
| if (!dhd->txp_percpu_run_cnt) { |
| DHD_ERROR(("%s(): txp_percpu_run_cnt malloc failed \n", |
| __FUNCTION__)); |
| return; |
| } |
| for (i = 0; i < num_cpus; i++) |
| DHD_LB_STATS_CLR(dhd->txp_percpu_run_cnt[i]); |
| |
| dhd->tx_start_percpu_run_cnt = (uint32 *)MALLOC(dhdp->osh, alloc_size); |
| if (!dhd->tx_start_percpu_run_cnt) { |
| DHD_ERROR(("%s(): tx_start_percpu_run_cnt malloc failed \n", |
| __FUNCTION__)); |
| return; |
| } |
| for (i = 0; i < num_cpus; i++) |
| DHD_LB_STATS_CLR(dhd->tx_start_percpu_run_cnt[i]); |
| |
| for (j = 0; j < HIST_BIN_SIZE; j++) { |
| dhd->napi_rx_hist[j] = (uint32 *)MALLOC(dhdp->osh, alloc_size); |
| if (!dhd->napi_rx_hist[j]) { |
| DHD_ERROR(("%s(): dhd->napi_rx_hist[%d] malloc failed \n", |
| __FUNCTION__, j)); |
| return; |
| } |
| for (i = 0; i < num_cpus; i++) { |
| DHD_LB_STATS_CLR(dhd->napi_rx_hist[j][i]); |
| } |
| } |
| |
| dhd->pub.lb_rxp_strt_thr_hitcnt = 0; |
| dhd->pub.lb_rxp_stop_thr_hitcnt = 0; |
| |
| dhd->pub.lb_rxp_napi_sched_cnt = 0; |
| dhd->pub.lb_rxp_napi_complete_cnt = 0; |
| return; |
| } |
| |
| void dhd_lb_stats_deinit(dhd_pub_t *dhdp) |
| { |
| dhd_info_t *dhd; |
| int j, num_cpus = num_possible_cpus(); |
| int alloc_size = sizeof(uint32) * num_cpus; |
| |
| if (dhdp == NULL) { |
| DHD_ERROR(("%s(): Invalid argument dhd pubb pointer is NULL \n", |
| __FUNCTION__)); |
| return; |
| } |
| |
| dhd = dhdp->info; |
| if (dhd == NULL) { |
| DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__)); |
| return; |
| } |
| |
| if (dhd->napi_percpu_run_cnt) { |
| MFREE(dhdp->osh, dhd->napi_percpu_run_cnt, alloc_size); |
| } |
| if (dhd->rxc_percpu_run_cnt) { |
| MFREE(dhdp->osh, dhd->rxc_percpu_run_cnt, alloc_size); |
| } |
| if (dhd->txc_percpu_run_cnt) { |
| MFREE(dhdp->osh, dhd->txc_percpu_run_cnt, alloc_size); |
| } |
| if (dhd->cpu_online_cnt) { |
| MFREE(dhdp->osh, dhd->cpu_online_cnt, alloc_size); |
| } |
| if (dhd->cpu_offline_cnt) { |
| MFREE(dhdp->osh, dhd->cpu_offline_cnt, alloc_size); |
| } |
| |
| if (dhd->txp_percpu_run_cnt) { |
| MFREE(dhdp->osh, dhd->txp_percpu_run_cnt, alloc_size); |
| } |
| if (dhd->tx_start_percpu_run_cnt) { |
| MFREE(dhdp->osh, dhd->tx_start_percpu_run_cnt, alloc_size); |
| } |
| if (dhd->napi_latency) { |
| MFREE(dhdp->osh, dhd->napi_latency, DHD_NAPI_LATENCY_SIZE); |
| } |
| |
| for (j = 0; j < HIST_BIN_SIZE; j++) { |
| if (dhd->napi_rx_hist[j]) { |
| MFREE(dhdp->osh, dhd->napi_rx_hist[j], alloc_size); |
| } |
| } |
| |
| return; |
| } |
| |
| void dhd_lb_stats_dump_napi_latency(dhd_pub_t *dhdp, |
| struct bcmstrbuf *strbuf, uint64 *napi_latency) |
| { |
| uint32 i; |
| |
| bcm_bprintf(strbuf, "napi-latency(us): \t count\n"); |
| for (i = 0; i < DHD_NUM_NAPI_LATENCY_ROWS; i++) { |
| bcm_bprintf(strbuf, "%16u: \t %llu\n", 1U<<i, napi_latency[i]); |
| } |
| } |
| |
| void dhd_lb_stats_dump_histo(dhd_pub_t *dhdp, |
| struct bcmstrbuf *strbuf, uint32 **hist) |
| { |
| int i, j; |
| uint32 *per_cpu_total; |
| uint32 total = 0; |
| uint32 num_cpus = num_possible_cpus(); |
| |
| per_cpu_total = (uint32 *)MALLOC(dhdp->osh, sizeof(uint32) * num_cpus); |
| if (!per_cpu_total) { |
| DHD_ERROR(("%s(): dhd->per_cpu_total malloc failed \n", __FUNCTION__)); |
| return; |
| } |
| bzero(per_cpu_total, sizeof(uint32) * num_cpus); |
| |
| bcm_bprintf(strbuf, "CPU: \t\t"); |
| for (i = 0; i < num_cpus; i++) |
| bcm_bprintf(strbuf, "%d\t", i); |
| bcm_bprintf(strbuf, "\nBin\n"); |
| |
| for (i = 0; i < HIST_BIN_SIZE; i++) { |
| bcm_bprintf(strbuf, "%d:\t\t", 1<<i); |
| for (j = 0; j < num_cpus; j++) { |
| bcm_bprintf(strbuf, "%d\t", hist[i][j]); |
| } |
| bcm_bprintf(strbuf, "\n"); |
| } |
| bcm_bprintf(strbuf, "Per CPU Total \t"); |
| total = 0; |
| for (i = 0; i < num_cpus; i++) { |
| for (j = 0; j < HIST_BIN_SIZE; j++) { |
| per_cpu_total[i] += (hist[j][i] * (1<<j)); |
| } |
| bcm_bprintf(strbuf, "%d\t", per_cpu_total[i]); |
| total += per_cpu_total[i]; |
| } |
| bcm_bprintf(strbuf, "\nTotal\t\t%d \n", total); |
| |
| if (per_cpu_total) { |
| MFREE(dhdp->osh, per_cpu_total, sizeof(uint32) * num_cpus); |
| } |
| return; |
| } |
| |
| void dhd_lb_stats_dump_cpu_array(struct bcmstrbuf *strbuf, uint32 *p) |
| { |
| int i, num_cpus = num_possible_cpus(); |
| |
| bcm_bprintf(strbuf, "CPU: \t\t"); |
| for (i = 0; i < num_cpus; i++) |
| bcm_bprintf(strbuf, "%d\t", i); |
| bcm_bprintf(strbuf, "\n"); |
| |
| bcm_bprintf(strbuf, "Val: \t\t"); |
| for (i = 0; i < num_cpus; i++) |
| bcm_bprintf(strbuf, "%u\t", *(p+i)); |
| bcm_bprintf(strbuf, "\n"); |
| return; |
| } |
| |
| #ifdef DHD_MEM_STATS |
| uint64 dhd_lb_mem_usage(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf) |
| { |
| dhd_info_t *dhd; |
| uint16 rxbufpost_sz; |
| uint16 rx_post_active = 0; |
| uint16 rx_cmpl_active = 0; |
| uint64 rx_path_memory_usage = 0; |
| |
| if (dhdp == NULL || strbuf == NULL) { |
| DHD_ERROR(("%s(): Invalid argument dhdp %p strbuf %p \n", |
| __FUNCTION__, dhdp, strbuf)); |
| return 0; |
| } |
| |
| dhd = dhdp->info; |
| if (dhd == NULL) { |
| DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__)); |
| return 0; |
| } |
| rxbufpost_sz = dhd_prot_get_rxbufpost_sz(dhdp); |
| if (rxbufpost_sz == 0) { |
| rxbufpost_sz = DHD_FLOWRING_RX_BUFPOST_PKTSZ; |
| } |
| rx_path_memory_usage = rxbufpost_sz * (skb_queue_len(&dhd->rx_pend_queue) + |
| skb_queue_len(&dhd->rx_napi_queue) + |
| skb_queue_len(&dhd->rx_process_queue)); |
| rx_post_active = dhd_prot_get_h2d_rx_post_active(dhdp); |
| if (rx_post_active != 0) { |
| rx_path_memory_usage += (rxbufpost_sz * rx_post_active); |
| } |
| |
| rx_cmpl_active = dhd_prot_get_d2h_rx_cpln_active(dhdp); |
| if (rx_cmpl_active != 0) { |
| rx_path_memory_usage += (rxbufpost_sz * rx_cmpl_active); |
| } |
| |
| dhdp->rxpath_mem = rx_path_memory_usage; |
| bcm_bprintf(strbuf, "\nrxbufpost_sz: %d rx_post_active: %d rx_cmpl_active: %d " |
| "pend_queue_len: %d napi_queue_len: %d process_queue_len: %d\n", |
| rxbufpost_sz, rx_post_active, rx_cmpl_active, |
| skb_queue_len(&dhd->rx_pend_queue), |
| skb_queue_len(&dhd->rx_napi_queue), skb_queue_len(&dhd->rx_process_queue)); |
| bcm_bprintf(strbuf, "DHD rx-path memory_usage: %llubytes %lluKB \n", |
| rx_path_memory_usage, (rx_path_memory_usage/ 1024)); |
| return rx_path_memory_usage; |
| } |
| #endif /* DHD_MEM_STATS */ |
| |
| void dhd_lb_stats_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf) |
| { |
| dhd_info_t *dhd; |
| |
| if (dhdp == NULL || strbuf == NULL) { |
| DHD_ERROR(("%s(): Invalid argument dhdp %p strbuf %p \n", |
| __FUNCTION__, dhdp, strbuf)); |
| return; |
| } |
| |
| dhd = dhdp->info; |
| if (dhd == NULL) { |
| DHD_ERROR(("%s(): DHD pointer is NULL \n", __FUNCTION__)); |
| return; |
| } |
| |
| bcm_bprintf(strbuf, "\ncpu_online_cnt:\n"); |
| dhd_lb_stats_dump_cpu_array(strbuf, dhd->cpu_online_cnt); |
| |
| bcm_bprintf(strbuf, "\ncpu_offline_cnt:\n"); |
| dhd_lb_stats_dump_cpu_array(strbuf, dhd->cpu_offline_cnt); |
| |
| bcm_bprintf(strbuf, "\nsched_cnt: dhd_dpc %u napi %u rxc %u txc %u\n", |
| dhd->dhd_dpc_cnt, dhd->napi_sched_cnt, dhd->rxc_sched_cnt, |
| dhd->txc_sched_cnt); |
| |
| bcm_bprintf(strbuf, "\nCPUs: dpc_cpu %u napi_cpu %u net_tx_cpu %u tx_cpu %u\n", |
| atomic_read(&dhd->dpc_cpu), |
| atomic_read(&dhd->rx_napi_cpu), |
| atomic_read(&dhd->net_tx_cpu), |
| atomic_read(&dhd->tx_cpu)); |
| |
| #ifdef DHD_LB_RXP |
| bcm_bprintf(strbuf, "\nnapi_percpu_run_cnt:\n"); |
| dhd_lb_stats_dump_cpu_array(strbuf, dhd->napi_percpu_run_cnt); |
| bcm_bprintf(strbuf, "\nNAPI Packets Received Histogram:\n"); |
| dhd_lb_stats_dump_histo(dhdp, strbuf, dhd->napi_rx_hist); |
| bcm_bprintf(strbuf, "\nNAPI poll latency stats ie from napi schedule to napi execution\n"); |
| dhd_lb_stats_dump_napi_latency(dhdp, strbuf, dhd->napi_latency); |
| #endif /* DHD_LB_RXP */ |
| |
| #ifdef DHD_LB_TXP |
| bcm_bprintf(strbuf, "\ntxp_percpu_run_cnt:\n"); |
| dhd_lb_stats_dump_cpu_array(strbuf, dhd->txp_percpu_run_cnt); |
| |
| bcm_bprintf(strbuf, "\ntx_start_percpu_run_cnt:\n"); |
| dhd_lb_stats_dump_cpu_array(strbuf, dhd->tx_start_percpu_run_cnt); |
| #endif /* DHD_LB_TXP */ |
| } |
| |
| void dhd_lb_stats_update_napi_latency(uint64 *bin, uint32 latency) |
| { |
| uint64 *p; |
| uint32 bin_power; |
| bin_power = next_larger_power2(latency); |
| |
| switch (bin_power) { |
| case 1: p = bin + 0; break; |
| case 2: p = bin + 1; break; |
| case 4: p = bin + 2; break; |
| case 8: p = bin + 3; break; |
| case 16: p = bin + 4; break; |
| case 32: p = bin + 5; break; |
| case 64: p = bin + 6; break; |
| case 128: p = bin + 7; break; |
| case 256: p = bin + 8; break; |
| case 512: p = bin + 9; break; |
| case 1024: p = bin + 10; break; |
| case 2048: p = bin + 11; break; |
| case 4096: p = bin + 12; break; |
| case 8192: p = bin + 13; break; |
| case 16384: p = bin + 14; break; |
| case 32768: p = bin + 15; break; |
| default : p = bin + 16; break; |
| } |
| ASSERT((p - bin) < DHD_NUM_NAPI_LATENCY_ROWS); |
| *p = *p + 1; |
| return; |
| |
| } |
| |
| void dhd_lb_stats_update_histo(uint32 **bin, uint32 count, uint32 cpu) |
| { |
| uint32 bin_power; |
| uint32 *p; |
| bin_power = next_larger_power2(count); |
| |
| switch (bin_power) { |
| case 1: p = bin[0] + cpu; break; |
| case 2: p = bin[1] + cpu; break; |
| case 4: p = bin[2] + cpu; break; |
| case 8: p = bin[3] + cpu; break; |
| case 16: p = bin[4] + cpu; break; |
| case 32: p = bin[5] + cpu; break; |
| case 64: p = bin[6] + cpu; break; |
| case 128: p = bin[7] + cpu; break; |
| default : p = bin[8] + cpu; break; |
| } |
| |
| *p = *p + 1; |
| return; |
| } |
| |
| void dhd_lb_stats_update_napi_histo(dhd_pub_t *dhdp, uint32 count) |
| { |
| int cpu; |
| dhd_info_t *dhd = dhdp->info; |
| |
| cpu = get_cpu(); |
| put_cpu(); |
| dhd_lb_stats_update_histo(dhd->napi_rx_hist, count, cpu); |
| |
| return; |
| } |
| |
| void dhd_lb_stats_update_txc_histo(dhd_pub_t *dhdp, uint32 count) |
| { |
| int cpu; |
| dhd_info_t *dhd = dhdp->info; |
| |
| cpu = get_cpu(); |
| put_cpu(); |
| dhd_lb_stats_update_histo(dhd->txc_hist, count, cpu); |
| |
| return; |
| } |
| |
| void dhd_lb_stats_update_rxc_histo(dhd_pub_t *dhdp, uint32 count) |
| { |
| int cpu; |
| dhd_info_t *dhd = dhdp->info; |
| |
| cpu = get_cpu(); |
| put_cpu(); |
| dhd_lb_stats_update_histo(dhd->rxc_hist, count, cpu); |
| |
| return; |
| } |
| |
| void dhd_lb_stats_txc_percpu_cnt_incr(dhd_pub_t *dhdp) |
| { |
| dhd_info_t *dhd = dhdp->info; |
| DHD_LB_STATS_PERCPU_ARR_INCR(dhd->txc_percpu_run_cnt); |
| } |
| |
| void dhd_lb_stats_rxc_percpu_cnt_incr(dhd_pub_t *dhdp) |
| { |
| dhd_info_t *dhd = dhdp->info; |
| DHD_LB_STATS_PERCPU_ARR_INCR(dhd->rxc_percpu_run_cnt); |
| } |
| #endif /* DHD_LB_STATS */ |
| |
| /** |
| * dhd_tasklet_schedule - Function that runs in IPI context of the destination |
| * CPU and schedules a tasklet. |
| * @tasklet: opaque pointer to the tasklet |
| */ |
| INLINE void |
| dhd_tasklet_schedule(void *tasklet) |
| { |
| tasklet_schedule((struct tasklet_struct *)tasklet); |
| } |
| |
| /** |
| * dhd_work_schedule_on - Executes the passed work in a given CPU |
| * @work: work to be scheduled |
| * @on_cpu: cpu core id |
| * |
| * If the requested cpu is online, then an IPI is sent to this cpu via the |
| * schedule_work_on and the work function |
| * will be invoked to schedule the specified work on the requested CPU. |
| */ |
| |
| INLINE void |
| dhd_work_schedule_on(struct work_struct *work, int on_cpu) |
| { |
| schedule_work_on(on_cpu, work); |
| } |
| |
| INLINE void |
| dhd_delayed_work_schedule_on(struct delayed_work *dwork, int on_cpu, ulong delay) |
| { |
| schedule_delayed_work_on(on_cpu, dwork, delay); |
| } |
| |
| #if defined(DHD_LB_TXP) |
| void dhd_tx_dispatcher_work(struct work_struct * work) |
| { |
| struct dhd_info *dhd; |
| |
| GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST(); |
| dhd = container_of(work, struct dhd_info, tx_dispatcher_work); |
| GCC_DIAGNOSTIC_POP(); |
| |
| dhd_tasklet_schedule(&dhd->tx_tasklet); |
| } |
| |
| /** |
| * dhd_lb_tx_dispatch - load balance by dispatching the tx_tasklet |
| * on another cpu. The tx_tasklet will take care of actually putting |
| * the skbs into appropriate flow ring and ringing H2D interrupt |
| * |
| * @dhdp: pointer to dhd_pub object |
| */ |
| void |
| dhd_lb_tx_dispatch(dhd_pub_t *dhdp) |
| { |
| dhd_info_t *dhd = dhdp->info; |
| int curr_cpu; |
| int tx_cpu; |
| int prev_net_tx_cpu; |
| |
| /* |
| * Get cpu will disable pre-ermption and will not allow any cpu to go offline |
| * and call put_cpu() only after scheduling rx_napi_dispatcher_work. |
| */ |
| curr_cpu = get_cpu(); |
| |
| /* Record the CPU in which the TX request from Network stack came */ |
| prev_net_tx_cpu = atomic_read(&dhd->net_tx_cpu); |
| atomic_set(&dhd->net_tx_cpu, curr_cpu); |
| |
| tx_cpu = atomic_read(&dhd->tx_cpu); |
| |
| /* |
| * Avoid cpu candidacy, if override is set via sysfs for changing cpu mannually |
| */ |
| if (dhd->dhd_lb_candidacy_override) { |
| if (!cpu_online(tx_cpu)) { |
| tx_cpu = curr_cpu; |
| } |
| } else { |
| /* |
| * Now if the NET TX has scheduled in the same CPU |
| * that is chosen for Tx processing |
| * OR scheduled on different cpu than previously it was scheduled, |
| * OR if tx_cpu is offline, |
| * Call cpu candidacy algorithm to recompute tx_cpu. |
| */ |
| if ((curr_cpu == tx_cpu) || (curr_cpu != prev_net_tx_cpu) || |
| !cpu_online(tx_cpu)) { |
| /* Re compute LB CPUs */ |
| dhd_select_cpu_candidacy(dhd); |
| /* Use updated tx cpu */ |
| tx_cpu = atomic_read(&dhd->tx_cpu); |
| } |
| } |
| /* |
| * Schedule tx_dispatcher_work to on the cpu which |
| * in turn will schedule tx_tasklet. |
| */ |
| dhd_work_schedule_on(&dhd->tx_dispatcher_work, tx_cpu); |
| |
| put_cpu(); |
| } |
| #endif /* DHD_LB_TXP */ |
| |
| #if defined(DHD_LB_RXP) |
| |
| /** |
| * dhd_napi_poll - Load balance napi poll function to process received |
| * packets and send up the network stack using netif_receive_skb() |
| * |
| * @napi: napi object in which context this poll function is invoked |
| * @budget: number of packets to be processed. |
| * |
| * Fetch the dhd_info given the rx_napi_struct. Move all packets from the |
| * rx_napi_queue into a local rx_process_queue (lock and queue move and unlock). |
| * Dequeue each packet from head of rx_process_queue, fetch the ifid from the |
| * packet tag and sendup. |
| */ |
| int |
| dhd_napi_poll(struct napi_struct *napi, int budget) |
| { |
| int ifid; |
| const int pkt_count = 1; |
| const int chan = 0; |
| struct sk_buff * skb; |
| unsigned long flags; |
| struct dhd_info *dhd; |
| int processed = 0; |
| int dpc_cpu; |
| #ifdef DHD_LB_STATS |
| uint32 napi_latency; |
| #endif /* DHD_LB_STATS */ |
| |
| GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST(); |
| dhd = container_of(napi, struct dhd_info, rx_napi_struct); |
| GCC_DIAGNOSTIC_POP(); |
| |
| #ifdef DHD_LB_STATS |
| napi_latency = (uint32)(OSL_SYSUPTIME_US() - dhd->napi_schedule_time); |
| dhd_lb_stats_update_napi_latency(dhd->napi_latency, napi_latency); |
| #endif /* DHD_LB_STATS */ |
| DHD_LB_INFO(("%s napi_queue<%d> budget<%d>\n", |
| __FUNCTION__, skb_queue_len(&dhd->rx_napi_queue), budget)); |
| |
| /* |
| * Extract the entire rx_napi_queue into another rx_process_queue |
| * and process only 'budget' number of skbs from rx_process_queue. |
| * If there are more items to be processed, napi poll will be rescheduled |
| * During the next iteration, next set of skbs from |
| * rx_napi_queue will be extracted and attached to the tail of rx_process_queue. |
| * Again budget number of skbs will be processed from rx_process_queue. |
| * If there are less than budget number of skbs in rx_process_queue, |
| * call napi_complete to stop rescheduling napi poll. |
| */ |
| DHD_RX_NAPI_QUEUE_LOCK(&dhd->rx_napi_queue.lock, flags); |
| skb_queue_splice_tail_init(&dhd->rx_napi_queue, &dhd->rx_process_queue); |
| DHD_RX_NAPI_QUEUE_UNLOCK(&dhd->rx_napi_queue.lock, flags); |
| |
| while ((processed < budget) && (skb = __skb_dequeue(&dhd->rx_process_queue)) != NULL) { |
| OSL_PREFETCH(skb->data); |
| |
| ifid = DHD_PKTTAG_IFID((dhd_pkttag_fr_t *)PKTTAG(skb)); |
| |
| DHD_LB_INFO(("%s dhd_rx_frame pkt<%p> ifid<%d>\n", |
| __FUNCTION__, skb, ifid)); |
| |
| dhd_rx_frame(&dhd->pub, ifid, skb, pkt_count, chan); |
| processed++; |
| } |
| |
| if (atomic_read(&dhd->pub.lb_rxp_flow_ctrl) && |
| (dhd_lb_rxp_process_qlen(&dhd->pub) <= dhd->pub.lb_rxp_strt_thr)) { |
| /* |
| * If the dpc CPU is online Schedule dhd_dpc_dispatcher_work on the dpc cpu which |
| * in turn will schedule dpc tasklet. Else schedule dpc takslet. |
| */ |
| get_cpu(); |
| dpc_cpu = atomic_read(&dhd->dpc_cpu); |
| if (!cpu_online(dpc_cpu)) { |
| dhd_tasklet_schedule(&dhd->tasklet); |
| } else { |
| dhd_delayed_work_schedule_on(&dhd->dhd_dpc_dispatcher_work, dpc_cpu, 0); |
| } |
| put_cpu(); |
| } |
| DHD_LB_STATS_UPDATE_NAPI_HISTO(&dhd->pub, processed); |
| |
| DHD_LB_INFO(("%s processed %d\n", __FUNCTION__, processed)); |
| |
| /* |
| * Signal napi complete only when no more packets are processed and |
| * none are left in the enqueued queue. |
| */ |
| if ((processed == 0) && (skb_queue_len(&dhd->rx_napi_queue) == 0)) { |
| napi_complete(napi); |
| #ifdef DHD_LB_STATS |
| dhd->pub.lb_rxp_napi_complete_cnt++; |
| #endif /* DHD_LB_STATS */ |
| DHD_GENERAL_LOCK(&dhd->pub, flags); |
| DHD_BUS_BUSY_CLEAR_IN_NAPI(&dhd->pub); |
| DHD_GENERAL_UNLOCK(&dhd->pub, flags); |
| return 0; |
| } |
| |
| #ifdef DHD_LB_STATS |
| dhd->napi_schedule_time = OSL_SYSUPTIME_US(); |
| #endif /* DHD_LB_STATS */ |
| |
| /* Return budget so that it gets rescheduled immediately */ |
| return budget; |
| } |
| |
| /** |
| * dhd_napi_schedule - Place the napi struct into the current cpus softnet napi |
| * poll list. This function may be invoked via the smp_call_function_single |
| * from a remote CPU. |
| * |
| * This function will essentially invoke __raise_softirq_irqoff(NET_RX_SOFTIRQ) |
| * after the napi_struct is added to the softnet data's poll_list |
| * |
| * @info: pointer to a dhd_info struct |
| */ |
| static void |
| dhd_napi_schedule(void *info) |
| { |
| dhd_info_t *dhd = (dhd_info_t *)info; |
| unsigned long flags; |
| |
| DHD_INFO(("%s rx_napi_struct<%p> on cpu<%d>\n", |
| __FUNCTION__, &dhd->rx_napi_struct, atomic_read(&dhd->rx_napi_cpu))); |
| |
| /* add napi_struct to softnet data poll list and raise NET_RX_SOFTIRQ */ |
| if (napi_schedule_prep(&dhd->rx_napi_struct)) { |
| |
| /* |
| * Set busbusystate in NAPI, which will be cleared after |
| * napi_complete from napi_poll context |
| */ |
| DHD_GENERAL_LOCK(&dhd->pub, flags); |
| DHD_BUS_BUSY_SET_IN_NAPI(&dhd->pub); |
| DHD_GENERAL_UNLOCK(&dhd->pub, flags); |
| |
| #ifdef DHD_LB_STATS |
| dhd->napi_schedule_time = OSL_SYSUPTIME_US(); |
| dhd->pub.lb_rxp_napi_sched_cnt++; |
| #endif /* DHD_LB_STATS */ |
| __napi_schedule(&dhd->rx_napi_struct); |
| #ifdef WAKEUP_KSOFTIRQD_POST_NAPI_SCHEDULE |
| raise_softirq(NET_RX_SOFTIRQ); |
| #endif /* WAKEUP_KSOFTIRQD_POST_NAPI_SCHEDULE */ |
| } |
| |
| /* |
| * If the rx_napi_struct was already running, then we let it complete |
| * processing all its packets. The rx_napi_struct may only run on one |
| * core at a time, to avoid out-of-order handling. |
| */ |
| } |
| |
| /** |
| * dhd_napi_schedule_on - API to schedule on a desired CPU core a NET_RX_SOFTIRQ |
| * action after placing the dhd's rx_process napi object in the the remote CPU's |
| * softnet data's poll_list. |
| * |
| * @dhd: dhd_info which has the rx_process napi object |
| * @on_cpu: desired remote CPU id |
| */ |
| static INLINE int |
| dhd_napi_schedule_on(dhd_info_t *dhd, int on_cpu) |
| { |
| int wait = 0; /* asynchronous IPI */ |
| DHD_INFO(("%s dhd<%p> napi<%p> on_cpu<%d>\n", |
| __FUNCTION__, dhd, &dhd->rx_napi_struct, on_cpu)); |
| |
| if (smp_call_function_single(on_cpu, dhd_napi_schedule, dhd, wait)) { |
| DHD_ERROR(("%s smp_call_function_single on_cpu<%d> failed\n", |
| __FUNCTION__, on_cpu)); |
| } |
| |
| DHD_LB_STATS_INCR(dhd->napi_sched_cnt); |
| |
| return 0; |
| } |
| |
| /* |
| * Call get_online_cpus/put_online_cpus around dhd_napi_schedule_on |
| * Why should we do this? |
| * The candidacy algorithm is run from the call back function |
| * registered to CPU hotplug notifier. This call back happens from Worker |
| * context. The dhd_napi_schedule_on is also from worker context. |
| * Note that both of this can run on two different CPUs at the same time. |
| * So we can possibly have a window where a given CPUn is being brought |
| * down from CPUm while we try to run a function on CPUn. |
| * To prevent this its better have the whole code to execute an SMP |
| * function under get_online_cpus. |
| * This function call ensures that hotplug mechanism does not kick-in |
| * until we are done dealing with online CPUs |
| * If the hotplug worker is already running, no worries because the |
| * candidacy algo would then reflect the same in dhd->rx_napi_cpu. |
| * |
| * The below mentioned code structure is proposed in |
| * https://www.kernel.org/doc/Documentation/cpu-hotplug.txt |
| * for the question |
| * Q: I need to ensure that a particular cpu is not removed when there is some |
| * work specific to this cpu is in progress |
| * |
| * According to the documentation calling get_online_cpus is NOT required, if |
| * we are running from tasklet context. Since dhd_rx_napi_dispatcher_work can |
| * run from Work Queue context we have to call these functions |
| */ |
| void dhd_rx_napi_dispatcher_work(struct work_struct * work) |
| { |
| struct dhd_info *dhd; |
| GCC_DIAGNOSTIC_PUSH_SUPPRESS_CAST(); |
| dhd = container_of(work, struct dhd_info, rx_napi_dispatcher_work); |
| GCC_DIAGNOSTIC_POP(); |
| |
| dhd_napi_schedule(dhd); |
| } |
| |
| /** |
| * dhd_lb_rx_napi_dispatch - load balance by dispatching the rx_napi_struct |
| * to run on another CPU. The rx_napi_struct's poll function will retrieve all |
| * the packets enqueued into the rx_napi_queue and sendup. |
| * The producer's rx packet queue is appended to the rx_napi_queue before |
| * dispatching the rx_napi_struct. |
| */ |
| void |
| dhd_lb_rx_napi_dispatch(dhd_pub_t *dhdp) |
| { |
| unsigned long flags; |
| dhd_info_t *dhd = dhdp->info; |
| int curr_cpu; |
| int rx_napi_cpu; |
| int prev_dpc_cpu; |
| |
| if (dhd->rx_napi_netdev == NULL) { |
| DHD_ERROR(("%s: dhd->rx_napi_netdev is NULL\n", __FUNCTION__)); |
| return; |
| } |
| |
| DHD_LB_INFO(("%s append napi_queue<%d> pend_queue<%d>\n", __FUNCTION__, |
| skb_queue_len(&dhd->rx_napi_queue), skb_queue_len(&dhd->rx_pend_queue))); |
| |
| /* append the producer's queue of packets to the napi's rx process queue */ |
| DHD_RX_NAPI_QUEUE_LOCK(&dhd->rx_napi_queue.lock, flags); |
| skb_queue_splice_tail_init(&dhd->rx_pend_queue, &dhd->rx_napi_queue); |
| DHD_RX_NAPI_QUEUE_UNLOCK(&dhd->rx_napi_queue.lock, flags); |
| |
| /* If sysfs lb_rxp_active is not set, schedule on current cpu */ |
| if (!atomic_read(&dhd->lb_rxp_active)) |
| { |
| dhd_napi_schedule(dhd); |
| return; |
| } |
| |
| /* |
| * Get cpu will disable pre-ermption and will not allow any cpu to go offline |
| * and call put_cpu() only after scheduling rx_napi_dispatcher_work. |
| */ |
| curr_cpu = get_cpu(); |
| |
| prev_dpc_cpu = atomic_read(&dhd->prev_dpc_cpu); |
| |
| rx_napi_cpu = atomic_read(&dhd->rx_napi_cpu); |
| |
| /* |
| * Avoid cpu candidacy, if override is set via sysfs for changing cpu mannually |
| */ |
| if (dhd->dhd_lb_candidacy_override) { |
| if (!cpu_online(rx_napi_cpu)) { |
| rx_napi_cpu = curr_cpu; |
| } |
| } else { |
| /* |
| * Now if the DPC has scheduled in the same CPU |
| * that is chosen for Rx napi processing |
| * OR scheduled on different cpu than previously it was scheduled, |
| * OR if rx_napi_cpu is offline, |
| * Call cpu candidacy algorithm to recompute napi_cpu. |
| */ |
| if ((curr_cpu == rx_napi_cpu) || (curr_cpu != prev_dpc_cpu) || |
| !cpu_online(rx_napi_cpu)) { |
| /* Re compute LB CPUs */ |
| dhd_select_cpu_candidacy(dhd); |
| /* Use updated napi cpu */ |
| rx_napi_cpu = atomic_read(&dhd->rx_napi_cpu); |
| } |
| |
| } |
| |
| DHD_LB_INFO(("%s : schedule to curr_cpu : %d, rx_napi_cpu : %d\n", |
| __FUNCTION__, curr_cpu, rx_napi_cpu)); |
| dhd_work_schedule_on(&dhd->rx_napi_dispatcher_work, rx_napi_cpu); |
| DHD_LB_STATS_INCR(dhd->napi_sched_cnt); |
| |
| put_cpu(); |
| } |
| |
| /** |
| * dhd_lb_rx_pkt_enqueue - Enqueue the packet into the producer's queue |
| */ |
| void |
| dhd_lb_rx_pkt_enqueue(dhd_pub_t *dhdp, void *pkt, int ifidx) |
| { |
| dhd_info_t *dhd = dhdp->info; |
| |
| DHD_LB_INFO(("%s enqueue pkt<%p> ifidx<%d> pend_queue<%d>\n", __FUNCTION__, |
| pkt, ifidx, skb_queue_len(&dhd->rx_pend_queue))); |
| DHD_PKTTAG_SET_IFID((dhd_pkttag_fr_t *)PKTTAG(pkt), ifidx); |
| __skb_queue_tail(&dhd->rx_pend_queue, pkt); |
| DHD_LB_STATS_PERCPU_ARR_INCR(dhd->napi_percpu_run_cnt); |
| } |
| |
| unsigned long |
| dhd_read_lb_rxp(dhd_pub_t *dhdp) |
| { |
| dhd_info_t *dhd = dhdp->info; |
| return atomic_read(&dhd->lb_rxp_active); |
| } |
| |
| uint32 |
| dhd_lb_rxp_process_qlen(dhd_pub_t *dhdp) |
| { |
| dhd_info_t *dhd = dhdp->info; |
| return skb_queue_len(&dhd->rx_process_queue); |
| } |
| #endif /* DHD_LB_RXP */ |
| |
| #if defined(DHD_LB_TXP) |
| int |
| BCMFASTPATH(dhd_lb_sendpkt)(dhd_info_t *dhd, struct net_device *net, |
| int ifidx, void *skb) |
| { |
| DHD_LB_STATS_PERCPU_ARR_INCR(dhd->tx_start_percpu_run_cnt); |
| |
| /* If the feature is disabled run-time do TX from here */ |
| if (atomic_read(&dhd->lb_txp_active) == 0) { |
| DHD_LB_STATS_PERCPU_ARR_INCR(dhd->txp_percpu_run_cnt); |
| return __dhd_sendpkt(&dhd->pub, ifidx, skb); |
| } |
| |
| /* Store the address of net device and interface index in the Packet tag */ |
| DHD_LB_TX_PKTTAG_SET_NETDEV((dhd_tx_lb_pkttag_fr_t *)PKTTAG(skb), net); |
| DHD_LB_TX_PKTTAG_SET_IFIDX((dhd_tx_lb_pkttag_fr_t *)PKTTAG(skb), ifidx); |
| |
| /* Enqueue the skb into tx_pend_queue */ |
| skb_queue_tail(&dhd->tx_pend_queue, skb); |
| |
| DHD_TRACE(("%s(): Added skb %p for netdev %p \r\n", __FUNCTION__, skb, net)); |
| |
| /* Dispatch the Tx job to be processed by the tx_tasklet */ |
| dhd_lb_tx_dispatch(&dhd->pub); |
| |
| return NETDEV_TX_OK; |
| } |
| #endif /* DHD_LB_TXP */ |
| |
| #ifdef DHD_LB_TXP |
| #define DHD_LB_TXBOUND 64 |
| /* |
| * Function that performs the TX processing on a given CPU |
| */ |
| bool |
| dhd_lb_tx_process(dhd_info_t *dhd) |
| { |
| struct sk_buff *skb; |
| int cnt = 0; |
| struct net_device *net; |
| int ifidx; |
| bool resched = FALSE; |
| |
| DHD_TRACE(("%s(): TX Processing \r\n", __FUNCTION__)); |
| if (dhd == NULL) { |
| DHD_ERROR((" Null pointer DHD \r\n")); |
| return resched; |
| } |
| |
| BCM_REFERENCE(net); |
| |
| DHD_LB_STATS_PERCPU_ARR_INCR(dhd->txp_percpu_run_cnt); |
| |
| /* Base Loop to perform the actual Tx */ |
| do { |
| skb = skb_dequeue(&dhd->tx_pend_queue); |
| if (skb == NULL) { |
| DHD_TRACE(("Dequeued a Null Packet \r\n")); |
| break; |
| } |
| cnt++; |
| |
| net = DHD_LB_TX_PKTTAG_NETDEV((dhd_tx_lb_pkttag_fr_t *)PKTTAG(skb)); |
| ifidx = DHD_LB_TX_PKTTAG_IFIDX((dhd_tx_lb_pkttag_fr_t *)PKTTAG(skb)); |
| |
| DHD_TRACE(("Processing skb %p for net %p index %d \r\n", skb, |
| net, ifidx)); |
| |
| __dhd_sendpkt(&dhd->pub, ifidx, skb); |
| |
| if (cnt >= DHD_LB_TXBOUND) { |
| resched = TRUE; |
| break; |
| } |
| |
| } while (1); |
| |
| DHD_LB_INFO(("%s(): Processed %d packets \r\n", __FUNCTION__, cnt)); |
| |
| return resched; |
| } |
| |
| void |
| dhd_lb_tx_handler(unsigned long data) |
| { |
| dhd_info_t *dhd = (dhd_info_t *)data; |
| |
| if (dhd_lb_tx_process(dhd)) { |
| dhd_tasklet_schedule(&dhd->tx_tasklet); |
| } |
| } |
| |
| #endif /* DHD_LB_TXP */ |
| #endif /* DHD_LB */ |
| |
| #if defined(DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON) |
| void |
| dhd_irq_set_affinity(dhd_pub_t *dhdp, const struct cpumask *cpumask) |
| { |
| unsigned int irq = (unsigned int)-1; |
| int err = BCME_OK; |
| |
| if (!dhdp) { |
| DHD_ERROR(("%s : dhdp is NULL\n", __FUNCTION__)); |
| return; |
| } |
| |
| if (!dhdp->bus) { |
| DHD_ERROR(("%s : bus is NULL\n", __FUNCTION__)); |
| return; |
| } |
| |
| DHD_ERROR(("%s : irq set affinity cpu:0x%lx\n", |
| __FUNCTION__, *cpumask_bits(cpumask))); |
| |
| dhdpcie_get_pcieirq(dhdp->bus, &irq); |
| #ifdef BCMDHD_MODULAR |
| err = irq_set_affinity_hint(irq, cpumask); |
| #else |
| err = irq_set_affinity(irq, cpumask); |
| #endif /* BCMDHD_MODULAR */ |
| if (err) |
| DHD_ERROR(("%s : irq set affinity is failed cpu:0x%lx\n", |
| __FUNCTION__, *cpumask_bits(cpumask))); |
| } |
| #endif /* DHD_CONTROL_PCIE_CPUCORE_WIFI_TURNON */ |