Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1 | /* |
Saeed Mahameed | 302bdf6 | 2015-04-02 17:07:29 +0300 | [diff] [blame] | 2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
| 33 | #include <linux/kernel.h> |
| 34 | #include <linux/module.h> |
| 35 | #include <linux/random.h> |
| 36 | #include <linux/vmalloc.h> |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 37 | #include <linux/hardirq.h> |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 38 | #include <linux/mlx5/driver.h> |
| 39 | #include <linux/mlx5/cmd.h> |
| 40 | #include "mlx5_core.h" |
| 41 | |
| 42 | enum { |
| 43 | MLX5_HEALTH_POLL_INTERVAL = 2 * HZ, |
| 44 | MAX_MISSES = 3, |
| 45 | }; |
| 46 | |
| 47 | enum { |
| 48 | MLX5_HEALTH_SYNDR_FW_ERR = 0x1, |
| 49 | MLX5_HEALTH_SYNDR_IRISC_ERR = 0x7, |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 50 | MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR = 0x8, |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 51 | MLX5_HEALTH_SYNDR_CRC_ERR = 0x9, |
| 52 | MLX5_HEALTH_SYNDR_FETCH_PCI_ERR = 0xa, |
| 53 | MLX5_HEALTH_SYNDR_HW_FTL_ERR = 0xb, |
| 54 | MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR = 0xc, |
| 55 | MLX5_HEALTH_SYNDR_EQ_ERR = 0xd, |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 56 | MLX5_HEALTH_SYNDR_EQ_INV = 0xe, |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 57 | MLX5_HEALTH_SYNDR_FFSER_ERR = 0xf, |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 58 | MLX5_HEALTH_SYNDR_HIGH_TEMP = 0x10 |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 59 | }; |
| 60 | |
Eli Cohen | fd76ee4 | 2015-10-14 17:43:45 +0300 | [diff] [blame] | 61 | enum { |
| 62 | MLX5_NIC_IFC_FULL = 0, |
| 63 | MLX5_NIC_IFC_DISABLED = 1, |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 64 | MLX5_NIC_IFC_NO_DRAM_NIC = 2, |
| 65 | MLX5_NIC_IFC_INVALID = 3 |
Eli Cohen | fd76ee4 | 2015-10-14 17:43:45 +0300 | [diff] [blame] | 66 | }; |
| 67 | |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 68 | enum { |
| 69 | MLX5_DROP_NEW_HEALTH_WORK, |
Mohamad Haj Yahia | e20204d | 2017-03-30 17:09:00 +0300 | [diff] [blame^] | 70 | MLX5_DROP_NEW_RECOVERY_WORK, |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 71 | }; |
| 72 | |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 73 | static u8 get_nic_state(struct mlx5_core_dev *dev) |
Eli Cohen | fd76ee4 | 2015-10-14 17:43:45 +0300 | [diff] [blame] | 74 | { |
| 75 | return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; |
| 76 | } |
| 77 | |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 78 | static void trigger_cmd_completions(struct mlx5_core_dev *dev) |
| 79 | { |
| 80 | unsigned long flags; |
| 81 | u64 vector; |
| 82 | |
| 83 | /* wait for pending handlers to complete */ |
| 84 | synchronize_irq(dev->priv.msix_arr[MLX5_EQ_VEC_CMD].vector); |
| 85 | spin_lock_irqsave(&dev->cmd.alloc_lock, flags); |
| 86 | vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1); |
| 87 | if (!vector) |
| 88 | goto no_trig; |
| 89 | |
| 90 | vector |= MLX5_TRIGGERED_CMD_COMP; |
| 91 | spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); |
| 92 | |
| 93 | mlx5_core_dbg(dev, "vector 0x%llx\n", vector); |
Mohamad Haj Yahia | 4fb5fd2 | 2017-02-23 11:19:36 +0200 | [diff] [blame] | 94 | mlx5_cmd_comp_handler(dev, vector, true); |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 95 | return; |
| 96 | |
| 97 | no_trig: |
| 98 | spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); |
| 99 | } |
| 100 | |
Eli Cohen | fd76ee4 | 2015-10-14 17:43:45 +0300 | [diff] [blame] | 101 | static int in_fatal(struct mlx5_core_dev *dev) |
| 102 | { |
| 103 | struct mlx5_core_health *health = &dev->priv.health; |
| 104 | struct health_buffer __iomem *h = health->health; |
| 105 | |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 106 | if (get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) |
Eli Cohen | fd76ee4 | 2015-10-14 17:43:45 +0300 | [diff] [blame] | 107 | return 1; |
| 108 | |
| 109 | if (ioread32be(&h->fw_ver) == 0xffffffff) |
| 110 | return 1; |
| 111 | |
| 112 | return 0; |
| 113 | } |
| 114 | |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 115 | void mlx5_enter_error_state(struct mlx5_core_dev *dev) |
| 116 | { |
Mohamad Haj Yahia | c1d4d2e | 2016-06-30 17:34:39 +0300 | [diff] [blame] | 117 | mutex_lock(&dev->intf_state_mutex); |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 118 | if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) |
Mohamad Haj Yahia | c1d4d2e | 2016-06-30 17:34:39 +0300 | [diff] [blame] | 119 | goto unlock; |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 120 | |
| 121 | mlx5_core_err(dev, "start\n"); |
Mohamad Haj Yahia | c1d4d2e | 2016-06-30 17:34:39 +0300 | [diff] [blame] | 122 | if (pci_channel_offline(dev->pdev) || in_fatal(dev)) { |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 123 | dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; |
Mohamad Haj Yahia | c1d4d2e | 2016-06-30 17:34:39 +0300 | [diff] [blame] | 124 | trigger_cmd_completions(dev); |
| 125 | } |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 126 | |
| 127 | mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0); |
| 128 | mlx5_core_err(dev, "end\n"); |
Mohamad Haj Yahia | c1d4d2e | 2016-06-30 17:34:39 +0300 | [diff] [blame] | 129 | |
| 130 | unlock: |
| 131 | mutex_unlock(&dev->intf_state_mutex); |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 132 | } |
| 133 | |
| 134 | static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) |
| 135 | { |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 136 | u8 nic_interface = get_nic_state(dev); |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 137 | |
| 138 | switch (nic_interface) { |
| 139 | case MLX5_NIC_IFC_FULL: |
| 140 | mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); |
| 141 | break; |
| 142 | |
| 143 | case MLX5_NIC_IFC_DISABLED: |
| 144 | mlx5_core_warn(dev, "starting teardown\n"); |
| 145 | break; |
| 146 | |
| 147 | case MLX5_NIC_IFC_NO_DRAM_NIC: |
| 148 | mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n"); |
| 149 | break; |
| 150 | default: |
| 151 | mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n", |
| 152 | nic_interface); |
| 153 | } |
| 154 | |
| 155 | mlx5_disable_device(dev); |
| 156 | } |
| 157 | |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 158 | static void health_recover(struct work_struct *work) |
| 159 | { |
| 160 | struct mlx5_core_health *health; |
| 161 | struct delayed_work *dwork; |
| 162 | struct mlx5_core_dev *dev; |
| 163 | struct mlx5_priv *priv; |
| 164 | u8 nic_state; |
| 165 | |
| 166 | dwork = container_of(work, struct delayed_work, work); |
| 167 | health = container_of(dwork, struct mlx5_core_health, recover_work); |
| 168 | priv = container_of(health, struct mlx5_priv, health); |
| 169 | dev = container_of(priv, struct mlx5_core_dev, priv); |
| 170 | |
| 171 | nic_state = get_nic_state(dev); |
| 172 | if (nic_state == MLX5_NIC_IFC_INVALID) { |
| 173 | dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n"); |
| 174 | return; |
| 175 | } |
| 176 | |
| 177 | dev_err(&dev->pdev->dev, "starting health recovery flow\n"); |
| 178 | mlx5_recover_device(dev); |
| 179 | } |
| 180 | |
| 181 | /* How much time to wait until health resetting the driver (in msecs) */ |
| 182 | #define MLX5_RECOVERY_DELAY_MSECS 60000 |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 183 | static void health_care(struct work_struct *work) |
| 184 | { |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 185 | unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS); |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 186 | struct mlx5_core_health *health; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 187 | struct mlx5_core_dev *dev; |
| 188 | struct mlx5_priv *priv; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 189 | |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 190 | health = container_of(work, struct mlx5_core_health, work); |
| 191 | priv = container_of(health, struct mlx5_priv, health); |
| 192 | dev = container_of(priv, struct mlx5_core_dev, priv); |
| 193 | mlx5_core_warn(dev, "handling bad device here\n"); |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 194 | mlx5_handle_bad_state(dev); |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 195 | |
| 196 | spin_lock(&health->wq_lock); |
Mohamad Haj Yahia | e20204d | 2017-03-30 17:09:00 +0300 | [diff] [blame^] | 197 | if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags)) |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 198 | schedule_delayed_work(&health->recover_work, recover_delay); |
| 199 | else |
| 200 | dev_err(&dev->pdev->dev, |
| 201 | "new health works are not permitted at this stage\n"); |
| 202 | spin_unlock(&health->wq_lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 203 | } |
| 204 | |
| 205 | static const char *hsynd_str(u8 synd) |
| 206 | { |
| 207 | switch (synd) { |
| 208 | case MLX5_HEALTH_SYNDR_FW_ERR: |
| 209 | return "firmware internal error"; |
| 210 | case MLX5_HEALTH_SYNDR_IRISC_ERR: |
| 211 | return "irisc not responding"; |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 212 | case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR: |
| 213 | return "unrecoverable hardware error"; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 214 | case MLX5_HEALTH_SYNDR_CRC_ERR: |
| 215 | return "firmware CRC error"; |
| 216 | case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR: |
| 217 | return "ICM fetch PCI error"; |
| 218 | case MLX5_HEALTH_SYNDR_HW_FTL_ERR: |
| 219 | return "HW fatal error\n"; |
| 220 | case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR: |
| 221 | return "async EQ buffer overrun"; |
| 222 | case MLX5_HEALTH_SYNDR_EQ_ERR: |
| 223 | return "EQ error"; |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 224 | case MLX5_HEALTH_SYNDR_EQ_INV: |
Masanari Iida | c01e015 | 2016-04-20 00:27:33 +0900 | [diff] [blame] | 225 | return "Invalid EQ referenced"; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 226 | case MLX5_HEALTH_SYNDR_FFSER_ERR: |
| 227 | return "FFSER error"; |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 228 | case MLX5_HEALTH_SYNDR_HIGH_TEMP: |
Masanari Iida | c01e015 | 2016-04-20 00:27:33 +0900 | [diff] [blame] | 229 | return "High temperature"; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 230 | default: |
| 231 | return "unrecognized error"; |
| 232 | } |
| 233 | } |
| 234 | |
Eli Cohen | 0144a95 | 2015-10-08 17:13:59 +0300 | [diff] [blame] | 235 | static u16 get_maj(u32 fw) |
Roland Dreier | 582c016 | 2013-07-08 10:52:28 -0700 | [diff] [blame] | 236 | { |
Eli Cohen | 0144a95 | 2015-10-08 17:13:59 +0300 | [diff] [blame] | 237 | return fw >> 28; |
Roland Dreier | 582c016 | 2013-07-08 10:52:28 -0700 | [diff] [blame] | 238 | } |
| 239 | |
Eli Cohen | 0144a95 | 2015-10-08 17:13:59 +0300 | [diff] [blame] | 240 | static u16 get_min(u32 fw) |
Roland Dreier | 582c016 | 2013-07-08 10:52:28 -0700 | [diff] [blame] | 241 | { |
Eli Cohen | 0144a95 | 2015-10-08 17:13:59 +0300 | [diff] [blame] | 242 | return fw >> 16 & 0xfff; |
| 243 | } |
| 244 | |
| 245 | static u16 get_sub(u32 fw) |
| 246 | { |
| 247 | return fw & 0xffff; |
Roland Dreier | 582c016 | 2013-07-08 10:52:28 -0700 | [diff] [blame] | 248 | } |
| 249 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 250 | static void print_health_info(struct mlx5_core_dev *dev) |
| 251 | { |
| 252 | struct mlx5_core_health *health = &dev->priv.health; |
| 253 | struct health_buffer __iomem *h = health->health; |
Eli Cohen | 0144a95 | 2015-10-08 17:13:59 +0300 | [diff] [blame] | 254 | char fw_str[18]; |
| 255 | u32 fw; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 256 | int i; |
| 257 | |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 258 | /* If the syndrom is 0, the device is OK and no need to print buffer */ |
| 259 | if (!ioread8(&h->synd)) |
| 260 | return; |
| 261 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 262 | for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) |
Eli Cohen | 0144a95 | 2015-10-08 17:13:59 +0300 | [diff] [blame] | 263 | dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 264 | |
Eli Cohen | 0144a95 | 2015-10-08 17:13:59 +0300 | [diff] [blame] | 265 | dev_err(&dev->pdev->dev, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr)); |
| 266 | dev_err(&dev->pdev->dev, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra)); |
| 267 | fw = ioread32be(&h->fw_ver); |
| 268 | sprintf(fw_str, "%d.%d.%d", get_maj(fw), get_min(fw), get_sub(fw)); |
| 269 | dev_err(&dev->pdev->dev, "fw_ver %s\n", fw_str); |
| 270 | dev_err(&dev->pdev->dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id)); |
| 271 | dev_err(&dev->pdev->dev, "irisc_index %d\n", ioread8(&h->irisc_index)); |
| 272 | dev_err(&dev->pdev->dev, "synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd))); |
| 273 | dev_err(&dev->pdev->dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 274 | } |
| 275 | |
Eli Cohen | fd76ee4 | 2015-10-14 17:43:45 +0300 | [diff] [blame] | 276 | static unsigned long get_next_poll_jiffies(void) |
| 277 | { |
| 278 | unsigned long next; |
| 279 | |
| 280 | get_random_bytes(&next, sizeof(next)); |
| 281 | next %= HZ; |
| 282 | next += jiffies + MLX5_HEALTH_POLL_INTERVAL; |
| 283 | |
| 284 | return next; |
| 285 | } |
| 286 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 287 | static void poll_health(unsigned long data) |
| 288 | { |
| 289 | struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data; |
| 290 | struct mlx5_core_health *health = &dev->priv.health; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 291 | u32 count; |
| 292 | |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 293 | if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 294 | mod_timer(&health->timer, get_next_poll_jiffies()); |
| 295 | return; |
| 296 | } |
| 297 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 298 | count = ioread32be(health->health_counter); |
| 299 | if (count == health->prev) |
| 300 | ++health->miss_counter; |
| 301 | else |
| 302 | health->miss_counter = 0; |
| 303 | |
| 304 | health->prev = count; |
| 305 | if (health->miss_counter == MAX_MISSES) { |
Eli Cohen | fd76ee4 | 2015-10-14 17:43:45 +0300 | [diff] [blame] | 306 | dev_err(&dev->pdev->dev, "device's health compromised - reached miss count\n"); |
| 307 | print_health_info(dev); |
| 308 | } else { |
| 309 | mod_timer(&health->timer, get_next_poll_jiffies()); |
| 310 | } |
| 311 | |
| 312 | if (in_fatal(dev) && !health->sick) { |
| 313 | health->sick = true; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 314 | print_health_info(dev); |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 315 | spin_lock(&health->wq_lock); |
| 316 | if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) |
| 317 | queue_work(health->wq, &health->work); |
| 318 | else |
| 319 | dev_err(&dev->pdev->dev, |
| 320 | "new health works are not permitted at this stage\n"); |
| 321 | spin_unlock(&health->wq_lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 322 | } |
| 323 | } |
| 324 | |
| 325 | void mlx5_start_health_poll(struct mlx5_core_dev *dev) |
| 326 | { |
| 327 | struct mlx5_core_health *health = &dev->priv.health; |
| 328 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 329 | init_timer(&health->timer); |
Mohamad Haj Yahia | 2241007 | 2016-10-25 18:36:32 +0300 | [diff] [blame] | 330 | health->sick = 0; |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 331 | clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); |
Mohamad Haj Yahia | e20204d | 2017-03-30 17:09:00 +0300 | [diff] [blame^] | 332 | clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 333 | health->health = &dev->iseg->health; |
| 334 | health->health_counter = &dev->iseg->health_counter; |
| 335 | |
| 336 | health->timer.data = (unsigned long)dev; |
| 337 | health->timer.function = poll_health; |
| 338 | health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL); |
| 339 | add_timer(&health->timer); |
| 340 | } |
| 341 | |
| 342 | void mlx5_stop_health_poll(struct mlx5_core_dev *dev) |
| 343 | { |
| 344 | struct mlx5_core_health *health = &dev->priv.health; |
| 345 | |
| 346 | del_timer_sync(&health->timer); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 347 | } |
| 348 | |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 349 | void mlx5_drain_health_wq(struct mlx5_core_dev *dev) |
| 350 | { |
| 351 | struct mlx5_core_health *health = &dev->priv.health; |
| 352 | |
| 353 | spin_lock(&health->wq_lock); |
| 354 | set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); |
Mohamad Haj Yahia | e20204d | 2017-03-30 17:09:00 +0300 | [diff] [blame^] | 355 | set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 356 | spin_unlock(&health->wq_lock); |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 357 | cancel_delayed_work_sync(&health->recover_work); |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 358 | cancel_work_sync(&health->work); |
| 359 | } |
| 360 | |
Mohamad Haj Yahia | e20204d | 2017-03-30 17:09:00 +0300 | [diff] [blame^] | 361 | void mlx5_drain_health_recovery(struct mlx5_core_dev *dev) |
| 362 | { |
| 363 | struct mlx5_core_health *health = &dev->priv.health; |
| 364 | |
| 365 | spin_lock(&health->wq_lock); |
| 366 | set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); |
| 367 | spin_unlock(&health->wq_lock); |
| 368 | cancel_delayed_work_sync(&dev->priv.health.recover_work); |
| 369 | } |
| 370 | |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 371 | void mlx5_health_cleanup(struct mlx5_core_dev *dev) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 372 | { |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 373 | struct mlx5_core_health *health = &dev->priv.health; |
| 374 | |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 375 | destroy_workqueue(health->wq); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 376 | } |
| 377 | |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 378 | int mlx5_health_init(struct mlx5_core_dev *dev) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 379 | { |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 380 | struct mlx5_core_health *health; |
| 381 | char *name; |
| 382 | |
| 383 | health = &dev->priv.health; |
| 384 | name = kmalloc(64, GFP_KERNEL); |
| 385 | if (!name) |
| 386 | return -ENOMEM; |
| 387 | |
| 388 | strcpy(name, "mlx5_health"); |
| 389 | strcat(name, dev_name(&dev->pdev->dev)); |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 390 | health->wq = create_singlethread_workqueue(name); |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 391 | kfree(name); |
Mohamad Haj Yahia | 05ac2c0 | 2016-10-25 18:36:33 +0300 | [diff] [blame] | 392 | if (!health->wq) |
| 393 | return -ENOMEM; |
| 394 | spin_lock_init(&health->wq_lock); |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 395 | INIT_WORK(&health->work, health_care); |
Mohamad Haj Yahia | 04c0c1ab | 2016-10-25 18:36:34 +0300 | [diff] [blame] | 396 | INIT_DELAYED_WORK(&health->recover_work, health_recover); |
Eli Cohen | ac6ea6e | 2015-10-08 17:14:00 +0300 | [diff] [blame] | 397 | |
| 398 | return 0; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 399 | } |