ARK-OS/arkos/kernel/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) Meta Platforms, Inc. and affiliates. */

#include <linux/etherdevice.h>
#include <linux/ipv6.h>
#include <linux/types.h>
#include <net/netdev_queues.h>

#include "fbnic.h"
#include "fbnic_netdev.h"
#include "fbnic_txrx.h"

int __fbnic_open(struct fbnic_net *fbn)
{
	struct fbnic_dev *fbd = fbn->fbd;
	int err;

	err = fbnic_alloc_napi_vectors(fbn);
	if (err)
		return err;

	err = fbnic_alloc_resources(fbn);
	if (err)
		goto free_napi_vectors;

	err = fbnic_set_netif_queues(fbn);
	if (err)
		goto free_resources;

	/* Send ownership message and flush to verify FW has seen it */
	err = fbnic_fw_xmit_ownership_msg(fbd, true);
	if (err) {
		dev_warn(fbd->dev,
			 "Error %d sending host ownership message to the firmware\n",
			 err);
		goto err_reset_queues;
	}

	err = fbnic_time_start(fbn);
	if (err)
		goto release_ownership;

	err = fbnic_fw_init_heartbeat(fbd, false);
	if (err)
		goto time_stop;

	err = fbnic_mac_request_irq(fbd);
	if (err)
		goto time_stop;

	/* Pull the BMC config and initialize the RPC */
	fbnic_bmc_rpc_init(fbd);
	fbnic_rss_reinit(fbd, fbn);

	phylink_resume(fbn->phylink);

	return 0;
time_stop:
	fbnic_time_stop(fbn);
release_ownership:
	fbnic_fw_xmit_ownership_msg(fbn->fbd, false);
err_reset_queues:
	fbnic_reset_netif_queues(fbn);
free_resources:
	fbnic_free_resources(fbn);
free_napi_vectors:
	fbnic_free_napi_vectors(fbn);
	return err;
}

static int fbnic_open(struct net_device *netdev)
{
	struct fbnic_net *fbn = netdev_priv(netdev);
	int err;

	fbnic_napi_name_irqs(fbn->fbd);

	err = __fbnic_open(fbn);
	if (!err)
		fbnic_up(fbn);

	return err;
}

static int fbnic_stop(struct net_device *netdev)
{
	struct fbnic_net *fbn = netdev_priv(netdev);

	fbnic_mac_free_irq(fbn->fbd);
	phylink_suspend(fbn->phylink, fbnic_bmc_present(fbn->fbd));

	fbnic_down(fbn);

	fbnic_time_stop(fbn);
	fbnic_fw_xmit_ownership_msg(fbn->fbd, false);

	fbnic_reset_netif_queues(fbn);
	fbnic_free_resources(fbn);
	fbnic_free_napi_vectors(fbn);

	return 0;
}

static int fbnic_uc_sync(struct net_device *netdev, const unsigned char *addr)
{
	struct fbnic_net *fbn = netdev_priv(netdev);
	struct fbnic_mac_addr *avail_addr;

	if (WARN_ON(!is_valid_ether_addr(addr)))
		return -EADDRNOTAVAIL;

	avail_addr = __fbnic_uc_sync(fbn->fbd, addr);
	if (!avail_addr)
		return -ENOSPC;

	/* Add type flag indicating this address is in use by the host */
	set_bit(FBNIC_MAC_ADDR_T_UNICAST, avail_addr->act_tcam);

	return 0;
}

static int fbnic_uc_unsync(struct net_device *netdev, const unsigned char *addr)
{
	struct fbnic_net *fbn = netdev_priv(netdev);
	struct fbnic_dev *fbd = fbn->fbd;
	int i, ret;

	/* Scan from middle of list to bottom, filling bottom up.
	 * Skip the first entry which is reserved for dev_addr and
	 * leave the last entry to use for promiscuous filtering.
	 */
	for (i = fbd->mac_addr_boundary, ret = -ENOENT;
	     i < FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX && ret; i++) {
		struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i];

		if (!ether_addr_equal(mac_addr->value.addr8, addr))
			continue;

		ret = __fbnic_uc_unsync(mac_addr);
	}

	return ret;
}

static int fbnic_mc_sync(struct net_device *netdev, const unsigned char *addr)
{
	struct fbnic_net *fbn = netdev_priv(netdev);
	struct fbnic_mac_addr *avail_addr;

	if (WARN_ON(!is_multicast_ether_addr(addr)))
		return -EADDRNOTAVAIL;

	avail_addr = __fbnic_mc_sync(fbn->fbd, addr);
	if (!avail_addr)
		return -ENOSPC;

	/* Add type flag indicating this address is in use by the host */
	set_bit(FBNIC_MAC_ADDR_T_MULTICAST, avail_addr->act_tcam);

	return 0;
}

static int fbnic_mc_unsync(struct net_device *netdev, const unsigned char *addr)
{
	struct fbnic_net *fbn = netdev_priv(netdev);
	struct fbnic_dev *fbd = fbn->fbd;
	int i, ret;

	/* Scan from middle of list to top, filling top down.
	 * Skip over the address reserved for the BMC MAC and
	 * exclude index 0 as that belongs to the broadcast address
	 */
	for (i = fbd->mac_addr_boundary, ret = -ENOENT;
	     --i > FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX && ret;) {
		struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i];

		if (!ether_addr_equal(mac_addr->value.addr8, addr))
			continue;

		ret = __fbnic_mc_unsync(mac_addr);
	}

	return ret;
}

void __fbnic_set_rx_mode(struct fbnic_dev *fbd)
{
	bool uc_promisc = false, mc_promisc = false;
	struct net_device *netdev = fbd->netdev;
	struct fbnic_mac_addr *mac_addr;
	int err;

	/* Populate host address from dev_addr */
	mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX];
	if (!ether_addr_equal(mac_addr->value.addr8, netdev->dev_addr) ||
	    mac_addr->state != FBNIC_TCAM_S_VALID) {
		ether_addr_copy(mac_addr->value.addr8, netdev->dev_addr);
		mac_addr->state = FBNIC_TCAM_S_UPDATE;
		set_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam);
	}

	/* Populate broadcast address if broadcast is enabled */
	mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX];
	if (netdev->flags & IFF_BROADCAST) {
		if (!is_broadcast_ether_addr(mac_addr->value.addr8) ||
		    mac_addr->state != FBNIC_TCAM_S_VALID) {
			eth_broadcast_addr(mac_addr->value.addr8);
			mac_addr->state = FBNIC_TCAM_S_ADD;
		}
		set_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam);
	} else if (mac_addr->state == FBNIC_TCAM_S_VALID) {
		__fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BROADCAST);
	}

	/* Synchronize unicast and multicast address lists */
	err = __dev_uc_sync(netdev, fbnic_uc_sync, fbnic_uc_unsync);
	if (err == -ENOSPC)
		uc_promisc = true;
	err = __dev_mc_sync(netdev, fbnic_mc_sync, fbnic_mc_unsync);
	if (err == -ENOSPC)
		mc_promisc = true;

	uc_promisc |= !!(netdev->flags & IFF_PROMISC);
	mc_promisc |= !!(netdev->flags & IFF_ALLMULTI) || uc_promisc;

	/* Update the promiscuous rules */
	fbnic_promisc_sync(fbd, uc_promisc, mc_promisc);

	/* Add rules for BMC all multicast if it is enabled */
	fbnic_bmc_rpc_all_multi_config(fbd, mc_promisc);

	/* Sift out any unshared BMC rules and place them in BMC only section */
	fbnic_sift_macda(fbd);

	/* Write updates to hardware */
	fbnic_write_rules(fbd);
	fbnic_write_macda(fbd);
	fbnic_write_tce_tcam(fbd);
}

static void fbnic_set_rx_mode(struct net_device *netdev)
{
	struct fbnic_net *fbn = netdev_priv(netdev);
	struct fbnic_dev *fbd = fbn->fbd;

	/* No need to update the hardware if we are not running */
	if (netif_running(netdev))
		__fbnic_set_rx_mode(fbd);
}

static int fbnic_set_mac(struct net_device *netdev, void *p)
{
	struct sockaddr *addr = p;

	if (!is_valid_ether_addr(addr->sa_data))
		return -EADDRNOTAVAIL;

	eth_hw_addr_set(netdev, addr->sa_data);

	fbnic_set_rx_mode(netdev);

	return 0;
}

static int fbnic_change_mtu(struct net_device *dev, int new_mtu)
{
	struct fbnic_net *fbn = netdev_priv(dev);

	if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) {
		dev_err(&dev->dev,
			"MTU %d is larger than HDS threshold %d in XDP mode\n",
			new_mtu, fbn->hds_thresh);

		return -EINVAL;
	}

	WRITE_ONCE(dev->mtu, new_mtu);

	return 0;
}

void fbnic_clear_rx_mode(struct fbnic_dev *fbd)
{
	struct net_device *netdev = fbd->netdev;
	int idx;

	for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) {
		struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx];

		if (mac_addr->state != FBNIC_TCAM_S_VALID)
			continue;

		bitmap_clear(mac_addr->act_tcam,
			     FBNIC_MAC_ADDR_T_HOST_START,
			     FBNIC_MAC_ADDR_T_HOST_LEN);

		if (bitmap_empty(mac_addr->act_tcam,
				 FBNIC_RPC_TCAM_ACT_NUM_ENTRIES))
			mac_addr->state = FBNIC_TCAM_S_DELETE;
	}

	/* Write updates to hardware */
	fbnic_write_macda(fbd);

	__dev_uc_unsync(netdev, NULL);
	__dev_mc_unsync(netdev, NULL);
}

static int fbnic_hwtstamp_get(struct net_device *netdev,
			      struct kernel_hwtstamp_config *config)
{
	struct fbnic_net *fbn = netdev_priv(netdev);

	*config = fbn->hwtstamp_config;

	return 0;
}

static int fbnic_hwtstamp_set(struct net_device *netdev,
			      struct kernel_hwtstamp_config *config,
			      struct netlink_ext_ack *extack)
{
	struct fbnic_net *fbn = netdev_priv(netdev);
	int old_rx_filter;

	if (config->source != HWTSTAMP_SOURCE_NETDEV)
		return -EOPNOTSUPP;

	if (!kernel_hwtstamp_config_changed(config, &fbn->hwtstamp_config))
		return 0;

	/* Upscale the filters */
	switch (config->rx_filter) {
	case HWTSTAMP_FILTER_NONE:
	case HWTSTAMP_FILTER_ALL:
	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
	case HWTSTAMP_FILTER_PTP_V2_EVENT:
		break;
	case HWTSTAMP_FILTER_NTP_ALL:
		config->rx_filter = HWTSTAMP_FILTER_ALL;
		break;
	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
		config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
		break;
	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
		break;
	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
		break;
	case HWTSTAMP_FILTER_PTP_V2_SYNC:
	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
		break;
	default:
		return -ERANGE;
	}

	/* Configure */
	old_rx_filter = fbn->hwtstamp_config.rx_filter;
	memcpy(&fbn->hwtstamp_config, config, sizeof(*config));

	if (old_rx_filter != config->rx_filter && netif_running(fbn->netdev)) {
		fbnic_rss_reinit(fbn->fbd, fbn);
		fbnic_write_rules(fbn->fbd);
	}

	/* Save / report back filter configuration
	 * Note that our filter configuration is inexact. Instead of
	 * filtering for a specific UDP port or L2 Ethertype we are
	 * filtering in all UDP or all non-IP packets for timestamping. So
	 * if anything other than FILTER_ALL is requested we report
	 * FILTER_SOME indicating that we will be timestamping a few
	 * additional packets.
	 */
	if (config->rx_filter > HWTSTAMP_FILTER_ALL)
		config->rx_filter = HWTSTAMP_FILTER_SOME;

	return 0;
}

static void fbnic_get_stats64(struct net_device *dev,
			      struct rtnl_link_stats64 *stats64)
{
	u64 rx_bytes, rx_packets, rx_dropped = 0, rx_errors = 0;
	u64 rx_over = 0, rx_missed = 0, rx_length = 0;
	u64 tx_bytes, tx_packets, tx_dropped = 0;
	struct fbnic_net *fbn = netdev_priv(dev);
	struct fbnic_dev *fbd = fbn->fbd;
	struct fbnic_queue_stats *stats;

	unsigned int start, i;

	fbnic_get_hw_stats(fbd);

	stats = &fbn->tx_stats;

	tx_bytes = stats->bytes;
	tx_packets = stats->packets;
	tx_dropped = stats->dropped;

	/* Record drops from Tx HW Datapath */
	spin_lock(&fbd->hw_stats.lock);
	tx_dropped += fbd->hw_stats.tmi.drop.frames.value +
		      fbd->hw_stats.tti.cm_drop.frames.value +
		      fbd->hw_stats.tti.frame_drop.frames.value +
		      fbd->hw_stats.tti.tbi_drop.frames.value;
	spin_unlock(&fbd->hw_stats.lock);

	stats64->tx_bytes = tx_bytes;
	stats64->tx_packets = tx_packets;
	stats64->tx_dropped = tx_dropped;

	for (i = 0; i < fbn->num_tx_queues; i++) {
		struct fbnic_ring *txr = fbn->tx[i];

		if (!txr)
			continue;

		stats = &txr->stats;
		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			tx_bytes = stats->bytes;
			tx_packets = stats->packets;
			tx_dropped = stats->dropped;
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		stats64->tx_bytes += tx_bytes;
		stats64->tx_packets += tx_packets;
		stats64->tx_dropped += tx_dropped;
	}

	stats = &fbn->rx_stats;

	rx_bytes = stats->bytes;
	rx_packets = stats->packets;
	rx_dropped = stats->dropped;

	spin_lock(&fbd->hw_stats.lock);
	/* Record drops for the host FIFOs.
	 * 4: network to Host,	6: BMC to Host
	 * Exclude the BMC and MC FIFOs as those stats may contain drops
	 * due to unrelated items such as TCAM misses. They are still
	 * accessible through the ethtool stats.
	 */
	i = FBNIC_RXB_FIFO_HOST;
	rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value;
	i = FBNIC_RXB_FIFO_BMC_TO_HOST;
	rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value;

	for (i = 0; i < fbd->max_num_queues; i++) {
		/* Report packets dropped due to CQ/BDQ being full/empty */
		rx_over += fbd->hw_stats.hw_q[i].rde_pkt_cq_drop.value;
		rx_over += fbd->hw_stats.hw_q[i].rde_pkt_bdq_drop.value;

		/* Report packets with errors */
		rx_errors += fbd->hw_stats.hw_q[i].rde_pkt_err.value;
	}
	spin_unlock(&fbd->hw_stats.lock);

	stats64->rx_bytes = rx_bytes;
	stats64->rx_packets = rx_packets;
	stats64->rx_dropped = rx_dropped;
	stats64->rx_over_errors = rx_over;
	stats64->rx_errors = rx_errors;
	stats64->rx_missed_errors = rx_missed;

	for (i = 0; i < fbn->num_rx_queues; i++) {
		struct fbnic_ring *xdpr = fbn->tx[FBNIC_MAX_TXQS + i];
		struct fbnic_ring *rxr = fbn->rx[i];

		if (!rxr)
			continue;

		stats = &rxr->stats;
		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			rx_bytes = stats->bytes;
			rx_packets = stats->packets;
			rx_dropped = stats->dropped;
			rx_length = stats->rx.length_errors;
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		stats64->rx_bytes += rx_bytes;
		stats64->rx_packets += rx_packets;
		stats64->rx_dropped += rx_dropped;
		stats64->rx_errors += rx_length;
		stats64->rx_length_errors += rx_length;

		if (!xdpr)
			continue;

		stats = &xdpr->stats;
		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			tx_bytes = stats->bytes;
			tx_packets = stats->packets;
			tx_dropped = stats->dropped;
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		stats64->tx_bytes += tx_bytes;
		stats64->tx_packets += tx_packets;
		stats64->tx_dropped += tx_dropped;
	}
}

bool fbnic_check_split_frames(struct bpf_prog *prog, unsigned int mtu,
			      u32 hds_thresh)
{
	if (!prog)
		return false;

	if (prog->aux->xdp_has_frags)
		return false;

	return mtu + ETH_HLEN > hds_thresh;
}

static int fbnic_bpf(struct net_device *netdev, struct netdev_bpf *bpf)
{
	struct bpf_prog *prog = bpf->prog, *prev_prog;
	struct fbnic_net *fbn = netdev_priv(netdev);

	if (bpf->command != XDP_SETUP_PROG)
		return -EINVAL;

	if (fbnic_check_split_frames(prog, netdev->mtu,
				     fbn->hds_thresh)) {
		NL_SET_ERR_MSG_MOD(bpf->extack,
				   "MTU too high, or HDS threshold is too low for single buffer XDP");
		return -EOPNOTSUPP;
	}

	prev_prog = xchg(&fbn->xdp_prog, prog);
	if (prev_prog)
		bpf_prog_put(prev_prog);

	return 0;
}

static const struct net_device_ops fbnic_netdev_ops = {
	.ndo_open		= fbnic_open,
	.ndo_stop		= fbnic_stop,
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_start_xmit		= fbnic_xmit_frame,
	.ndo_features_check	= fbnic_features_check,
	.ndo_set_mac_address	= fbnic_set_mac,
	.ndo_change_mtu		= fbnic_change_mtu,
	.ndo_set_rx_mode	= fbnic_set_rx_mode,
	.ndo_get_stats64	= fbnic_get_stats64,
	.ndo_bpf		= fbnic_bpf,
	.ndo_hwtstamp_get	= fbnic_hwtstamp_get,
	.ndo_hwtstamp_set	= fbnic_hwtstamp_set,
};

static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx,
				     struct netdev_queue_stats_rx *rx)
{
	u64 bytes, packets, alloc_fail, alloc_fail_bdq;
	struct fbnic_net *fbn = netdev_priv(dev);
	struct fbnic_ring *rxr = fbn->rx[idx];
	struct fbnic_dev *fbd = fbn->fbd;
	struct fbnic_queue_stats *stats;
	u64 csum_complete, csum_none;
	struct fbnic_q_triad *qt;
	unsigned int start;

	if (!rxr)
		return;

	/* fbn->rx points to completion queues */
	qt = container_of(rxr, struct fbnic_q_triad, cmpl);

	stats = &rxr->stats;
	do {
		start = u64_stats_fetch_begin(&stats->syncp);
		bytes = stats->bytes;
		packets = stats->packets;
		alloc_fail = stats->rx.alloc_failed;
		csum_complete = stats->rx.csum_complete;
		csum_none = stats->rx.csum_none;
	} while (u64_stats_fetch_retry(&stats->syncp, start));

	stats = &qt->sub0.stats;
	do {
		start = u64_stats_fetch_begin(&stats->syncp);
		alloc_fail_bdq = stats->bdq.alloc_failed;
	} while (u64_stats_fetch_retry(&stats->syncp, start));
	alloc_fail += alloc_fail_bdq;

	stats = &qt->sub1.stats;
	do {
		start = u64_stats_fetch_begin(&stats->syncp);
		alloc_fail_bdq = stats->bdq.alloc_failed;
	} while (u64_stats_fetch_retry(&stats->syncp, start));
	alloc_fail += alloc_fail_bdq;

	rx->bytes = bytes;
	rx->packets = packets;
	rx->alloc_fail = alloc_fail;
	rx->csum_complete = csum_complete;
	rx->csum_none = csum_none;

	fbnic_get_hw_q_stats(fbd, fbd->hw_stats.hw_q);

	spin_lock(&fbd->hw_stats.lock);
	rx->hw_drop_overruns = fbd->hw_stats.hw_q[idx].rde_pkt_cq_drop.value +
			       fbd->hw_stats.hw_q[idx].rde_pkt_bdq_drop.value;
	rx->hw_drops = fbd->hw_stats.hw_q[idx].rde_pkt_err.value +
		       rx->hw_drop_overruns;
	spin_unlock(&fbd->hw_stats.lock);
}

static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx,
				     struct netdev_queue_stats_tx *tx)
{
	struct fbnic_net *fbn = netdev_priv(dev);
	struct fbnic_ring *txr = fbn->tx[idx];
	struct fbnic_queue_stats *stats;
	u64 stop, wake, csum, lso;
	struct fbnic_ring *xdpr;
	unsigned int start;
	u64 bytes, packets;

	if (!txr)
		return;

	stats = &txr->stats;
	do {
		start = u64_stats_fetch_begin(&stats->syncp);
		bytes = stats->bytes;
		packets = stats->packets;
		csum = stats->twq.csum_partial;
		lso = stats->twq.lso;
		stop = stats->twq.stop;
		wake = stats->twq.wake;
	} while (u64_stats_fetch_retry(&stats->syncp, start));

	tx->bytes = bytes;
	tx->packets = packets;
	tx->needs_csum = csum + lso;
	tx->hw_gso_wire_packets = lso;
	tx->stop = stop;
	tx->wake = wake;

	xdpr = fbn->tx[FBNIC_MAX_TXQS + idx];
	if (xdpr) {
		stats = &xdpr->stats;
		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			bytes = stats->bytes;
			packets = stats->packets;
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		tx->bytes += bytes;
		tx->packets += packets;
	}
}

static void fbnic_get_base_stats(struct net_device *dev,
				 struct netdev_queue_stats_rx *rx,
				 struct netdev_queue_stats_tx *tx)
{
	struct fbnic_net *fbn = netdev_priv(dev);

	tx->bytes = fbn->tx_stats.bytes;
	tx->packets = fbn->tx_stats.packets;
	tx->needs_csum = fbn->tx_stats.twq.csum_partial + fbn->tx_stats.twq.lso;
	tx->hw_gso_wire_packets = fbn->tx_stats.twq.lso;
	tx->stop = fbn->tx_stats.twq.stop;
	tx->wake = fbn->tx_stats.twq.wake;

	rx->bytes = fbn->rx_stats.bytes;
	rx->packets = fbn->rx_stats.packets;
	rx->alloc_fail = fbn->rx_stats.rx.alloc_failed +
		fbn->bdq_stats.bdq.alloc_failed;
	rx->csum_complete = fbn->rx_stats.rx.csum_complete;
	rx->csum_none = fbn->rx_stats.rx.csum_none;
}

static const struct netdev_stat_ops fbnic_stat_ops = {
	.get_queue_stats_rx	= fbnic_get_queue_stats_rx,
	.get_queue_stats_tx	= fbnic_get_queue_stats_tx,
	.get_base_stats		= fbnic_get_base_stats,
};

void fbnic_reset_queues(struct fbnic_net *fbn,
			unsigned int tx, unsigned int rx)
{
	struct fbnic_dev *fbd = fbn->fbd;
	unsigned int max_napis;

	max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS;

	tx = min(tx, max_napis);
	fbn->num_tx_queues = tx;

	rx = min(rx, max_napis);
	fbn->num_rx_queues = rx;

	fbn->num_napi = max(tx, rx);
}

/**
 * fbnic_netdev_free - Free the netdev associate with fbnic
 * @fbd: Driver specific structure to free netdev from
 *
 * Allocate and initialize the netdev and netdev private structure. Bind
 * together the hardware, netdev, and pci data structures.
 **/
void fbnic_netdev_free(struct fbnic_dev *fbd)
{
	fbnic_phylink_destroy(fbd->netdev);

	free_netdev(fbd->netdev);
	fbd->netdev = NULL;
}

/**
 * fbnic_netdev_alloc - Allocate a netdev and associate with fbnic
 * @fbd: Driver specific structure to associate netdev with
 *
 * Allocate and initialize the netdev and netdev private structure. Bind
 * together the hardware, netdev, and pci data structures.
 *
 *  Return: Pointer to net_device on success, NULL on failure
 **/
struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd)
{
	struct net_device *netdev;
	struct fbnic_net *fbn;
	int default_queues;

	netdev = alloc_etherdev_mq(sizeof(*fbn), FBNIC_MAX_RXQS);
	if (!netdev)
		return NULL;

	SET_NETDEV_DEV(netdev, fbd->dev);
	fbd->netdev = netdev;

	netdev->netdev_ops = &fbnic_netdev_ops;
	netdev->stat_ops = &fbnic_stat_ops;
	netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops;
	netdev->netmem_tx = true;

	fbnic_set_ethtool_ops(netdev);

	fbn = netdev_priv(netdev);

	fbn->netdev = netdev;
	fbn->fbd = fbd;

	fbn->txq_size = FBNIC_TXQ_SIZE_DEFAULT;
	fbn->hpq_size = FBNIC_HPQ_SIZE_DEFAULT;
	fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT;
	fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT;

	fbn->tx_usecs = FBNIC_TX_USECS_DEFAULT;
	fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT;
	fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT;

	/* Initialize the hds_thresh */
	netdev->cfg->hds_thresh = FBNIC_HDS_THRESH_DEFAULT;
	fbn->hds_thresh = FBNIC_HDS_THRESH_DEFAULT;

	default_queues = netif_get_num_default_rss_queues();
	if (default_queues > fbd->max_num_queues)
		default_queues = fbd->max_num_queues;

	fbnic_reset_queues(fbn, default_queues, default_queues);

	fbnic_reset_indir_tbl(fbn);
	fbnic_rss_key_fill(fbn->rss_key);
	fbnic_rss_init_en_mask(fbn);

	netdev->priv_flags |= IFF_UNICAST_FLT;

	netdev->gso_partial_features =
		NETIF_F_GSO_GRE |
		NETIF_F_GSO_GRE_CSUM |
		NETIF_F_GSO_IPXIP4 |
		NETIF_F_GSO_UDP_TUNNEL |
		NETIF_F_GSO_UDP_TUNNEL_CSUM;

	netdev->features |=
		netdev->gso_partial_features |
		FBNIC_TUN_GSO_FEATURES |
		NETIF_F_RXHASH |
		NETIF_F_SG |
		NETIF_F_HW_CSUM |
		NETIF_F_RXCSUM |
		NETIF_F_TSO |
		NETIF_F_TSO_ECN |
		NETIF_F_TSO6 |
		NETIF_F_GSO_PARTIAL |
		NETIF_F_GSO_UDP_L4;

	netdev->hw_features |= netdev->features;
	netdev->vlan_features |= netdev->features;
	netdev->hw_enc_features |= netdev->features;
	netdev->features |= NETIF_F_NTUPLE;

	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG;

	netdev->min_mtu = IPV6_MIN_MTU;
	netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN;

	/* TBD: This is workaround for BMC as phylink doesn't have support
	 * for leavling the link enabled if a BMC is present.
	 */
	netdev->ethtool->wol_enabled = true;

	netif_carrier_off(netdev);

	netif_tx_stop_all_queues(netdev);

	if (fbnic_phylink_create(netdev)) {
		fbnic_netdev_free(fbd);
		return NULL;
	}

	return netdev;
}

static int fbnic_dsn_to_mac_addr(u64 dsn, char *addr)
{
	addr[0] = (dsn >> 56) & 0xFF;
	addr[1] = (dsn >> 48) & 0xFF;
	addr[2] = (dsn >> 40) & 0xFF;
	addr[3] = (dsn >> 16) & 0xFF;
	addr[4] = (dsn >> 8) & 0xFF;
	addr[5] = dsn & 0xFF;

	return is_valid_ether_addr(addr) ? 0 : -EINVAL;
}

/**
 * fbnic_netdev_register - Initialize general software structures
 * @netdev: Netdev containing structure to initialize and register
 *
 * Initialize the MAC address for the netdev and register it.
 *
 *  Return: 0 on success, negative on failure
 **/
int fbnic_netdev_register(struct net_device *netdev)
{
	struct fbnic_net *fbn = netdev_priv(netdev);
	struct fbnic_dev *fbd = fbn->fbd;
	u64 dsn = fbd->dsn;
	u8 addr[ETH_ALEN];
	int err;

	err = fbnic_dsn_to_mac_addr(dsn, addr);
	if (!err) {
		ether_addr_copy(netdev->perm_addr, addr);
		eth_hw_addr_set(netdev, addr);
	} else {
		/* A randomly assigned MAC address will cause provisioning
		 * issues so instead just fail to spawn the netdev and
		 * avoid any confusion.
		 */
		dev_err(fbd->dev, "MAC addr %pM invalid\n", addr);
		return err;
	}

	return register_netdev(netdev);
}

void fbnic_netdev_unregister(struct net_device *netdev)
{
	unregister_netdev(netdev);
}