Differential D46690 Diff 144949 sys/dev/gve/gve.h

Changeset View

Standalone View

sys/dev/gve/gve.h

/*-		/*-
* SPDX-License-Identifier: BSD-3-Clause		* SPDX-License-Identifier: BSD-3-Clause
*		*
* Copyright (c) 2023 Google LLC		* Copyright (c) 2023-2024 Google LLC
*		*
* Redistribution and use in source and binary forms, with or without modification,		* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:		* are permitted provided that the following conditions are met:
*		*
* 1. Redistributions of source code must retain the above copyright notice, this		* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.		* list of conditions and the following disclaimer.
*		*
* 2. Redistributions in binary form must reproduce the above copyright notice,		* 2. Redistributions in binary form must reproduce the above copyright notice,
Show All 35 Lines
#define GVE_TX_BUFRING_ENTRIES 4096		#define GVE_TX_BUFRING_ENTRIES 4096

#define ADMINQ_SIZE PAGE_SIZE		#define ADMINQ_SIZE PAGE_SIZE

#define GVE_DEFAULT_RX_BUFFER_SIZE 2048		#define GVE_DEFAULT_RX_BUFFER_SIZE 2048
/* Each RX bounce buffer page can fit two packet buffers. */		/* Each RX bounce buffer page can fit two packet buffers. */
#define GVE_DEFAULT_RX_BUFFER_OFFSET (PAGE_SIZE / 2)		#define GVE_DEFAULT_RX_BUFFER_OFFSET (PAGE_SIZE / 2)

		/* PTYPEs are always 10 bits. */
		#define GVE_NUM_PTYPES 1024

/*		/*
* Number of descriptors per queue page list.		* Number of descriptors per queue page list.
* Page count AKA QPL size can be derived by dividing the number of elements in		* Page count AKA QPL size can be derived by dividing the number of elements in
* a page by the number of descriptors available.		* a page by the number of descriptors available.
*/		*/
#define GVE_QPL_DIVISOR 16		#define GVE_QPL_DIVISOR 16

static MALLOC_DEFINE(M_GVE, "gve", "gve allocations");		static MALLOC_DEFINE(M_GVE, "gve", "gve allocations");
▲ Show 20 Lines • Show All 155 Lines • ▼ Show 20 Lines	struct gve_rxq_stats {
counter_u64_t rbytes;		counter_u64_t rbytes;
counter_u64_t rpackets;		counter_u64_t rpackets;
counter_u64_t rx_dropped_pkt;		counter_u64_t rx_dropped_pkt;
counter_u64_t rx_copybreak_cnt;		counter_u64_t rx_copybreak_cnt;
counter_u64_t rx_frag_flip_cnt;		counter_u64_t rx_frag_flip_cnt;
counter_u64_t rx_frag_copy_cnt;		counter_u64_t rx_frag_copy_cnt;
counter_u64_t rx_dropped_pkt_desc_err;		counter_u64_t rx_dropped_pkt_desc_err;
counter_u64_t rx_dropped_pkt_mbuf_alloc_fail;		counter_u64_t rx_dropped_pkt_mbuf_alloc_fail;
		counter_u64_t rx_mbuf_dmamap_err;
		counter_u64_t rx_mbuf_mclget_null;
};		};

#define NUM_RX_STATS (sizeof(struct gve_rxq_stats) / sizeof(counter_u64_t))		#define NUM_RX_STATS (sizeof(struct gve_rxq_stats) / sizeof(counter_u64_t))

		struct gve_rx_buf_dqo {
		struct mbuf *mbuf;
		bus_dmamap_t dmamap;
		uint64_t addr;
		bool mapped;
		SLIST_ENTRY(gve_rx_buf_dqo) slist_entry;
		};

/* power-of-2 sized receive ring */		/* power-of-2 sized receive ring */
struct gve_rx_ring {		struct gve_rx_ring {
struct gve_ring_com com;		struct gve_ring_com com;
struct gve_dma_handle desc_ring_mem;		struct gve_dma_handle desc_ring_mem;
struct gve_dma_handle data_ring_mem;		uint32_t cnt; /* free-running total number of completed packets */
		uint32_t fill_cnt; /* free-running total number of descs and buffs posted */

/* accessed in the receive hot path */		union {
		/* GQI-only fields */
struct {		struct {
		struct gve_dma_handle data_ring_mem;

		/* accessed in the GQ receive hot path */
struct gve_rx_desc *desc_ring;		struct gve_rx_desc *desc_ring;
union gve_rx_data_slot *data_ring;		union gve_rx_data_slot *data_ring;
struct gve_rx_slot_page_info *page_info;		struct gve_rx_slot_page_info *page_info;
		uint32_t mask; /* masks the cnt and fill_cnt to the size of the ring */
		uint8_t seq_no; /* helps traverse the descriptor ring */
		};
		markjUnsubmitted Done Inline Actions Why not give a name to the GQI struct? Is it just to avoid churn? markj: Why not give a name to the GQI struct? Is it just to avoid churn?
		shailend_google.comAuthorUnsubmitted Done Inline Actions Yup, didnt want to have a lot of diff in the existing datapath shailend_google.com: Yup, didnt want to have a lot of diff in the existing datapath

struct gve_rx_ctx ctx;		/* DQO-only fields */
		struct {
		struct gve_dma_handle compl_ring_mem;

		struct gve_rx_compl_desc_dqo *compl_ring;
		struct gve_rx_desc_dqo *desc_ring;
		struct gve_rx_buf_dqo bufs; / Parking place for posted buffers */
		bus_dma_tag_t buf_dmatag; /* To dmamap posted mbufs with */

		uint32_t buf_cnt; /* Size of the bufs array */
		uint32_t mask; /* One less than the sizes of the desc and compl rings */
		uint32_t head; /* The index at which to post the next buffer at */
		uint32_t tail; /* The index at which to receive the next compl at */
		uint8_t cur_gen_bit; /* Gets flipped on every cycle of the compl ring */
		SLIST_HEAD(, gve_rx_buf_dqo) free_bufs;
		} dqo;
		};

struct lro_ctrl lro;		struct lro_ctrl lro;
uint8_t seq_no; /* helps traverse the descriptor ring */		struct gve_rx_ctx ctx;
uint32_t cnt; /* free-running total number of completed packets */
uint32_t fill_cnt; /* free-running total number of descs and buffs posted */
uint32_t mask; /* masks the cnt and fill_cnt to the size of the ring */
struct gve_rxq_stats stats;		struct gve_rxq_stats stats;
} __aligned(CACHE_LINE_SIZE);

} __aligned(CACHE_LINE_SIZE);		} __aligned(CACHE_LINE_SIZE);

/*		/*
* A contiguous representation of the pages composing the Tx bounce buffer.		* A contiguous representation of the pages composing the Tx bounce buffer.
* The xmit taskqueue and the completion taskqueue both simultaneously use it.		* The xmit taskqueue and the completion taskqueue both simultaneously use it.
* Both operate on `available`: the xmit tq lowers it and the completion tq		* Both operate on `available`: the xmit tq lowers it and the completion tq
* raises it. `head` is the last location written at and so only the xmit tq		* raises it. `head` is the last location written at and so only the xmit tq
Show All 13 Lines

struct gve_txq_stats {		struct gve_txq_stats {
counter_u64_t tbytes;		counter_u64_t tbytes;
counter_u64_t tpackets;		counter_u64_t tpackets;
counter_u64_t tso_packet_cnt;		counter_u64_t tso_packet_cnt;
counter_u64_t tx_dropped_pkt;		counter_u64_t tx_dropped_pkt;
counter_u64_t tx_dropped_pkt_nospace_device;		counter_u64_t tx_dropped_pkt_nospace_device;
counter_u64_t tx_dropped_pkt_nospace_bufring;		counter_u64_t tx_dropped_pkt_nospace_bufring;
		counter_u64_t tx_delayed_pkt_nospace_descring;
		counter_u64_t tx_delayed_pkt_nospace_compring;
		counter_u64_t tx_delayed_pkt_tsoerr;
counter_u64_t tx_dropped_pkt_vlan;		counter_u64_t tx_dropped_pkt_vlan;
		counter_u64_t tx_mbuf_collapse;
		counter_u64_t tx_mbuf_defrag;
		counter_u64_t tx_mbuf_defrag_err;
		counter_u64_t tx_mbuf_dmamap_enomem_err;
		counter_u64_t tx_mbuf_dmamap_err;
};		};

#define NUM_TX_STATS (sizeof(struct gve_txq_stats) / sizeof(counter_u64_t))		#define NUM_TX_STATS (sizeof(struct gve_txq_stats) / sizeof(counter_u64_t))

		struct gve_tx_pending_pkt_dqo {
		struct mbuf *mbuf;
		bus_dmamap_t dmamap;
		uint8_t state; /* the gve_packet_state enum */
		int next; /* To chain the free_pending_pkts lists */
		};

/* power-of-2 sized transmit ring */		/* power-of-2 sized transmit ring */
struct gve_tx_ring {		struct gve_tx_ring {
struct gve_ring_com com;		struct gve_ring_com com;
struct gve_dma_handle desc_ring_mem;		struct gve_dma_handle desc_ring_mem;

struct task xmit_task;		struct task xmit_task;
struct taskqueue *xmit_tq;		struct taskqueue *xmit_tq;

/* accessed in the transmit hot path */		/* Accessed when writing descriptors */
		struct buf_ring *br;
		struct mtx ring_mtx;

		uint32_t req; /* free-running total number of packets written to the nic */
		uint32_t done; /* free-running total number of completed packets */

		union {
		/* GQI specific stuff */
struct {		struct {
union gve_tx_desc *desc_ring;		union gve_tx_desc *desc_ring;
struct gve_tx_buffer_state *info;		struct gve_tx_buffer_state *info;
struct buf_ring *br;

struct gve_tx_fifo fifo;		struct gve_tx_fifo fifo;
struct mtx ring_mtx;

uint32_t req; /* free-running total number of packets written to the nic */
uint32_t done; /* free-running total number of completed packets */
uint32_t mask; /* masks the req and done to the size of the ring */		uint32_t mask; /* masks the req and done to the size of the ring */
struct gve_txq_stats stats;		};
		markjUnsubmitted Done Inline Actions Why not give this struct field a name, like you did with `dqo` below? markj: Why not give this struct field a name, like you did with `dqo` below?
		shailend_google.comAuthorUnsubmitted Done Inline Actions I was trying to avoid introducing diff in the existing "GQI" datapath shailend_google.com: I was trying to avoid introducing diff in the existing "GQI" datapath

		/* DQO specific stuff */
		struct {
		struct gve_dma_handle compl_ring_mem;

		/* Accessed when writing descriptors */
		struct {
		union gve_tx_desc_dqo *desc_ring;
		uint32_t desc_mask; /* masks head and tail to the size of desc_ring */
		uint32_t desc_head; /* last desc read by NIC, cached value of hw_tx_head */
		uint32_t desc_tail; /* last desc written by driver */
		uint32_t last_re_idx; /* desc which last had "report event" set */

		/*
		* The head index of a singly linked list containing pending packet objects
		* to park mbufs till the NIC sends completions. Once this list is depleted,
		* the "_prd" suffixed producer list, grown by the completion taskqueue,
		* is stolen.
		*/
		int32_t free_pending_pkts_csm;

		bus_dma_tag_t buf_dmatag; /* DMA params for mapping Tx mbufs */
} __aligned(CACHE_LINE_SIZE);		} __aligned(CACHE_LINE_SIZE);

		/* Accessed when processing completions */
		struct {
		struct gve_tx_compl_desc_dqo *compl_ring;
		uint32_t compl_mask; /* masks head to the size of compl_ring */
		uint32_t compl_head; /* last completion read by driver */
		uint8_t cur_gen_bit; /* NIC flips a bit on every pass */
		volatile uint32_t hw_tx_head; /* last desc read by NIC */
		markjUnsubmitted Done Inline Actions It shouldn't be necessary to use `volatile` if all accesses are via atomic(9), which they seem to be. markj: It shouldn't be necessary to use `volatile` if all accesses are via atomic(9), which they seem…

		/*
		* The completion taskqueue moves pending-packet objects to this
		* list after freeing the mbuf. The "_prd" denotes that this is
		* a producer list. The trasnmit taskqueue steals this list once
		* its consumer list, with the "_csm" suffix, is depleted.
		*/
		volatile int32_t free_pending_pkts_prd;
} __aligned(CACHE_LINE_SIZE);		} __aligned(CACHE_LINE_SIZE);

		/* Accessed by both the completion and xmit loops */
		struct {
		/* completion tags index into this array */
		struct gve_tx_pending_pkt_dqo *pending_pkts;
		uint16_t num_pending_pkts;
		} __aligned(CACHE_LINE_SIZE);
		} dqo;
		};
		struct gve_txq_stats stats;
		} __aligned(CACHE_LINE_SIZE);

		enum gve_packet_state {
		/*
		* Packet does not yet have a dmamap created.
		* This should always be zero since state is not explicitly initialized.
		*/
		GVE_PACKET_STATE_UNALLOCATED,
		/* Packet has a dmamap and is in free list, available to be allocated. */
		GVE_PACKET_STATE_FREE,
		/* Packet is expecting a regular data completion */
		GVE_PACKET_STATE_PENDING_DATA_COMPL,
		};

		struct gve_ptype {
		uint8_t l3_type; /* `gve_l3_type` in gve_adminq.h */
		uint8_t l4_type; /* `gve_l4_type` in gve_adminq.h */
		};

		struct gve_ptype_lut {
		struct gve_ptype ptypes[GVE_NUM_PTYPES];
		};

struct gve_priv {		struct gve_priv {
if_t ifp;		if_t ifp;
device_t dev;		device_t dev;
struct ifmedia media;		struct ifmedia media;

uint8_t mac[ETHER_ADDR_LEN];		uint8_t mac[ETHER_ADDR_LEN];

struct gve_dma_handle aq_mem;		struct gve_dma_handle aq_mem;
Show All 25 Lines	struct gve_priv {
struct gve_queue_config tx_cfg;		struct gve_queue_config tx_cfg;
struct gve_queue_config rx_cfg;		struct gve_queue_config rx_cfg;
uint32_t num_queues;		uint32_t num_queues;

struct gve_irq *irq_tbl;		struct gve_irq *irq_tbl;
struct gve_tx_ring *tx;		struct gve_tx_ring *tx;
struct gve_rx_ring *rx;		struct gve_rx_ring *rx;

		struct gve_ptype_lut *ptype_lut_dqo;

/*		/*
* Admin queue - see gve_adminq.h		* Admin queue - see gve_adminq.h
* Since AQ cmds do not run in steady state, 32 bit counters suffice		* Since AQ cmds do not run in steady state, 32 bit counters suffice
*/		*/
struct gve_adminq_command *adminq;		struct gve_adminq_command *adminq;
vm_paddr_t adminq_bus_addr;		vm_paddr_t adminq_bus_addr;
uint32_t adminq_mask; /* masks prod_cnt to adminq size */		uint32_t adminq_mask; /* masks prod_cnt to adminq size */
uint32_t adminq_prod_cnt; /* free-running count of AQ cmds executed */		uint32_t adminq_prod_cnt; /* free-running count of AQ cmds executed */
uint32_t adminq_cmd_fail; /* free-running count of AQ cmds failed */		uint32_t adminq_cmd_fail; /* free-running count of AQ cmds failed */
uint32_t adminq_timeouts; /* free-running count of AQ cmds timeouts */		uint32_t adminq_timeouts; /* free-running count of AQ cmds timeouts */
/* free-running count of each distinct AQ cmd executed */		/* free-running count of each distinct AQ cmd executed */
uint32_t adminq_describe_device_cnt;		uint32_t adminq_describe_device_cnt;
uint32_t adminq_cfg_device_resources_cnt;		uint32_t adminq_cfg_device_resources_cnt;
uint32_t adminq_register_page_list_cnt;		uint32_t adminq_register_page_list_cnt;
uint32_t adminq_unregister_page_list_cnt;		uint32_t adminq_unregister_page_list_cnt;
uint32_t adminq_create_tx_queue_cnt;		uint32_t adminq_create_tx_queue_cnt;
uint32_t adminq_create_rx_queue_cnt;		uint32_t adminq_create_rx_queue_cnt;
uint32_t adminq_destroy_tx_queue_cnt;		uint32_t adminq_destroy_tx_queue_cnt;
uint32_t adminq_destroy_rx_queue_cnt;		uint32_t adminq_destroy_rx_queue_cnt;
uint32_t adminq_dcfg_device_resources_cnt;		uint32_t adminq_dcfg_device_resources_cnt;
uint32_t adminq_set_driver_parameter_cnt;		uint32_t adminq_set_driver_parameter_cnt;
uint32_t adminq_verify_driver_compatibility_cnt;		uint32_t adminq_verify_driver_compatibility_cnt;
		uint32_t adminq_get_ptype_map_cnt;

uint32_t interface_up_cnt;		uint32_t interface_up_cnt;
uint32_t interface_down_cnt;		uint32_t interface_down_cnt;
uint32_t reset_cnt;		uint32_t reset_cnt;

struct task service_task;		struct task service_task;
struct taskqueue *service_tq;		struct taskqueue *service_tq;

Show All 14 Lines
}		}

static inline void		static inline void
gve_clear_state_flag(struct gve_priv *priv, int pos)		gve_clear_state_flag(struct gve_priv *priv, int pos)
{		{
BIT_CLR_ATOMIC(GVE_NUM_STATE_FLAGS, pos, &priv->state_flags);		BIT_CLR_ATOMIC(GVE_NUM_STATE_FLAGS, pos, &priv->state_flags);
}		}

		static inline bool
		gve_is_gqi(struct gve_priv *priv)
		{
		return (priv->queue_format == GVE_GQI_QPL_FORMAT);
		}

/* Defined in gve_main.c */		/* Defined in gve_main.c */
void gve_schedule_reset(struct gve_priv *priv);		void gve_schedule_reset(struct gve_priv *priv);

/* Register access functions defined in gve_utils.c */		/* Register access functions defined in gve_utils.c */
uint32_t gve_reg_bar_read_4(struct gve_priv *priv, bus_size_t offset);		uint32_t gve_reg_bar_read_4(struct gve_priv *priv, bus_size_t offset);
void gve_reg_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val);		void gve_reg_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val);
void gve_db_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val);		void gve_db_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val);
		void gve_db_bar_dqo_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val);

/* QPL (Queue Page List) functions defined in gve_qpl.c */		/* QPL (Queue Page List) functions defined in gve_qpl.c */
int gve_alloc_qpls(struct gve_priv *priv);		int gve_alloc_qpls(struct gve_priv *priv);
void gve_free_qpls(struct gve_priv *priv);		void gve_free_qpls(struct gve_priv *priv);
int gve_register_qpls(struct gve_priv *priv);		int gve_register_qpls(struct gve_priv *priv);
int gve_unregister_qpls(struct gve_priv *priv);		int gve_unregister_qpls(struct gve_priv *priv);

/* TX functions defined in gve_tx.c */		/* TX functions defined in gve_tx.c */
int gve_alloc_tx_rings(struct gve_priv *priv);		int gve_alloc_tx_rings(struct gve_priv *priv);
void gve_free_tx_rings(struct gve_priv *priv);		void gve_free_tx_rings(struct gve_priv *priv);
int gve_create_tx_rings(struct gve_priv *priv);		int gve_create_tx_rings(struct gve_priv *priv);
int gve_destroy_tx_rings(struct gve_priv *priv);		int gve_destroy_tx_rings(struct gve_priv *priv);
int gve_tx_intr(void *arg);		int gve_tx_intr(void *arg);
int gve_xmit_ifp(if_t ifp, struct mbuf *mbuf);		int gve_xmit_ifp(if_t ifp, struct mbuf *mbuf);
void gve_qflush(if_t ifp);		void gve_qflush(if_t ifp);
void gve_xmit_tq(void *arg, int pending);		void gve_xmit_tq(void *arg, int pending);
void gve_tx_cleanup_tq(void *arg, int pending);		void gve_tx_cleanup_tq(void *arg, int pending);

		/* TX functions defined in gve_tx_dqo.c */
		int gve_tx_alloc_ring_dqo(struct gve_priv *priv, int i);
		void gve_tx_free_ring_dqo(struct gve_priv *priv, int i);
		void gve_clear_tx_ring_dqo(struct gve_priv *priv, int i);
		int gve_tx_intr_dqo(void *arg);
		int gve_xmit_dqo(struct gve_tx_ring tx, struct mbuf *mbuf_ptr);
		void gve_tx_cleanup_tq_dqo(void *arg, int pending);

/* RX functions defined in gve_rx.c */		/* RX functions defined in gve_rx.c */
int gve_alloc_rx_rings(struct gve_priv *priv);		int gve_alloc_rx_rings(struct gve_priv *priv);
void gve_free_rx_rings(struct gve_priv *priv);		void gve_free_rx_rings(struct gve_priv *priv);
int gve_create_rx_rings(struct gve_priv *priv);		int gve_create_rx_rings(struct gve_priv *priv);
int gve_destroy_rx_rings(struct gve_priv *priv);		int gve_destroy_rx_rings(struct gve_priv *priv);
int gve_rx_intr(void *arg);		int gve_rx_intr(void *arg);
void gve_rx_cleanup_tq(void *arg, int pending);		void gve_rx_cleanup_tq(void *arg, int pending);

		/* RX functions defined in gve_rx_dqo.c */
		int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int i);
		void gve_rx_free_ring_dqo(struct gve_priv *priv, int i);
		void gve_rx_prefill_buffers_dqo(struct gve_rx_ring *rx);
		void gve_clear_rx_ring_dqo(struct gve_priv *priv, int i);
		int gve_rx_intr_dqo(void *arg);
		void gve_rx_cleanup_tq_dqo(void *arg, int pending);

/* DMA functions defined in gve_utils.c */		/* DMA functions defined in gve_utils.c */
int gve_dma_alloc_coherent(struct gve_priv *priv, int size, int align,		int gve_dma_alloc_coherent(struct gve_priv *priv, int size, int align,
struct gve_dma_handle *dma);		struct gve_dma_handle *dma);
void gve_dma_free_coherent(struct gve_dma_handle *dma);		void gve_dma_free_coherent(struct gve_dma_handle *dma);
int gve_dmamap_create(struct gve_priv *priv, int size, int align,		int gve_dmamap_create(struct gve_priv *priv, int size, int align,
struct gve_dma_handle *dma);		struct gve_dma_handle *dma);
void gve_dmamap_destroy(struct gve_dma_handle *dma);		void gve_dmamap_destroy(struct gve_dma_handle *dma);

/* IRQ functions defined in gve_utils.c */		/* IRQ functions defined in gve_utils.c */
void gve_free_irqs(struct gve_priv *priv);		void gve_free_irqs(struct gve_priv *priv);
int gve_alloc_irqs(struct gve_priv *priv);		int gve_alloc_irqs(struct gve_priv *priv);
void gve_unmask_all_queue_irqs(struct gve_priv *priv);		void gve_unmask_all_queue_irqs(struct gve_priv *priv);
void gve_mask_all_queue_irqs(struct gve_priv *priv);		void gve_mask_all_queue_irqs(struct gve_priv *priv);

/* Systcl functions defined in gve_sysctl.c*/		/* Systcl functions defined in gve_sysctl.c */
		extern bool gve_disable_hw_lro;
		extern char gve_queue_format[8];
		extern char gve_version[8];
void gve_setup_sysctl(struct gve_priv *priv);		void gve_setup_sysctl(struct gve_priv *priv);
void gve_accum_stats(struct gve_priv priv, uint64_t rpackets,		void gve_accum_stats(struct gve_priv priv, uint64_t rpackets,
uint64_t rbytes, uint64_t rx_dropped_pkt, uint64_t *tpackets,		uint64_t rbytes, uint64_t rx_dropped_pkt, uint64_t *tpackets,
uint64_t tbytes, uint64_t tx_dropped_pkt);		uint64_t tbytes, uint64_t tx_dropped_pkt);

/* Stats functions defined in gve_utils.c */		/* Stats functions defined in gve_utils.c */
void gve_alloc_counters(counter_u64_t *stat, int num_stats);		void gve_alloc_counters(counter_u64_t *stat, int num_stats);
void gve_free_counters(counter_u64_t *stat, int num_stats);		void gve_free_counters(counter_u64_t *stat, int num_stats);

#endif /* _GVE_FBSD_H_ */		#endif /* _GVE_FBSD_H_ */