Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/cxgbe/tom/t4_ddp.c
Show First 20 Lines • Show All 1,075 Lines • ▼ Show 20 Lines | #endif | ||||
t4_wrq_tx(sc, wr); | t4_wrq_tx(sc, wr); | ||||
} | } | ||||
ps->flags |= PS_PPODS_WRITTEN; | ps->flags |= PS_PPODS_WRITTEN; | ||||
return (0); | return (0); | ||||
} | } | ||||
static struct mbuf * | |||||
alloc_raw_wr_mbuf(int len) | |||||
{ | |||||
struct mbuf *m; | |||||
if (len <= MHLEN) | |||||
m = m_gethdr(M_NOWAIT, MT_DATA); | |||||
else if (len <= MCLBYTES) | |||||
m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); | |||||
else | |||||
m = NULL; | |||||
if (m == NULL) | |||||
return (NULL); | |||||
m->m_pkthdr.len = len; | |||||
m->m_len = len; | |||||
set_mbuf_raw_wr(m, true); | |||||
return (m); | |||||
} | |||||
int | int | ||||
t4_write_page_pods_for_buf(struct adapter *sc, struct sge_wrq *wrq, int tid, | t4_write_page_pods_for_buf(struct adapter *sc, struct toepcb *toep, | ||||
struct ppod_reservation *prsv, vm_offset_t buf, int buflen) | struct ppod_reservation *prsv, vm_offset_t buf, int buflen) | ||||
{ | { | ||||
struct wrqe *wr; | struct inpcb *inp = toep->inp; | ||||
struct ulp_mem_io *ulpmc; | struct ulp_mem_io *ulpmc; | ||||
struct ulptx_idata *ulpsc; | struct ulptx_idata *ulpsc; | ||||
struct pagepod *ppod; | struct pagepod *ppod; | ||||
int i, j, k, n, chunk, len, ddp_pgsz; | int i, j, k, n, chunk, len, ddp_pgsz; | ||||
u_int ppod_addr, offset; | u_int ppod_addr, offset; | ||||
uint32_t cmd; | uint32_t cmd; | ||||
struct ppod_region *pr = prsv->prsv_pr; | struct ppod_region *pr = prsv->prsv_pr; | ||||
uintptr_t end_pva, pva, pa; | uintptr_t end_pva, pva, pa; | ||||
struct mbuf *m; | |||||
struct mbufq wrq; | |||||
cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); | cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); | ||||
if (is_t4(sc)) | if (is_t4(sc)) | ||||
cmd |= htobe32(F_ULP_MEMIO_ORDER); | cmd |= htobe32(F_ULP_MEMIO_ORDER); | ||||
else | else | ||||
cmd |= htobe32(F_T5_ULP_MEMIO_IMM); | cmd |= htobe32(F_T5_ULP_MEMIO_IMM); | ||||
ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)]; | ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)]; | ||||
offset = buf & PAGE_MASK; | offset = buf & PAGE_MASK; | ||||
ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask); | ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask); | ||||
pva = trunc_page(buf); | pva = trunc_page(buf); | ||||
end_pva = trunc_page(buf + buflen - 1); | end_pva = trunc_page(buf + buflen - 1); | ||||
mbufq_init(&wrq, INT_MAX); | |||||
for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) { | for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) { | ||||
/* How many page pods are we writing in this cycle */ | /* How many page pods are we writing in this cycle */ | ||||
n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS); | n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS); | ||||
MPASS(n > 0); | MPASS(n > 0); | ||||
chunk = PPOD_SZ(n); | chunk = PPOD_SZ(n); | ||||
len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16); | len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16); | ||||
wr = alloc_wrqe(len, wrq); | m = alloc_raw_wr_mbuf(len); | ||||
if (wr == NULL) | if (m == NULL) { | ||||
return (ENOMEM); /* ok to just bail out */ | mbufq_drain(&wrq); | ||||
ulpmc = wrtod(wr); | return (ENOMEM); | ||||
} | |||||
ulpmc = mtod(m, struct ulp_mem_io *); | |||||
INIT_ULPTX_WR(ulpmc, len, 0, 0); | INIT_ULPTX_WR(ulpmc, len, 0, toep->tid); | ||||
ulpmc->cmd = cmd; | ulpmc->cmd = cmd; | ||||
ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); | ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); | ||||
ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); | ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); | ||||
ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); | ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); | ||||
ulpsc = (struct ulptx_idata *)(ulpmc + 1); | ulpsc = (struct ulptx_idata *)(ulpmc + 1); | ||||
ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); | ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); | ||||
ulpsc->len = htobe32(chunk); | ulpsc->len = htobe32(chunk); | ||||
ppod = (struct pagepod *)(ulpsc + 1); | ppod = (struct pagepod *)(ulpsc + 1); | ||||
for (j = 0; j < n; i++, j++, ppod++) { | for (j = 0; j < n; i++, j++, ppod++) { | ||||
ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID | | ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID | | ||||
V_PPOD_TID(tid) | | V_PPOD_TID(toep->tid) | | ||||
(prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ))); | (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ))); | ||||
ppod->len_offset = htobe64(V_PPOD_LEN(buflen) | | ppod->len_offset = htobe64(V_PPOD_LEN(buflen) | | ||||
V_PPOD_OFST(offset)); | V_PPOD_OFST(offset)); | ||||
ppod->rsvd = 0; | ppod->rsvd = 0; | ||||
for (k = 0; k < nitems(ppod->addr); k++) { | for (k = 0; k < nitems(ppod->addr); k++) { | ||||
if (pva > end_pva) | if (pva > end_pva) | ||||
ppod->addr[k] = 0; | ppod->addr[k] = 0; | ||||
else { | else { | ||||
pa = pmap_kextract(pva); | pa = pmap_kextract(pva); | ||||
ppod->addr[k] = htobe64(pa); | ppod->addr[k] = htobe64(pa); | ||||
pva += ddp_pgsz; | pva += ddp_pgsz; | ||||
} | } | ||||
#if 0 | #if 0 | ||||
CTR5(KTR_CXGBE, | CTR5(KTR_CXGBE, | ||||
"%s: tid %d ppod[%d]->addr[%d] = %p", | "%s: tid %d ppod[%d]->addr[%d] = %p", | ||||
__func__, tid, i, k, | __func__, toep->tid, i, k, | ||||
htobe64(ppod->addr[k])); | htobe64(ppod->addr[k])); | ||||
#endif | #endif | ||||
} | } | ||||
/* | /* | ||||
* Walk back 1 segment so that the first address in the | * Walk back 1 segment so that the first address in the | ||||
* next pod is the same as the last one in the current | * next pod is the same as the last one in the current | ||||
* pod. | * pod. | ||||
*/ | */ | ||||
pva -= ddp_pgsz; | pva -= ddp_pgsz; | ||||
} | } | ||||
t4_wrq_tx(sc, wr); | mbufq_enqueue(&wrq, m); | ||||
} | } | ||||
INP_WLOCK(inp); | |||||
mbufq_concat(&toep->ulp_pduq, &wrq); | |||||
INP_WUNLOCK(inp); | |||||
MPASS(pva <= end_pva); | MPASS(pva <= end_pva); | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Prepare a pageset for DDP. This sets up page pods. | * Prepare a pageset for DDP. This sets up page pods. | ||||
▲ Show 20 Lines • Show All 765 Lines • Show Last 20 Lines |