Index: head/sys/arm/amlogic/aml8726/aml8726_mmc.c =================================================================== --- head/sys/arm/amlogic/aml8726/aml8726_mmc.c (revision 299068) +++ head/sys/arm/amlogic/aml8726/aml8726_mmc.c (revision 299069) @@ -1,1102 +1,1102 @@ /*- * Copyright 2013-2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Amlogic aml8726 MMC host controller driver. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include "mmcbr_if.h" struct aml8726_mmc_gpio { device_t dev; uint32_t pin; uint32_t pol; }; struct aml8726_mmc_softc { device_t dev; struct resource *res[2]; struct mtx mtx; struct callout ch; uint32_t port; unsigned int ref_freq; struct aml8726_mmc_gpio pwr_en; int voltages[2]; struct aml8726_mmc_gpio vselect; bus_dma_tag_t dmatag; bus_dmamap_t dmamap; void *ih_cookie; struct mmc_host host; int bus_busy; struct mmc_command *cmd; uint32_t stop_timeout; }; static struct resource_spec aml8726_mmc_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { -1, 0 } }; #define AML_MMC_LOCK(sc) mtx_lock(&(sc)->mtx) #define AML_MMC_UNLOCK(sc) mtx_unlock(&(sc)->mtx) #define AML_MMC_LOCK_ASSERT(sc) mtx_assert(&(sc)->mtx, MA_OWNED) #define AML_MMC_LOCK_INIT(sc) \ mtx_init(&(sc)->mtx, device_get_nameunit((sc)->dev), \ "mmc", MTX_DEF) #define AML_MMC_LOCK_DESTROY(sc) mtx_destroy(&(sc)->mtx); #define CSR_WRITE_4(sc, reg, val) bus_write_4((sc)->res[0], reg, (val)) #define CSR_READ_4(sc, reg) bus_read_4((sc)->res[0], reg) #define CSR_BARRIER(sc, reg) bus_barrier((sc)->res[0], reg, 4, \ (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) #define PWR_ON_FLAG(pol) ((pol) == 0 ? GPIO_PIN_LOW : \ GPIO_PIN_HIGH) #define PWR_OFF_FLAG(pol) ((pol) == 0 ? GPIO_PIN_HIGH : \ GPIO_PIN_LOW) #define MSECS_TO_TICKS(ms) (((ms)*hz)/1000 + 1) static void aml8726_mmc_timeout(void *arg); static unsigned int aml8726_mmc_clk(phandle_t node) { pcell_t prop; ssize_t len; phandle_t clk_node; len = OF_getencprop(node, "clocks", &prop, sizeof(prop)); if ((len / sizeof(prop)) != 1 || prop == 0 || (clk_node = OF_node_from_xref(prop)) == 0) return (0); len = OF_getencprop(clk_node, "clock-frequency", &prop, sizeof(prop)); if ((len / sizeof(prop)) != 1 || prop == 0) return (0); return ((unsigned int)prop); } static uint32_t aml8726_mmc_freq(struct aml8726_mmc_softc *sc, uint32_t divisor) { return (sc->ref_freq / ((divisor + 1) * 2)); } static uint32_t aml8726_mmc_div(struct aml8726_mmc_softc *sc, uint32_t desired_freq) { uint32_t divisor; divisor = sc->ref_freq / (desired_freq * 2); if (divisor == 0) divisor = 1; divisor -= 1; if (aml8726_mmc_freq(sc, divisor) > desired_freq) divisor += 1; if (divisor > (AML_MMC_CONFIG_CMD_CLK_DIV_MASK >> AML_MMC_CONFIG_CMD_CLK_DIV_SHIFT)) { divisor = AML_MMC_CONFIG_CMD_CLK_DIV_MASK >> AML_MMC_CONFIG_CMD_CLK_DIV_SHIFT; } return (divisor); } static void aml8726_mmc_mapmem(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *busaddrp; /* * There should only be one bus space address since * bus_dma_tag_create was called with nsegments = 1. */ busaddrp = (bus_addr_t *)arg; *busaddrp = segs->ds_addr; } static int aml8726_mmc_power_off(struct aml8726_mmc_softc *sc) { if (sc->pwr_en.dev == NULL) return (0); return (GPIO_PIN_SET(sc->pwr_en.dev, sc->pwr_en.pin, PWR_OFF_FLAG(sc->pwr_en.pol))); } static int aml8726_mmc_power_on(struct aml8726_mmc_softc *sc) { if (sc->pwr_en.dev == NULL) return (0); return (GPIO_PIN_SET(sc->pwr_en.dev, sc->pwr_en.pin, PWR_ON_FLAG(sc->pwr_en.pol))); } static void aml8726_mmc_soft_reset(struct aml8726_mmc_softc *sc, boolean_t enable_irq) { uint32_t icr; icr = AML_MMC_IRQ_CONFIG_SOFT_RESET; if (enable_irq == true) icr |= AML_MMC_IRQ_CONFIG_CMD_DONE_EN; CSR_WRITE_4(sc, AML_MMC_IRQ_CONFIG_REG, icr); CSR_BARRIER(sc, AML_MMC_IRQ_CONFIG_REG); } static int aml8726_mmc_start_command(struct aml8726_mmc_softc *sc, struct mmc_command *cmd) { struct mmc_ios *ios = &sc->host.ios; bus_addr_t baddr; uint32_t block_size; uint32_t bus_width; uint32_t cmdr; uint32_t extr; uint32_t mcfgr; uint32_t nbits_per_pkg; uint32_t timeout; int error; struct mmc_data *data; if (cmd->opcode > 0x3f) return (MMC_ERR_INVALID); /* * Ensure the hardware state machine is in a known state. */ aml8726_mmc_soft_reset(sc, true); /* * Start and transmission bits are per section 4.7.2 of the: * * SD Specifications Part 1 - * Physicaly Layer Simplified Specification + * Physical Layer Simplified Specification * Version 4.10 */ cmdr = AML_MMC_CMD_START_BIT | AML_MMC_CMD_TRANS_BIT_HOST | cmd->opcode; baddr = 0; extr = 0; mcfgr = sc->port; timeout = AML_MMC_CMD_TIMEOUT; /* * If this is a linked command, then use the previous timeout. */ if (cmd == cmd->mrq->stop && sc->stop_timeout) timeout = sc->stop_timeout; sc->stop_timeout = 0; if ((cmd->flags & MMC_RSP_136) != 0) { cmdr |= AML_MMC_CMD_RESP_CRC7_FROM_8; cmdr |= (133 << AML_MMC_CMD_RESP_BITS_SHIFT); } else if ((cmd->flags & MMC_RSP_PRESENT) != 0) cmdr |= (45 << AML_MMC_CMD_RESP_BITS_SHIFT); if ((cmd->flags & MMC_RSP_CRC) == 0) cmdr |= AML_MMC_CMD_RESP_NO_CRC7; if ((cmd->flags & MMC_RSP_BUSY) != 0) cmdr |= AML_MMC_CMD_CHECK_DAT0_BUSY; data = cmd->data; if (data && data->len && (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) != 0) { block_size = data->len; if ((data->flags & MMC_DATA_MULTI) != 0) { block_size = MMC_SECTOR_SIZE; if ((data->len % block_size) != 0) return (MMC_ERR_INVALID); } cmdr |= (((data->len / block_size) - 1) << AML_MMC_CMD_REP_PKG_CNT_SHIFT); mcfgr |= (data->flags & MMC_DATA_STREAM) ? AML_MMC_MULT_CONFIG_STREAM_EN : 0; /* * The number of bits per package equals the number * of data bits + the number of CRC bits. There are * 16 bits of CRC calculate per bus line. * * A completed package appears to be detected by when * a counter decremented by the width underflows, thus * a value of zero always transfers 1 (or 4 bits depending * on the mode) which is why bus_width is subtracted. */ bus_width = (ios->bus_width == bus_width_4) ? 4 : 1; nbits_per_pkg = block_size * 8 + 16 * bus_width - bus_width; if (nbits_per_pkg > 0x3fff) return (MMC_ERR_INVALID); extr |= (nbits_per_pkg << AML_MMC_EXTENSION_PKT_SIZE_SHIFT); error = bus_dmamap_load(sc->dmatag, sc->dmamap, data->data, data->len, aml8726_mmc_mapmem, &baddr, BUS_DMA_NOWAIT); if (error) return (MMC_ERR_NO_MEMORY); if ((data->flags & MMC_DATA_READ) != 0) { cmdr |= AML_MMC_CMD_RESP_HAS_DATA; bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_PREREAD); timeout = AML_MMC_READ_TIMEOUT * (data->len / block_size); } else { cmdr |= AML_MMC_CMD_CMD_HAS_DATA; bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_PREWRITE); timeout = AML_MMC_WRITE_TIMEOUT * (data->len / block_size); } /* * Stop terminates a multiblock read / write and thus * can take as long to execute as an actual read / write. */ if (cmd->mrq->stop != NULL) sc->stop_timeout = timeout; } sc->cmd = cmd; cmd->error = MMC_ERR_NONE; if (timeout > AML_MMC_MAX_TIMEOUT) timeout = AML_MMC_MAX_TIMEOUT; callout_reset(&sc->ch, MSECS_TO_TICKS(timeout), aml8726_mmc_timeout, sc); CSR_WRITE_4(sc, AML_MMC_CMD_ARGUMENT_REG, cmd->arg); CSR_WRITE_4(sc, AML_MMC_MULT_CONFIG_REG, mcfgr); CSR_WRITE_4(sc, AML_MMC_EXTENSION_REG, extr); CSR_WRITE_4(sc, AML_MMC_DMA_ADDR_REG, (uint32_t)baddr); CSR_WRITE_4(sc, AML_MMC_CMD_SEND_REG, cmdr); CSR_BARRIER(sc, AML_MMC_CMD_SEND_REG); return (MMC_ERR_NONE); } static void aml8726_mmc_finish_command(struct aml8726_mmc_softc *sc, int mmc_error) { int mmc_stop_error; struct mmc_command *cmd; struct mmc_command *stop_cmd; struct mmc_data *data; AML_MMC_LOCK_ASSERT(sc); /* Clear all interrupts since the request is no longer in flight. */ CSR_WRITE_4(sc, AML_MMC_IRQ_STATUS_REG, AML_MMC_IRQ_STATUS_CLEAR_IRQ); CSR_BARRIER(sc, AML_MMC_IRQ_STATUS_REG); /* In some cases (e.g. finish called via timeout) this is a NOP. */ callout_stop(&sc->ch); cmd = sc->cmd; sc->cmd = NULL; cmd->error = mmc_error; data = cmd->data; if (data && data->len && (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) != 0) { if ((data->flags & MMC_DATA_READ) != 0) bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_POSTREAD); else bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->dmatag, sc->dmamap); } /* * If there's a linked stop command, then start the stop command. * In order to establish a known state attempt the stop command * even if the original request encountered an error. */ stop_cmd = (cmd->mrq->stop != cmd) ? cmd->mrq->stop : NULL; if (stop_cmd != NULL) { mmc_stop_error = aml8726_mmc_start_command(sc, stop_cmd); if (mmc_stop_error == MMC_ERR_NONE) { AML_MMC_UNLOCK(sc); return; } stop_cmd->error = mmc_stop_error; } AML_MMC_UNLOCK(sc); /* Execute the callback after dropping the lock. */ if (cmd->mrq) cmd->mrq->done(cmd->mrq); } static void aml8726_mmc_timeout(void *arg) { struct aml8726_mmc_softc *sc = (struct aml8726_mmc_softc *)arg; /* * The command failed to complete in time so forcefully * terminate it. */ aml8726_mmc_soft_reset(sc, false); /* * Ensure the command has terminated before continuing on * to things such as bus_dmamap_sync / bus_dmamap_unload. */ while ((CSR_READ_4(sc, AML_MMC_IRQ_STATUS_REG) & AML_MMC_IRQ_STATUS_CMD_BUSY) != 0) cpu_spinwait(); aml8726_mmc_finish_command(sc, MMC_ERR_TIMEOUT); } static void aml8726_mmc_intr(void *arg) { struct aml8726_mmc_softc *sc = (struct aml8726_mmc_softc *)arg; uint32_t cmdr; uint32_t isr; uint32_t mcfgr; uint32_t previous_byte; uint32_t resp; int mmc_error; unsigned int i; AML_MMC_LOCK(sc); isr = CSR_READ_4(sc, AML_MMC_IRQ_STATUS_REG); cmdr = CSR_READ_4(sc, AML_MMC_CMD_SEND_REG); if (sc->cmd == NULL) goto spurious; mmc_error = MMC_ERR_NONE; if ((isr & AML_MMC_IRQ_STATUS_CMD_DONE_IRQ) != 0) { /* Check for CRC errors if the command has completed. */ if ((cmdr & AML_MMC_CMD_RESP_NO_CRC7) == 0 && (isr & AML_MMC_IRQ_STATUS_RESP_CRC7_OK) == 0) mmc_error = MMC_ERR_BADCRC; if ((cmdr & AML_MMC_CMD_RESP_HAS_DATA) != 0 && (isr & AML_MMC_IRQ_STATUS_RD_CRC16_OK) == 0) mmc_error = MMC_ERR_BADCRC; if ((cmdr & AML_MMC_CMD_CMD_HAS_DATA) != 0 && (isr & AML_MMC_IRQ_STATUS_WR_CRC16_OK) == 0) mmc_error = MMC_ERR_BADCRC; } else { spurious: /* * Clear spurious interrupts while leaving intacted any * interrupts that may have occurred after we read the * interrupt status register. */ CSR_WRITE_4(sc, AML_MMC_IRQ_STATUS_REG, (AML_MMC_IRQ_STATUS_CLEAR_IRQ & isr)); CSR_BARRIER(sc, AML_MMC_IRQ_STATUS_REG); AML_MMC_UNLOCK(sc); return; } if ((cmdr & AML_MMC_CMD_RESP_BITS_MASK) != 0) { mcfgr = sc->port; mcfgr |= AML_MMC_MULT_CONFIG_RESP_READOUT_EN; CSR_WRITE_4(sc, AML_MMC_MULT_CONFIG_REG, mcfgr); if ((cmdr & AML_MMC_CMD_RESP_CRC7_FROM_8) != 0) { /* * Controller supplies 135:8 instead of * 127:0 so discard the leading 8 bits * and provide a trailing 8 zero bits * where the CRC belongs. */ previous_byte = 0; for (i = 0; i < 4; i++) { resp = CSR_READ_4(sc, AML_MMC_CMD_ARGUMENT_REG); sc->cmd->resp[3 - i] = (resp << 8) | previous_byte; previous_byte = (resp >> 24) & 0xff; } } else sc->cmd->resp[0] = CSR_READ_4(sc, AML_MMC_CMD_ARGUMENT_REG); } if ((isr & AML_MMC_IRQ_STATUS_CMD_BUSY) != 0 && /* * A multiblock operation may keep the hardware * busy until stop transmission is executed. */ (isr & AML_MMC_IRQ_STATUS_CMD_DONE_IRQ) == 0) { if (mmc_error == MMC_ERR_NONE) mmc_error = MMC_ERR_FAILED; /* * Issue a soft reset to terminate the command. * * Ensure the command has terminated before continuing on * to things such as bus_dmamap_sync / bus_dmamap_unload. */ aml8726_mmc_soft_reset(sc, false); while ((CSR_READ_4(sc, AML_MMC_IRQ_STATUS_REG) & AML_MMC_IRQ_STATUS_CMD_BUSY) != 0) cpu_spinwait(); } aml8726_mmc_finish_command(sc, mmc_error); } static int aml8726_mmc_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "amlogic,aml8726-mmc")) return (ENXIO); device_set_desc(dev, "Amlogic aml8726 MMC"); return (BUS_PROBE_DEFAULT); } static int aml8726_mmc_attach(device_t dev) { struct aml8726_mmc_softc *sc = device_get_softc(dev); char *function_name; char *voltages; char *voltage; int error; int nvoltages; pcell_t prop[3]; phandle_t node; ssize_t len; device_t child; sc->dev = dev; node = ofw_bus_get_node(dev); sc->ref_freq = aml8726_mmc_clk(node); if (sc->ref_freq == 0) { device_printf(dev, "missing clocks attribute in FDT\n"); return (ENXIO); } /* * The pins must be specified as part of the device in order * to know which port to used. */ len = OF_getencprop(node, "pinctrl-0", prop, sizeof(prop)); if ((len / sizeof(prop[0])) != 1 || prop[0] == 0) { device_printf(dev, "missing pinctrl-0 attribute in FDT\n"); return (ENXIO); } len = OF_getprop_alloc(OF_node_from_xref(prop[0]), "amlogic,function", sizeof(char), (void **)&function_name); if (len < 0) { device_printf(dev, "missing amlogic,function attribute in FDT\n"); return (ENXIO); } if (strncmp("sdio-a", function_name, len) == 0) sc->port = AML_MMC_MULT_CONFIG_PORT_A; else if (strncmp("sdio-b", function_name, len) == 0) sc->port = AML_MMC_MULT_CONFIG_PORT_B; else if (strncmp("sdio-c", function_name, len) == 0) sc->port = AML_MMC_MULT_CONFIG_PORT_C; else { device_printf(dev, "unknown function attribute %.*s in FDT\n", len, function_name); free(function_name, M_OFWPROP); return (ENXIO); } free(function_name, M_OFWPROP); sc->pwr_en.dev = NULL; len = OF_getencprop(node, "mmc-pwr-en", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 3) { sc->pwr_en.dev = OF_device_from_xref(prop[0]); sc->pwr_en.pin = prop[1]; sc->pwr_en.pol = prop[2]; } if (sc->pwr_en.dev == NULL) { device_printf(dev, "unable to process mmc-pwr-en attribute in FDT\n"); return (ENXIO); } /* Turn off power and then configure the output driver. */ if (aml8726_mmc_power_off(sc) != 0 || GPIO_PIN_SETFLAGS(sc->pwr_en.dev, sc->pwr_en.pin, GPIO_PIN_OUTPUT) != 0) { device_printf(dev, "could not use gpio to control power\n"); return (ENXIO); } } len = OF_getprop_alloc(node, "mmc-voltages", sizeof(char), (void **)&voltages); if (len < 0) { device_printf(dev, "missing mmc-voltages attribute in FDT\n"); return (ENXIO); } sc->voltages[0] = 0; sc->voltages[1] = 0; voltage = voltages; nvoltages = 0; while (len && nvoltages < 2) { if (strncmp("1.8", voltage, len) == 0) sc->voltages[nvoltages] = MMC_OCR_LOW_VOLTAGE; else if (strncmp("3.3", voltage, len) == 0) sc->voltages[nvoltages] = MMC_OCR_320_330 | MMC_OCR_330_340; else { device_printf(dev, "unknown voltage attribute %.*s in FDT\n", len, voltage); free(voltages, M_OFWPROP); return (ENXIO); } nvoltages++; /* queue up next string */ while (*voltage && len) { voltage++; len--; } if (len) { voltage++; len--; } } free(voltages, M_OFWPROP); sc->vselect.dev = NULL; len = OF_getencprop(node, "mmc-vselect", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 2) { sc->vselect.dev = OF_device_from_xref(prop[0]); sc->vselect.pin = prop[1]; sc->vselect.pol = 1; } if (sc->vselect.dev == NULL) { device_printf(dev, "unable to process mmc-vselect attribute in FDT\n"); return (ENXIO); } /* * With the power off select voltage 0 and then * configure the output driver. */ if (GPIO_PIN_SET(sc->vselect.dev, sc->vselect.pin, 0) != 0 || GPIO_PIN_SETFLAGS(sc->vselect.dev, sc->vselect.pin, GPIO_PIN_OUTPUT) != 0) { device_printf(dev, "could not use gpio to set voltage\n"); return (ENXIO); } } if (nvoltages == 0) { device_printf(dev, "no voltages in FDT\n"); return (ENXIO); } else if (nvoltages == 1 && sc->vselect.dev != NULL) { device_printf(dev, "only one voltage in FDT\n"); return (ENXIO); } else if (nvoltages == 2 && sc->vselect.dev == NULL) { device_printf(dev, "too many voltages in FDT\n"); return (ENXIO); } if (bus_alloc_resources(dev, aml8726_mmc_spec, sc->res)) { device_printf(dev, "could not allocate resources for device\n"); return (ENXIO); } AML_MMC_LOCK_INIT(sc); error = bus_dma_tag_create(bus_get_dma_tag(dev), AML_MMC_ALIGN_DMA, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, AML_MMC_MAX_DMA, 1, AML_MMC_MAX_DMA, 0, NULL, NULL, &sc->dmatag); if (error) goto fail; error = bus_dmamap_create(sc->dmatag, 0, &sc->dmamap); if (error) goto fail; error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, NULL, aml8726_mmc_intr, sc, &sc->ih_cookie); if (error) { device_printf(dev, "could not setup interrupt handler\n"); goto fail; } callout_init_mtx(&sc->ch, &sc->mtx, CALLOUT_RETURNUNLOCKED); sc->host.f_min = aml8726_mmc_freq(sc, aml8726_mmc_div(sc, 200000)); sc->host.f_max = aml8726_mmc_freq(sc, aml8726_mmc_div(sc, 50000000)); sc->host.host_ocr = sc->voltages[0] | sc->voltages[1]; sc->host.caps = MMC_CAP_4_BIT_DATA | MMC_CAP_HSPEED; child = device_add_child(dev, "mmc", -1); if (!child) { device_printf(dev, "could not add mmc\n"); error = ENXIO; goto fail; } error = device_probe_and_attach(child); if (error) { device_printf(dev, "could not attach mmc\n"); goto fail; } return (0); fail: if (sc->ih_cookie) bus_teardown_intr(dev, sc->res[1], sc->ih_cookie); if (sc->dmamap) bus_dmamap_destroy(sc->dmatag, sc->dmamap); if (sc->dmatag) bus_dma_tag_destroy(sc->dmatag); AML_MMC_LOCK_DESTROY(sc); aml8726_mmc_power_off(sc); bus_release_resources(dev, aml8726_mmc_spec, sc->res); return (error); } static int aml8726_mmc_detach(device_t dev) { struct aml8726_mmc_softc *sc = device_get_softc(dev); AML_MMC_LOCK(sc); if (sc->cmd != NULL) { AML_MMC_UNLOCK(sc); return (EBUSY); } /* * Turn off the power, reset the hardware state machine, * disable the interrupts, and clear the interrupts. */ (void)aml8726_mmc_power_off(sc); aml8726_mmc_soft_reset(sc, false); CSR_WRITE_4(sc, AML_MMC_IRQ_STATUS_REG, AML_MMC_IRQ_STATUS_CLEAR_IRQ); /* This should be a NOP since no command was in flight. */ callout_stop(&sc->ch); AML_MMC_UNLOCK(sc); bus_generic_detach(dev); bus_teardown_intr(dev, sc->res[1], sc->ih_cookie); bus_dmamap_destroy(sc->dmatag, sc->dmamap); bus_dma_tag_destroy(sc->dmatag); AML_MMC_LOCK_DESTROY(sc); bus_release_resources(dev, aml8726_mmc_spec, sc->res); return (0); } static int aml8726_mmc_shutdown(device_t dev) { struct aml8726_mmc_softc *sc = device_get_softc(dev); /* * Turn off the power, reset the hardware state machine, * disable the interrupts, and clear the interrupts. */ (void)aml8726_mmc_power_off(sc); aml8726_mmc_soft_reset(sc, false); CSR_WRITE_4(sc, AML_MMC_IRQ_STATUS_REG, AML_MMC_IRQ_STATUS_CLEAR_IRQ); return (0); } static int aml8726_mmc_update_ios(device_t bus, device_t child) { struct aml8726_mmc_softc *sc = device_get_softc(bus); struct mmc_ios *ios = &sc->host.ios; int error; int i; uint32_t cfgr; cfgr = (2 << AML_MMC_CONFIG_WR_CRC_STAT_SHIFT) | (2 << AML_MMC_CONFIG_WR_DELAY_SHIFT) | AML_MMC_CONFIG_DMA_ENDIAN_SBW | (39 << AML_MMC_CONFIG_CMD_ARG_BITS_SHIFT); switch (ios->bus_width) { case bus_width_4: cfgr |= AML_MMC_CONFIG_BUS_WIDTH_4; break; case bus_width_1: cfgr |= AML_MMC_CONFIG_BUS_WIDTH_1; break; default: return (EINVAL); } cfgr |= aml8726_mmc_div(sc, ios->clock) << AML_MMC_CONFIG_CMD_CLK_DIV_SHIFT; CSR_WRITE_4(sc, AML_MMC_CONFIG_REG, cfgr); error = 0; switch (ios->power_mode) { case power_up: /* * Configure and power on the regulator so that the * voltage stabilizes prior to powering on the card. */ if (sc->vselect.dev != NULL) { for (i = 0; i < 2; i++) if ((sc->voltages[i] & (1 << ios->vdd)) != 0) break; if (i >= 2) return (EINVAL); error = GPIO_PIN_SET(sc->vselect.dev, sc->vselect.pin, i); } break; case power_on: error = aml8726_mmc_power_on(sc); break; case power_off: error = aml8726_mmc_power_off(sc); break; default: return (EINVAL); } return (error); } static int aml8726_mmc_request(device_t bus, device_t child, struct mmc_request *req) { struct aml8726_mmc_softc *sc = device_get_softc(bus); int mmc_error; AML_MMC_LOCK(sc); if (sc->cmd != NULL) { AML_MMC_UNLOCK(sc); return (EBUSY); } mmc_error = aml8726_mmc_start_command(sc, req->cmd); AML_MMC_UNLOCK(sc); /* Execute the callback after dropping the lock. */ if (mmc_error != MMC_ERR_NONE) { req->cmd->error = mmc_error; req->done(req); } return (0); } static int aml8726_mmc_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct aml8726_mmc_softc *sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: *(int *)result = sc->host.ios.bus_mode; break; case MMCBR_IVAR_BUS_WIDTH: *(int *)result = sc->host.ios.bus_width; break; case MMCBR_IVAR_CHIP_SELECT: *(int *)result = sc->host.ios.chip_select; break; case MMCBR_IVAR_CLOCK: *(int *)result = sc->host.ios.clock; break; case MMCBR_IVAR_F_MIN: *(int *)result = sc->host.f_min; break; case MMCBR_IVAR_F_MAX: *(int *)result = sc->host.f_max; break; case MMCBR_IVAR_HOST_OCR: *(int *)result = sc->host.host_ocr; break; case MMCBR_IVAR_MODE: *(int *)result = sc->host.mode; break; case MMCBR_IVAR_OCR: *(int *)result = sc->host.ocr; break; case MMCBR_IVAR_POWER_MODE: *(int *)result = sc->host.ios.power_mode; break; case MMCBR_IVAR_VDD: *(int *)result = sc->host.ios.vdd; break; case MMCBR_IVAR_CAPS: *(int *)result = sc->host.caps; break; case MMCBR_IVAR_MAX_DATA: *(int *)result = AML_MMC_MAX_DMA / MMC_SECTOR_SIZE; break; default: return (EINVAL); } return (0); } static int aml8726_mmc_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct aml8726_mmc_softc *sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: sc->host.ios.bus_mode = value; break; case MMCBR_IVAR_BUS_WIDTH: sc->host.ios.bus_width = value; break; case MMCBR_IVAR_CHIP_SELECT: sc->host.ios.chip_select = value; break; case MMCBR_IVAR_CLOCK: sc->host.ios.clock = value; break; case MMCBR_IVAR_MODE: sc->host.mode = value; break; case MMCBR_IVAR_OCR: sc->host.ocr = value; break; case MMCBR_IVAR_POWER_MODE: sc->host.ios.power_mode = value; break; case MMCBR_IVAR_VDD: sc->host.ios.vdd = value; break; /* These are read-only */ case MMCBR_IVAR_CAPS: case MMCBR_IVAR_HOST_OCR: case MMCBR_IVAR_F_MIN: case MMCBR_IVAR_F_MAX: case MMCBR_IVAR_MAX_DATA: default: return (EINVAL); } return (0); } static int aml8726_mmc_get_ro(device_t bus, device_t child) { return (0); } static int aml8726_mmc_acquire_host(device_t bus, device_t child) { struct aml8726_mmc_softc *sc = device_get_softc(bus); AML_MMC_LOCK(sc); while (sc->bus_busy) mtx_sleep(sc, &sc->mtx, PZERO, "mmc", hz / 5); sc->bus_busy++; AML_MMC_UNLOCK(sc); return (0); } static int aml8726_mmc_release_host(device_t bus, device_t child) { struct aml8726_mmc_softc *sc = device_get_softc(bus); AML_MMC_LOCK(sc); sc->bus_busy--; wakeup(sc); AML_MMC_UNLOCK(sc); return (0); } static device_method_t aml8726_mmc_methods[] = { /* Device interface */ DEVMETHOD(device_probe, aml8726_mmc_probe), DEVMETHOD(device_attach, aml8726_mmc_attach), DEVMETHOD(device_detach, aml8726_mmc_detach), DEVMETHOD(device_shutdown, aml8726_mmc_shutdown), /* Bus interface */ DEVMETHOD(bus_read_ivar, aml8726_mmc_read_ivar), DEVMETHOD(bus_write_ivar, aml8726_mmc_write_ivar), /* MMC bridge interface */ DEVMETHOD(mmcbr_update_ios, aml8726_mmc_update_ios), DEVMETHOD(mmcbr_request, aml8726_mmc_request), DEVMETHOD(mmcbr_get_ro, aml8726_mmc_get_ro), DEVMETHOD(mmcbr_acquire_host, aml8726_mmc_acquire_host), DEVMETHOD(mmcbr_release_host, aml8726_mmc_release_host), DEVMETHOD_END }; static driver_t aml8726_mmc_driver = { "aml8726_mmc", aml8726_mmc_methods, sizeof(struct aml8726_mmc_softc), }; static devclass_t aml8726_mmc_devclass; DRIVER_MODULE(aml8726_mmc, simplebus, aml8726_mmc_driver, aml8726_mmc_devclass, 0, 0); MODULE_DEPEND(aml8726_mmc, aml8726_gpio, 1, 1, 1); DRIVER_MODULE(mmc, aml8726_mmc, mmc_driver, mmc_devclass, NULL, NULL); MODULE_DEPEND(aml8726_mmc, mmc, 1, 1, 1); Index: head/sys/arm/amlogic/aml8726/aml8726_mmc.h =================================================================== --- head/sys/arm/amlogic/aml8726/aml8726_mmc.h (revision 299068) +++ head/sys/arm/amlogic/aml8726/aml8726_mmc.h (revision 299069) @@ -1,123 +1,123 @@ /*- * Copyright 2013-2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _ARM_AMLOGIC_AML8726_MMC_H #define _ARM_AMLOGIC_AML8726_MMC_H #define AML_MMC_ALIGN_DMA 4 #define AML_MMC_MAX_DMA 4096 /* * Timeouts are in milliseconds * * Read and write are per section 4.6.2 of the: * * SD Specifications Part 1 - * Physicaly Layer Simplified Specification + * Physical Layer Simplified Specification * Version 4.10 */ #define AML_MMC_CMD_TIMEOUT 50 #define AML_MMC_READ_TIMEOUT 100 #define AML_MMC_WRITE_TIMEOUT 500 #define AML_MMC_MAX_TIMEOUT 5000 #define AML_MMC_CMD_ARGUMENT_REG 0 #define AML_MMC_CMD_SEND_REG 4 #define AML_MMC_CMD_REP_PKG_CNT_MASK (0xffU << 24) #define AML_MMC_CMD_REP_PKG_CNT_SHIFT 24 #define AML_MMC_CMD_CMD_HAS_DATA (1 << 20) #define AML_MMC_CMD_CHECK_DAT0_BUSY (1 << 19) #define AML_MMC_CMD_RESP_CRC7_FROM_8 (1 << 18) #define AML_MMC_CMD_RESP_HAS_DATA (1 << 17) #define AML_MMC_CMD_RESP_NO_CRC7 (1 << 16) #define AML_MMC_CMD_RESP_BITS_MASK (0xff << 8) #define AML_MMC_CMD_RESP_BITS_SHIFT 8 #define AML_MMC_CMD_START_BIT (0 << 7) #define AML_MMC_CMD_TRANS_BIT_HOST (1 << 6) #define AML_MMC_CMD_INDEX_MASK 0x3f #define AML_MMC_CMD_INDEX_SHIFT 0 #define AML_MMC_CONFIG_REG 8 #define AML_MMC_CONFIG_WR_CRC_STAT_MASK (7U << 29) #define AML_MMC_CONFIG_WR_CRC_STAT_SHIFT 29 #define AML_MMC_CONFIG_WR_DELAY_MASK (0x3f << 23) #define AML_MMC_CONFIG_WR_DELAY_SHIFT 23 #define AML_MMC_CONFIG_DMA_ENDIAN_MASK (3 << 21) #define AML_MMC_CONFIG_DMA_ENDIAN_NC (0 << 21) #define AML_MMC_CONFIG_DMA_ENDIAN_SB (1 << 21) #define AML_MMC_CONFIG_DMA_ENDIAN_SW (2 << 21) #define AML_MMC_CONFIG_DMA_ENDIAN_SBW (3 << 21) #define AML_MMC_CONFIG_BUS_WIDTH_MASK (1 << 20) #define AML_MMC_CONFIG_BUS_WIDTH_1 (0 << 20) #define AML_MMC_CONFIG_BUS_WIDTH_4 (1 << 20) #define AML_MMC_CONFIG_DATA_NEG_EDGE (1 << 19) #define AML_MMC_CONFIG_DONT_DELAY_DATA (1 << 18) #define AML_MMC_CONFIG_CMD_ARG_BITS_MASK (0x3f << 12) #define AML_MMC_CONFIG_CMD_ARG_BITS_SHIFT 12 #define AML_MMC_CONFIG_CMD_POS_EDGE (1 << 11) #define AML_MMC_CONFIG_CMD_NO_CRC (1 << 10) #define AML_MMC_CONFIG_CMD_CLK_DIV_MASK 0x3ff #define AML_MMC_CONFIG_CMD_CLK_DIV_SHIFT 0 #define AML_MMC_IRQ_STATUS_REG 12 #define AML_MMC_IRQ_STATUS_TIMER_CNT_MASK (0x1fffU << 19) #define AML_MMC_IRQ_STATUS_TIMER_CNT_SHIFT 19 #define AML_MMC_IRQ_STATUS_TIMER_EN (1 << 18) #define AML_MMC_IRQ_STATUS_TIMEOUT_IRQ (1 << 16) #define AML_MMC_IRQ_STATUS_CMD_DONE_IRQ (1 << 9) #define AML_MMC_IRQ_STATUS_WR_CRC16_OK (1 << 7) #define AML_MMC_IRQ_STATUS_RD_CRC16_OK (1 << 6) #define AML_MMC_IRQ_STATUS_RESP_CRC7_OK (1 << 5) #define AML_MMC_IRQ_STATUS_CMD_BUSY (1 << 4) #define AML_MMC_IRQ_STATUS_CLEAR_IRQ 0x10700 #define AML_MMC_IRQ_CONFIG_REG 16 #define AML_MMC_IRQ_CONFIG_SOFT_RESET (1 << 15) #define AML_MMC_IRQ_CONFIG_CMD_DONE_EN (1 << 4) #define AML_MMC_MULT_CONFIG_REG 20 #define AML_MMC_MULT_CONFIG_RESP_INDEX_MASK (0xf << 12) #define AML_MMC_MULT_CONFIG_RESP_INDEX_SHIFT 12 #define AML_MMC_MULT_CONFIG_RESP_READOUT_EN (1 << 8) #define AML_MMC_MULT_CONFIG_STREAM_8_MODE (1 << 5) #define AML_MMC_MULT_CONFIG_STREAM_EN (1 << 4) #define AML_MMC_MULT_CONFIG_PORT_MASK 3 #define AML_MMC_MULT_CONFIG_PORT_A 0 #define AML_MMC_MULT_CONFIG_PORT_B 1 #define AML_MMC_MULT_CONFIG_PORT_C 2 #define AML_MMC_DMA_ADDR_REG 24 #define AML_MMC_EXTENSION_REG 28 #define AML_MMC_EXTENSION_NO_CRC16 (1 << 30) #define AML_MMC_EXTENSION_PKT_SIZE_MASK (0x3fff << 16) #define AML_MMC_EXTENSION_PKT_SIZE_SHIFT 16 #endif /* _ARM_AMLOGIC_AML8726_MMC_H */ Index: head/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.c =================================================================== --- head/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.c (revision 299068) +++ head/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.c (revision 299069) @@ -1,1381 +1,1381 @@ /*- * Copyright 2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Amlogic aml8726-m8 (and later) SDXC host controller driver. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include "mmcbr_if.h" /* * The table is sorted from highest to lowest and * last entry in the table is mark by freq == 0. */ struct { uint32_t voltage; uint32_t freq; uint32_t rx_phase; } aml8726_sdxc_clk_phases[] = { { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 100000000, 1 }, { MMC_OCR_320_330 | MMC_OCR_330_340, 45000000, 15 }, { MMC_OCR_LOW_VOLTAGE, 45000000, 11 }, { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 24999999, 15 }, { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 5000000, 23 }, { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 1000000, 55 }, { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 0, 1061 }, }; struct aml8726_sdxc_gpio { device_t dev; uint32_t pin; uint32_t pol; }; struct aml8726_sdxc_softc { device_t dev; boolean_t auto_fill_flush; struct resource *res[2]; struct mtx mtx; struct callout ch; unsigned int ref_freq; struct aml8726_sdxc_gpio pwr_en; int voltages[2]; struct aml8726_sdxc_gpio vselect; struct aml8726_sdxc_gpio card_rst; bus_dma_tag_t dmatag; bus_dmamap_t dmamap; void *ih_cookie; struct mmc_host host; int bus_busy; struct { uint32_t time; uint32_t error; } busy; struct mmc_command *cmd; }; static struct resource_spec aml8726_sdxc_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { -1, 0 } }; #define AML_SDXC_LOCK(sc) mtx_lock(&(sc)->mtx) #define AML_SDXC_UNLOCK(sc) mtx_unlock(&(sc)->mtx) #define AML_SDXC_LOCK_ASSERT(sc) mtx_assert(&(sc)->mtx, MA_OWNED) #define AML_SDXC_LOCK_INIT(sc) \ mtx_init(&(sc)->mtx, device_get_nameunit((sc)->dev), \ "sdxc", MTX_DEF) #define AML_SDXC_LOCK_DESTROY(sc) mtx_destroy(&(sc)->mtx); #define CSR_WRITE_4(sc, reg, val) bus_write_4((sc)->res[0], reg, (val)) #define CSR_READ_4(sc, reg) bus_read_4((sc)->res[0], reg) #define CSR_BARRIER(sc, reg) bus_barrier((sc)->res[0], reg, 4, \ (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) #define PIN_ON_FLAG(pol) ((pol) == 0 ? \ GPIO_PIN_LOW : GPIO_PIN_HIGH) #define PIN_OFF_FLAG(pol) ((pol) == 0 ? \ GPIO_PIN_HIGH : GPIO_PIN_LOW) #define msecs_to_ticks(ms) (((ms)*hz)/1000 + 1) static void aml8726_sdxc_timeout(void *arg); static void aml8726_sdxc_mapmem(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *busaddrp; /* * There should only be one bus space address since * bus_dma_tag_create was called with nsegments = 1. */ busaddrp = (bus_addr_t *)arg; *busaddrp = segs->ds_addr; } static int aml8726_sdxc_power_off(struct aml8726_sdxc_softc *sc) { if (sc->pwr_en.dev == NULL) return (0); return (GPIO_PIN_SET(sc->pwr_en.dev, sc->pwr_en.pin, PIN_OFF_FLAG(sc->pwr_en.pol))); } static int aml8726_sdxc_power_on(struct aml8726_sdxc_softc *sc) { if (sc->pwr_en.dev == NULL) return (0); return (GPIO_PIN_SET(sc->pwr_en.dev, sc->pwr_en.pin, PIN_ON_FLAG(sc->pwr_en.pol))); } static void aml8726_sdxc_soft_reset(struct aml8726_sdxc_softc *sc) { CSR_WRITE_4(sc, AML_SDXC_SOFT_RESET_REG, AML_SDXC_SOFT_RESET); CSR_BARRIER(sc, AML_SDXC_SOFT_RESET_REG); DELAY(5); } static void aml8726_sdxc_engage_dma(struct aml8726_sdxc_softc *sc) { int i; uint32_t pdmar; uint32_t sr; struct mmc_data *data; data = sc->cmd->data; if (data == NULL || data->len == 0) return; /* * Engaging the DMA hardware is recommended before writing * to AML_SDXC_SEND_REG so that the FIFOs are ready to go. * * Presumably AML_SDXC_CNTRL_REG and AML_SDXC_DMA_ADDR_REG * must be set up prior to this happening. */ pdmar = CSR_READ_4(sc, AML_SDXC_PDMA_REG); pdmar &= ~AML_SDXC_PDMA_RX_FLUSH_MODE_SW; pdmar |= AML_SDXC_PDMA_DMA_EN; if (sc->auto_fill_flush == true) { CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); return; } if ((data->flags & MMC_DATA_READ) != 0) { pdmar |= AML_SDXC_PDMA_RX_FLUSH_MODE_SW; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); } else { pdmar |= AML_SDXC_PDMA_TX_FILL; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); /* * Wait up to 100us for data to show up. */ for (i = 0; i < 100; i++) { sr = CSR_READ_4(sc, AML_SDXC_STATUS_REG); if ((sr & AML_SDXC_STATUS_TX_CNT_MASK) != 0) break; DELAY(1); } if (i >= 100) device_printf(sc->dev, "TX FIFO fill timeout\n"); } } static void aml8726_sdxc_disengage_dma(struct aml8726_sdxc_softc *sc) { int i; uint32_t pdmar; uint32_t sr; struct mmc_data *data; data = sc->cmd->data; if (data == NULL || data->len == 0) return; pdmar = CSR_READ_4(sc, AML_SDXC_PDMA_REG); if (sc->auto_fill_flush == true) { pdmar &= ~AML_SDXC_PDMA_DMA_EN; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); return; } if ((data->flags & MMC_DATA_READ) != 0) { pdmar |= AML_SDXC_PDMA_RX_FLUSH_NOW; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); /* * Wait up to 100us for data to drain. */ for (i = 0; i < 100; i++) { sr = CSR_READ_4(sc, AML_SDXC_STATUS_REG); if ((sr & AML_SDXC_STATUS_RX_CNT_MASK) == 0) break; DELAY(1); } if (i >= 100) device_printf(sc->dev, "RX FIFO drain timeout\n"); } pdmar &= ~(AML_SDXC_PDMA_DMA_EN | AML_SDXC_PDMA_RX_FLUSH_MODE_SW); CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); } static int aml8726_sdxc_start_command(struct aml8726_sdxc_softc *sc, struct mmc_command *cmd) { bus_addr_t baddr; uint32_t block_size; uint32_t ctlr; uint32_t ier; uint32_t sndr; uint32_t timeout; int error; struct mmc_data *data; AML_SDXC_LOCK_ASSERT(sc); if (cmd->opcode > 0x3f) return (MMC_ERR_INVALID); /* * Ensure the hardware state machine is in a known state. */ aml8726_sdxc_soft_reset(sc); sndr = cmd->opcode; if ((cmd->flags & MMC_RSP_136) != 0) { sndr |= AML_SDXC_SEND_CMD_HAS_RESP; sndr |= AML_SDXC_SEND_RESP_136; /* * According to the SD spec the 136 bit response is * used for getting the CID or CSD in which case the * CRC7 is embedded in the contents rather than being * calculated over the entire response (the controller * always checks the CRC7 over the entire response). */ sndr |= AML_SDXC_SEND_RESP_NO_CRC7_CHECK; } else if ((cmd->flags & MMC_RSP_PRESENT) != 0) sndr |= AML_SDXC_SEND_CMD_HAS_RESP; if ((cmd->flags & MMC_RSP_CRC) == 0) sndr |= AML_SDXC_SEND_RESP_NO_CRC7_CHECK; if (cmd->opcode == MMC_STOP_TRANSMISSION) sndr |= AML_SDXC_SEND_DATA_STOP; data = cmd->data; baddr = 0; ctlr = CSR_READ_4(sc, AML_SDXC_CNTRL_REG); ier = AML_SDXC_IRQ_ENABLE_STANDARD; timeout = AML_SDXC_CMD_TIMEOUT; ctlr &= ~AML_SDXC_CNTRL_PKG_LEN_MASK; if (data && data->len && (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) != 0) { block_size = data->len; if ((data->flags & MMC_DATA_MULTI) != 0) { block_size = MMC_SECTOR_SIZE; if ((data->len % block_size) != 0) return (MMC_ERR_INVALID); } if (block_size > 512) return (MMC_ERR_INVALID); sndr |= AML_SDXC_SEND_CMD_HAS_DATA; sndr |= ((data->flags & MMC_DATA_WRITE) != 0) ? AML_SDXC_SEND_DATA_WRITE : 0; sndr |= (((data->len / block_size) - 1) << AML_SDXC_SEND_REP_PKG_CNT_SHIFT); ctlr |= ((block_size < 512) ? block_size : 0) << AML_SDXC_CNTRL_PKG_LEN_SHIFT; ier &= ~AML_SDXC_IRQ_ENABLE_RESP_OK; ier |= (sc->auto_fill_flush == true || (data->flags & MMC_DATA_WRITE) != 0) ? AML_SDXC_IRQ_ENABLE_DMA_DONE : AML_SDXC_IRQ_ENABLE_TRANSFER_DONE_OK; error = bus_dmamap_load(sc->dmatag, sc->dmamap, data->data, data->len, aml8726_sdxc_mapmem, &baddr, BUS_DMA_NOWAIT); if (error) return (MMC_ERR_NO_MEMORY); if ((data->flags & MMC_DATA_READ) != 0) { bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_PREREAD); timeout = AML_SDXC_READ_TIMEOUT * (data->len / block_size); } else { bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_PREWRITE); timeout = AML_SDXC_WRITE_TIMEOUT * (data->len / block_size); } } sc->cmd = cmd; cmd->error = MMC_ERR_NONE; sc->busy.time = 0; sc->busy.error = MMC_ERR_NONE; if (timeout > AML_SDXC_MAX_TIMEOUT) timeout = AML_SDXC_MAX_TIMEOUT; callout_reset(&sc->ch, msecs_to_ticks(timeout), aml8726_sdxc_timeout, sc); CSR_WRITE_4(sc, AML_SDXC_IRQ_ENABLE_REG, ier); CSR_WRITE_4(sc, AML_SDXC_CNTRL_REG, ctlr); CSR_WRITE_4(sc, AML_SDXC_DMA_ADDR_REG, (uint32_t)baddr); CSR_WRITE_4(sc, AML_SDXC_CMD_ARGUMENT_REG, cmd->arg); aml8726_sdxc_engage_dma(sc); CSR_WRITE_4(sc, AML_SDXC_SEND_REG, sndr); CSR_BARRIER(sc, AML_SDXC_SEND_REG); return (MMC_ERR_NONE); } static void aml8726_sdxc_finish_command(struct aml8726_sdxc_softc *sc, int mmc_error) { int mmc_stop_error; struct mmc_command *cmd; struct mmc_command *stop_cmd; struct mmc_data *data; AML_SDXC_LOCK_ASSERT(sc); /* Clear all interrupts since the request is no longer in flight. */ CSR_WRITE_4(sc, AML_SDXC_IRQ_STATUS_REG, AML_SDXC_IRQ_STATUS_CLEAR); CSR_BARRIER(sc, AML_SDXC_IRQ_STATUS_REG); /* In some cases (e.g. finish called via timeout) this is a NOP. */ callout_stop(&sc->ch); cmd = sc->cmd; sc->cmd = NULL; cmd->error = mmc_error; data = cmd->data; if (data && data->len && (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) != 0) { if ((data->flags & MMC_DATA_READ) != 0) bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_POSTREAD); else bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->dmatag, sc->dmamap); } /* * If there's a linked stop command, then start the stop command. * In order to establish a known state attempt the stop command * even if the original request encountered an error. */ stop_cmd = (cmd->mrq->stop != cmd) ? cmd->mrq->stop : NULL; if (stop_cmd != NULL) { /* - * If the original command executed successfuly, then + * If the original command executed successfully, then * the hardware will also have automatically executed * a stop command so don't bother with the one supplied * with the original request. */ if (mmc_error == MMC_ERR_NONE) { stop_cmd->error = MMC_ERR_NONE; stop_cmd->resp[0] = cmd->resp[0]; stop_cmd->resp[1] = cmd->resp[1]; stop_cmd->resp[2] = cmd->resp[2]; stop_cmd->resp[3] = cmd->resp[3]; } else { mmc_stop_error = aml8726_sdxc_start_command(sc, stop_cmd); if (mmc_stop_error == MMC_ERR_NONE) { AML_SDXC_UNLOCK(sc); return; } stop_cmd->error = mmc_stop_error; } } AML_SDXC_UNLOCK(sc); /* Execute the callback after dropping the lock. */ if (cmd->mrq != NULL) cmd->mrq->done(cmd->mrq); } static void aml8726_sdxc_timeout(void *arg) { struct aml8726_sdxc_softc *sc = (struct aml8726_sdxc_softc *)arg; /* * The command failed to complete in time so forcefully * terminate it. */ aml8726_sdxc_soft_reset(sc); /* * Ensure the command has terminated before continuing on * to things such as bus_dmamap_sync / bus_dmamap_unload. */ while ((CSR_READ_4(sc, AML_SDXC_STATUS_REG) & AML_SDXC_STATUS_BUSY) != 0) cpu_spinwait(); aml8726_sdxc_finish_command(sc, MMC_ERR_TIMEOUT); } static void aml8726_sdxc_busy_check(void *arg) { struct aml8726_sdxc_softc *sc = (struct aml8726_sdxc_softc *)arg; uint32_t sr; sc->busy.time += AML_SDXC_BUSY_POLL_INTVL; sr = CSR_READ_4(sc, AML_SDXC_STATUS_REG); if ((sr & AML_SDXC_STATUS_DAT0) == 0) { if (sc->busy.time < AML_SDXC_BUSY_TIMEOUT) { callout_reset(&sc->ch, msecs_to_ticks(AML_SDXC_BUSY_POLL_INTVL), aml8726_sdxc_busy_check, sc); AML_SDXC_UNLOCK(sc); return; } if (sc->busy.error == MMC_ERR_NONE) sc->busy.error = MMC_ERR_TIMEOUT; } aml8726_sdxc_finish_command(sc, sc->busy.error); } static void aml8726_sdxc_intr(void *arg) { struct aml8726_sdxc_softc *sc = (struct aml8726_sdxc_softc *)arg; uint32_t isr; uint32_t pdmar; uint32_t sndr; uint32_t sr; int i; int mmc_error; int start; int stop; AML_SDXC_LOCK(sc); isr = CSR_READ_4(sc, AML_SDXC_IRQ_STATUS_REG); sndr = CSR_READ_4(sc, AML_SDXC_SEND_REG); sr = CSR_READ_4(sc, AML_SDXC_STATUS_REG); if (sc->cmd == NULL) goto spurious; mmc_error = MMC_ERR_NONE; if ((isr & (AML_SDXC_IRQ_STATUS_TX_FIFO_EMPTY | AML_SDXC_IRQ_STATUS_RX_FIFO_FULL)) != 0) mmc_error = MMC_ERR_FIFO; else if ((isr & (AML_SDXC_IRQ_ENABLE_A_PKG_CRC_ERR | AML_SDXC_IRQ_ENABLE_RESP_CRC_ERR)) != 0) mmc_error = MMC_ERR_BADCRC; else if ((isr & (AML_SDXC_IRQ_ENABLE_A_PKG_TIMEOUT_ERR | AML_SDXC_IRQ_ENABLE_RESP_TIMEOUT_ERR)) != 0) mmc_error = MMC_ERR_TIMEOUT; else if ((isr & (AML_SDXC_IRQ_STATUS_RESP_OK | AML_SDXC_IRQ_STATUS_DMA_DONE | AML_SDXC_IRQ_STATUS_TRANSFER_DONE_OK)) != 0) { ; } else { spurious: /* * Clear spurious interrupts while leaving intacted any * interrupts that may have occurred after we read the * interrupt status register. */ CSR_WRITE_4(sc, AML_SDXC_IRQ_STATUS_REG, (AML_SDXC_IRQ_STATUS_CLEAR & isr)); CSR_BARRIER(sc, AML_SDXC_IRQ_STATUS_REG); AML_SDXC_UNLOCK(sc); return; } aml8726_sdxc_disengage_dma(sc); if ((sndr & AML_SDXC_SEND_CMD_HAS_RESP) != 0) { start = 0; stop = 1; if ((sndr & AML_SDXC_SEND_RESP_136) != 0) { start = 1; stop = start + 4; } for (i = start; i < stop; i++) { pdmar = CSR_READ_4(sc, AML_SDXC_PDMA_REG); pdmar &= ~(AML_SDXC_PDMA_DMA_EN | AML_SDXC_PDMA_RESP_INDEX_MASK); pdmar |= i << AML_SDXC_PDMA_RESP_INDEX_SHIFT; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); sc->cmd->resp[(stop - 1) - i] = CSR_READ_4(sc, AML_SDXC_CMD_ARGUMENT_REG); } } if ((sr & AML_SDXC_STATUS_BUSY) != 0 && /* * A multiblock operation may keep the hardware * busy until stop transmission is executed. */ (isr & (AML_SDXC_IRQ_STATUS_DMA_DONE | AML_SDXC_IRQ_STATUS_TRANSFER_DONE_OK)) == 0) { if (mmc_error == MMC_ERR_NONE) mmc_error = MMC_ERR_FAILED; /* * Issue a soft reset to terminate the command. * * Ensure the command has terminated before continuing on * to things such as bus_dmamap_sync / bus_dmamap_unload. */ aml8726_sdxc_soft_reset(sc); while ((CSR_READ_4(sc, AML_SDXC_STATUS_REG) & AML_SDXC_STATUS_BUSY) != 0) cpu_spinwait(); } /* * The stop command can be generated either manually or * automatically by the hardware if MISC_MANUAL_STOP_MODE * has not been set. In either case check for busy. */ if (((sc->cmd->flags & MMC_RSP_BUSY) != 0 || (sndr & AML_SDXC_SEND_INDEX_MASK) == MMC_STOP_TRANSMISSION) && (sr & AML_SDXC_STATUS_DAT0) == 0) { sc->busy.error = mmc_error; callout_reset(&sc->ch, msecs_to_ticks(AML_SDXC_BUSY_POLL_INTVL), aml8726_sdxc_busy_check, sc); CSR_WRITE_4(sc, AML_SDXC_IRQ_STATUS_REG, (AML_SDXC_IRQ_STATUS_CLEAR & isr)); CSR_BARRIER(sc, AML_SDXC_IRQ_STATUS_REG); AML_SDXC_UNLOCK(sc); return; } aml8726_sdxc_finish_command(sc, mmc_error); } static int aml8726_sdxc_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "amlogic,aml8726-sdxc-m8")) return (ENXIO); device_set_desc(dev, "Amlogic aml8726-m8 SDXC"); return (BUS_PROBE_DEFAULT); } static int aml8726_sdxc_attach(device_t dev) { struct aml8726_sdxc_softc *sc = device_get_softc(dev); char *voltages; char *voltage; int error; int nvoltages; pcell_t prop[3]; phandle_t node; ssize_t len; device_t child; uint32_t ectlr; uint32_t miscr; uint32_t pdmar; sc->dev = dev; sc->auto_fill_flush = false; pdmar = AML_SDXC_PDMA_DMA_URGENT | (49 << AML_SDXC_PDMA_TX_THOLD_SHIFT) | (7 << AML_SDXC_PDMA_RX_THOLD_SHIFT) | (15 << AML_SDXC_PDMA_RD_BURST_SHIFT) | (7 << AML_SDXC_PDMA_WR_BURST_SHIFT); miscr = (2 << AML_SDXC_MISC_WCRC_OK_PAT_SHIFT) | (5 << AML_SDXC_MISC_WCRC_ERR_PAT_SHIFT); ectlr = (12 << AML_SDXC_ENH_CNTRL_SDIO_IRQ_PERIOD_SHIFT); /* * Certain bitfields are dependent on the hardware revision. */ switch (aml8726_soc_hw_rev) { case AML_SOC_HW_REV_M8: switch (aml8726_soc_metal_rev) { case AML_SOC_M8_METAL_REV_M2_A: sc->auto_fill_flush = true; miscr |= (6 << AML_SDXC_MISC_TXSTART_THOLD_SHIFT); ectlr |= (64 << AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_SHIFT) | AML_SDXC_ENH_CNTRL_WR_RESP_MODE_SKIP_M8M2; break; default: miscr |= (7 << AML_SDXC_MISC_TXSTART_THOLD_SHIFT); ectlr |= (63 << AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_SHIFT) | AML_SDXC_ENH_CNTRL_DMA_NO_WR_RESP_CHECK_M8 | (255 << AML_SDXC_ENH_CNTRL_RX_TIMEOUT_SHIFT_M8); break; } break; case AML_SOC_HW_REV_M8B: miscr |= (7 << AML_SDXC_MISC_TXSTART_THOLD_SHIFT); ectlr |= (63 << AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_SHIFT) | AML_SDXC_ENH_CNTRL_DMA_NO_WR_RESP_CHECK_M8 | (255 << AML_SDXC_ENH_CNTRL_RX_TIMEOUT_SHIFT_M8); break; default: device_printf(dev, "unsupported SoC\n"); return (ENXIO); /* NOTREACHED */ } node = ofw_bus_get_node(dev); len = OF_getencprop(node, "clock-frequency", prop, sizeof(prop)); if ((len / sizeof(prop[0])) != 1 || prop[0] == 0) { device_printf(dev, "missing clock-frequency attribute in FDT\n"); return (ENXIO); } sc->ref_freq = prop[0]; sc->pwr_en.dev = NULL; len = OF_getencprop(node, "mmc-pwr-en", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 3) { sc->pwr_en.dev = OF_device_from_xref(prop[0]); sc->pwr_en.pin = prop[1]; sc->pwr_en.pol = prop[2]; } if (sc->pwr_en.dev == NULL) { device_printf(dev, "unable to process mmc-pwr-en attribute in FDT\n"); return (ENXIO); } /* Turn off power and then configure the output driver. */ if (aml8726_sdxc_power_off(sc) != 0 || GPIO_PIN_SETFLAGS(sc->pwr_en.dev, sc->pwr_en.pin, GPIO_PIN_OUTPUT) != 0) { device_printf(dev, "could not use gpio to control power\n"); return (ENXIO); } } len = OF_getprop_alloc(node, "mmc-voltages", sizeof(char), (void **)&voltages); if (len < 0) { device_printf(dev, "missing mmc-voltages attribute in FDT\n"); return (ENXIO); } sc->voltages[0] = 0; sc->voltages[1] = 0; voltage = voltages; nvoltages = 0; while (len && nvoltages < 2) { if (strncmp("1.8", voltage, len) == 0) sc->voltages[nvoltages] = MMC_OCR_LOW_VOLTAGE; else if (strncmp("3.3", voltage, len) == 0) sc->voltages[nvoltages] = MMC_OCR_320_330 | MMC_OCR_330_340; else { device_printf(dev, "unknown voltage attribute %.*s in FDT\n", len, voltage); free(voltages, M_OFWPROP); return (ENXIO); } nvoltages++; /* queue up next string */ while (*voltage && len) { voltage++; len--; } if (len) { voltage++; len--; } } free(voltages, M_OFWPROP); sc->vselect.dev = NULL; len = OF_getencprop(node, "mmc-vselect", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 2) { sc->vselect.dev = OF_device_from_xref(prop[0]); sc->vselect.pin = prop[1]; sc->vselect.pol = 1; } if (sc->vselect.dev == NULL) { device_printf(dev, "unable to process mmc-vselect attribute in FDT\n"); return (ENXIO); } /* * With the power off select voltage 0 and then * configure the output driver. */ if (GPIO_PIN_SET(sc->vselect.dev, sc->vselect.pin, 0) != 0 || GPIO_PIN_SETFLAGS(sc->vselect.dev, sc->vselect.pin, GPIO_PIN_OUTPUT) != 0) { device_printf(dev, "could not use gpio to set voltage\n"); return (ENXIO); } } if (nvoltages == 0) { device_printf(dev, "no voltages in FDT\n"); return (ENXIO); } else if (nvoltages == 1 && sc->vselect.dev != NULL) { device_printf(dev, "only one voltage in FDT\n"); return (ENXIO); } else if (nvoltages == 2 && sc->vselect.dev == NULL) { device_printf(dev, "too many voltages in FDT\n"); return (ENXIO); } sc->card_rst.dev = NULL; len = OF_getencprop(node, "mmc-rst", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 3) { sc->card_rst.dev = OF_device_from_xref(prop[0]); sc->card_rst.pin = prop[1]; sc->card_rst.pol = prop[2]; } if (sc->card_rst.dev == NULL) { device_printf(dev, "unable to process mmc-rst attribute in FDT\n"); return (ENXIO); } } if (bus_alloc_resources(dev, aml8726_sdxc_spec, sc->res)) { device_printf(dev, "could not allocate resources for device\n"); return (ENXIO); } AML_SDXC_LOCK_INIT(sc); error = bus_dma_tag_create(bus_get_dma_tag(dev), AML_SDXC_ALIGN_DMA, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, AML_SDXC_MAX_DMA, 1, AML_SDXC_MAX_DMA, 0, NULL, NULL, &sc->dmatag); if (error) goto fail; error = bus_dmamap_create(sc->dmatag, 0, &sc->dmamap); if (error) goto fail; error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, NULL, aml8726_sdxc_intr, sc, &sc->ih_cookie); if (error) { device_printf(dev, "could not setup interrupt handler\n"); goto fail; } callout_init_mtx(&sc->ch, &sc->mtx, CALLOUT_RETURNUNLOCKED); sc->host.f_min = 200000; sc->host.f_max = 100000000; sc->host.host_ocr = sc->voltages[0] | sc->voltages[1]; sc->host.caps = MMC_CAP_8_BIT_DATA | MMC_CAP_4_BIT_DATA | MMC_CAP_HSPEED; aml8726_sdxc_soft_reset(sc); CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_WRITE_4(sc, AML_SDXC_MISC_REG, miscr); CSR_WRITE_4(sc, AML_SDXC_ENH_CNTRL_REG, ectlr); child = device_add_child(dev, "mmc", -1); if (!child) { device_printf(dev, "could not add mmc\n"); error = ENXIO; goto fail; } error = device_probe_and_attach(child); if (error) { device_printf(dev, "could not attach mmc\n"); goto fail; } return (0); fail: if (sc->ih_cookie) bus_teardown_intr(dev, sc->res[1], sc->ih_cookie); if (sc->dmamap) bus_dmamap_destroy(sc->dmatag, sc->dmamap); if (sc->dmatag) bus_dma_tag_destroy(sc->dmatag); AML_SDXC_LOCK_DESTROY(sc); (void)aml8726_sdxc_power_off(sc); bus_release_resources(dev, aml8726_sdxc_spec, sc->res); return (error); } static int aml8726_sdxc_detach(device_t dev) { struct aml8726_sdxc_softc *sc = device_get_softc(dev); AML_SDXC_LOCK(sc); if (sc->cmd != NULL) { AML_SDXC_UNLOCK(sc); return (EBUSY); } /* * Turn off the power, reset the hardware state machine, * and disable the interrupts. */ aml8726_sdxc_power_off(sc); aml8726_sdxc_soft_reset(sc); CSR_WRITE_4(sc, AML_SDXC_IRQ_ENABLE_REG, 0); AML_SDXC_UNLOCK(sc); bus_generic_detach(dev); bus_teardown_intr(dev, sc->res[1], sc->ih_cookie); bus_dmamap_destroy(sc->dmatag, sc->dmamap); bus_dma_tag_destroy(sc->dmatag); AML_SDXC_LOCK_DESTROY(sc); bus_release_resources(dev, aml8726_sdxc_spec, sc->res); return (0); } static int aml8726_sdxc_shutdown(device_t dev) { struct aml8726_sdxc_softc *sc = device_get_softc(dev); /* * Turn off the power, reset the hardware state machine, * and disable the interrupts. */ aml8726_sdxc_power_off(sc); aml8726_sdxc_soft_reset(sc); CSR_WRITE_4(sc, AML_SDXC_IRQ_ENABLE_REG, 0); return (0); } static int aml8726_sdxc_update_ios(device_t bus, device_t child) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); struct mmc_ios *ios = &sc->host.ios; unsigned int divisor; int error; int i; uint32_t cctlr; uint32_t clk2r; uint32_t ctlr; uint32_t freq; ctlr = (7 << AML_SDXC_CNTRL_TX_ENDIAN_SHIFT) | (7 << AML_SDXC_CNTRL_RX_ENDIAN_SHIFT) | (0xf << AML_SDXC_CNTRL_RX_PERIOD_SHIFT) | (0x7f << AML_SDXC_CNTRL_RX_TIMEOUT_SHIFT); switch (ios->bus_width) { case bus_width_8: ctlr |= AML_SDXC_CNTRL_BUS_WIDTH_8; break; case bus_width_4: ctlr |= AML_SDXC_CNTRL_BUS_WIDTH_4; break; case bus_width_1: ctlr |= AML_SDXC_CNTRL_BUS_WIDTH_1; break; default: return (EINVAL); } CSR_WRITE_4(sc, AML_SDXC_CNTRL_REG, ctlr); /* * Disable clocks and then clock module prior to setting desired values. */ cctlr = CSR_READ_4(sc, AML_SDXC_CLK_CNTRL_REG); cctlr &= ~(AML_SDXC_CLK_CNTRL_TX_CLK_EN | AML_SDXC_CLK_CNTRL_RX_CLK_EN | AML_SDXC_CLK_CNTRL_SD_CLK_EN); CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); cctlr &= ~AML_SDXC_CLK_CNTRL_CLK_MODULE_EN; CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); /* * aml8726-m8 * * Clock select 1 fclk_div2 (1.275 GHz) */ cctlr &= ~AML_SDXC_CLK_CNTRL_CLK_SEL_MASK; cctlr |= (1 << AML_SDXC_CLK_CNTRL_CLK_SEL_SHIFT); divisor = sc->ref_freq / ios->clock - 1; if (divisor == 0 || divisor == -1) divisor = 1; if ((sc->ref_freq / (divisor + 1)) > ios->clock) divisor += 1; if (divisor > (AML_SDXC_CLK_CNTRL_CLK_DIV_MASK >> AML_SDXC_CLK_CNTRL_CLK_DIV_SHIFT)) divisor = AML_SDXC_CLK_CNTRL_CLK_DIV_MASK >> AML_SDXC_CLK_CNTRL_CLK_DIV_SHIFT; cctlr &= ~AML_SDXC_CLK_CNTRL_CLK_DIV_MASK; cctlr |= divisor << AML_SDXC_CLK_CNTRL_CLK_DIV_SHIFT; cctlr &= ~AML_SDXC_CLK_CNTRL_MEM_PWR_MASK; cctlr |= AML_SDXC_CLK_CNTRL_MEM_PWR_ON; CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); /* * Enable clock module and then clocks after setting desired values. */ cctlr |= AML_SDXC_CLK_CNTRL_CLK_MODULE_EN; CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); cctlr |= AML_SDXC_CLK_CNTRL_TX_CLK_EN | AML_SDXC_CLK_CNTRL_RX_CLK_EN | AML_SDXC_CLK_CNTRL_SD_CLK_EN; CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); freq = sc->ref_freq / divisor; for (i = 0; aml8726_sdxc_clk_phases[i].voltage; i++) { if ((aml8726_sdxc_clk_phases[i].voltage & (1 << ios->vdd)) != 0 && freq > aml8726_sdxc_clk_phases[i].freq) break; if (aml8726_sdxc_clk_phases[i].freq == 0) break; } clk2r = (1 << AML_SDXC_CLK2_SD_PHASE_SHIFT) | (aml8726_sdxc_clk_phases[i].rx_phase << AML_SDXC_CLK2_RX_PHASE_SHIFT); CSR_WRITE_4(sc, AML_SDXC_CLK2_REG, clk2r); CSR_BARRIER(sc, AML_SDXC_CLK2_REG); error = 0; switch (ios->power_mode) { case power_up: /* * Configure and power on the regulator so that the * voltage stabilizes prior to powering on the card. */ if (sc->vselect.dev != NULL) { for (i = 0; i < 2; i++) if ((sc->voltages[i] & (1 << ios->vdd)) != 0) break; if (i >= 2) return (EINVAL); error = GPIO_PIN_SET(sc->vselect.dev, sc->vselect.pin, i); } break; case power_on: error = aml8726_sdxc_power_on(sc); if (error) break; if (sc->card_rst.dev != NULL) { if (GPIO_PIN_SET(sc->card_rst.dev, sc->card_rst.pin, PIN_ON_FLAG(sc->card_rst.pol)) != 0 || GPIO_PIN_SETFLAGS(sc->card_rst.dev, sc->card_rst.pin, GPIO_PIN_OUTPUT) != 0) error = ENXIO; DELAY(5); if (GPIO_PIN_SET(sc->card_rst.dev, sc->card_rst.pin, PIN_OFF_FLAG(sc->card_rst.pol)) != 0) error = ENXIO; DELAY(5); if (error) { device_printf(sc->dev, "could not use gpio to reset card\n"); break; } } break; case power_off: error = aml8726_sdxc_power_off(sc); break; default: return (EINVAL); } return (error); } static int aml8726_sdxc_request(device_t bus, device_t child, struct mmc_request *req) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); int mmc_error; AML_SDXC_LOCK(sc); if (sc->cmd != NULL) { AML_SDXC_UNLOCK(sc); return (EBUSY); } mmc_error = aml8726_sdxc_start_command(sc, req->cmd); AML_SDXC_UNLOCK(sc); /* Execute the callback after dropping the lock. */ if (mmc_error != MMC_ERR_NONE) { req->cmd->error = mmc_error; req->done(req); } return (0); } static int aml8726_sdxc_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: *(int *)result = sc->host.ios.bus_mode; break; case MMCBR_IVAR_BUS_WIDTH: *(int *)result = sc->host.ios.bus_width; break; case MMCBR_IVAR_CHIP_SELECT: *(int *)result = sc->host.ios.chip_select; break; case MMCBR_IVAR_CLOCK: *(int *)result = sc->host.ios.clock; break; case MMCBR_IVAR_F_MIN: *(int *)result = sc->host.f_min; break; case MMCBR_IVAR_F_MAX: *(int *)result = sc->host.f_max; break; case MMCBR_IVAR_HOST_OCR: *(int *)result = sc->host.host_ocr; break; case MMCBR_IVAR_MODE: *(int *)result = sc->host.mode; break; case MMCBR_IVAR_OCR: *(int *)result = sc->host.ocr; break; case MMCBR_IVAR_POWER_MODE: *(int *)result = sc->host.ios.power_mode; break; case MMCBR_IVAR_VDD: *(int *)result = sc->host.ios.vdd; break; case MMCBR_IVAR_CAPS: *(int *)result = sc->host.caps; break; case MMCBR_IVAR_MAX_DATA: *(int *)result = AML_SDXC_MAX_DMA / MMC_SECTOR_SIZE; break; default: return (EINVAL); } return (0); } static int aml8726_sdxc_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: sc->host.ios.bus_mode = value; break; case MMCBR_IVAR_BUS_WIDTH: sc->host.ios.bus_width = value; break; case MMCBR_IVAR_CHIP_SELECT: sc->host.ios.chip_select = value; break; case MMCBR_IVAR_CLOCK: sc->host.ios.clock = value; break; case MMCBR_IVAR_MODE: sc->host.mode = value; break; case MMCBR_IVAR_OCR: sc->host.ocr = value; break; case MMCBR_IVAR_POWER_MODE: sc->host.ios.power_mode = value; break; case MMCBR_IVAR_VDD: sc->host.ios.vdd = value; break; /* These are read-only */ case MMCBR_IVAR_CAPS: case MMCBR_IVAR_HOST_OCR: case MMCBR_IVAR_F_MIN: case MMCBR_IVAR_F_MAX: case MMCBR_IVAR_MAX_DATA: default: return (EINVAL); } return (0); } static int aml8726_sdxc_get_ro(device_t bus, device_t child) { return (0); } static int aml8726_sdxc_acquire_host(device_t bus, device_t child) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); AML_SDXC_LOCK(sc); while (sc->bus_busy) mtx_sleep(sc, &sc->mtx, PZERO, "sdxc", hz / 5); sc->bus_busy++; AML_SDXC_UNLOCK(sc); return (0); } static int aml8726_sdxc_release_host(device_t bus, device_t child) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); AML_SDXC_LOCK(sc); sc->bus_busy--; wakeup(sc); AML_SDXC_UNLOCK(sc); return (0); } static device_method_t aml8726_sdxc_methods[] = { /* Device interface */ DEVMETHOD(device_probe, aml8726_sdxc_probe), DEVMETHOD(device_attach, aml8726_sdxc_attach), DEVMETHOD(device_detach, aml8726_sdxc_detach), DEVMETHOD(device_shutdown, aml8726_sdxc_shutdown), /* Bus interface */ DEVMETHOD(bus_read_ivar, aml8726_sdxc_read_ivar), DEVMETHOD(bus_write_ivar, aml8726_sdxc_write_ivar), /* MMC bridge interface */ DEVMETHOD(mmcbr_update_ios, aml8726_sdxc_update_ios), DEVMETHOD(mmcbr_request, aml8726_sdxc_request), DEVMETHOD(mmcbr_get_ro, aml8726_sdxc_get_ro), DEVMETHOD(mmcbr_acquire_host, aml8726_sdxc_acquire_host), DEVMETHOD(mmcbr_release_host, aml8726_sdxc_release_host), DEVMETHOD_END }; static driver_t aml8726_sdxc_driver = { "aml8726_sdxc", aml8726_sdxc_methods, sizeof(struct aml8726_sdxc_softc), }; static devclass_t aml8726_sdxc_devclass; DRIVER_MODULE(aml8726_sdxc, simplebus, aml8726_sdxc_driver, aml8726_sdxc_devclass, 0, 0); MODULE_DEPEND(aml8726_sdxc, aml8726_gpio, 1, 1, 1); DRIVER_MODULE(mmc, aml8726_sdxc, mmc_driver, mmc_devclass, NULL, NULL); MODULE_DEPEND(aml8726_sdxc, mmc, 1, 1, 1); Index: head/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.h =================================================================== --- head/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.h (revision 299068) +++ head/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.h (revision 299069) @@ -1,223 +1,223 @@ /*- * Copyright 2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _ARM_AMLOGIC_AML8726_SDXC_M8_H #define _ARM_AMLOGIC_AML8726_SDXC_M8_H #define AML_SDXC_ALIGN_DMA 4 #define AML_SDXC_MAX_DMA 4096 /* * Timeouts are in milliseconds * * Read and write are per section 4.6.2 of the: * * SD Specifications Part 1 - * Physicaly Layer Simplified Specification + * Physical Layer Simplified Specification * Version 4.10 */ #define AML_SDXC_CMD_TIMEOUT 50 #define AML_SDXC_READ_TIMEOUT 100 #define AML_SDXC_WRITE_TIMEOUT 500 #define AML_SDXC_MAX_TIMEOUT 5000 #define AML_SDXC_BUSY_POLL_INTVL 1 #define AML_SDXC_BUSY_TIMEOUT 1000 /* * There's some disagreements between the S805 documentation * and the Amlogic Linux platform code regarding the exact * layout of various registers ... when in doubt we follow * the platform code. */ #define AML_SDXC_CMD_ARGUMENT_REG 0 #define AML_SDXC_SEND_REG 4 #define AML_SDXC_SEND_REP_PKG_CNT_MASK (0xffffU << 16) #define AML_SDXC_SEND_REP_PKG_CNT_SHIFT 16 #define AML_SDXC_SEND_DATA_STOP (1 << 11) #define AML_SDXC_SEND_DATA_WRITE (1 << 10) #define AML_SDXC_SEND_RESP_NO_CRC7_CHECK (1 << 9) #define AML_SDXC_SEND_RESP_136 (1 << 8) #define AML_SDXC_SEND_CMD_HAS_DATA (1 << 7) #define AML_SDXC_SEND_CMD_HAS_RESP (1 << 6) #define AML_SDXC_SEND_INDEX_MASK 0x3f #define AML_SDXC_SEND_INDEX_SHIFT 0 #define AML_SDXC_CNTRL_REG 8 #define AML_SDXC_CNTRL_TX_ENDIAN_MASK (7 << 29) #define AML_SDXC_CNTRL_TX_ENDIAN_SHIFT 29 #define AML_SDXC_CNTRL_RX_ENDIAN_MASK (7 << 24) #define AML_SDXC_CNTRL_RX_ENDIAN_SHIFT 24 #define AML_SDXC_CNTRL_RX_PERIOD_SHIFT 20 #define AML_SDXC_CNTRL_RX_TIMEOUT_SHIFT 13 #define AML_SDXC_CNTRL_PKG_LEN_MASK (0x1ff << 4) #define AML_SDXC_CNTRL_PKG_LEN_SHIFT 4 #define AML_SDXC_CNTRL_BUS_WIDTH_MASK (3 << 0) #define AML_SDXC_CNTRL_BUS_WIDTH_1 (0 << 0) #define AML_SDXC_CNTRL_BUS_WIDTH_4 (1 << 0) #define AML_SDXC_CNTRL_BUS_WIDTH_8 (2 << 0) #define AML_SDXC_STATUS_REG 12 #define AML_SDXC_STATUS_TX_CNT_MASK (0x7f << 13) #define AML_SDXC_STATUS_TX_CNT_SHIFT 13 #define AML_SDXC_STATUS_RX_CNT_MASK (0x7f << 6) #define AML_SDXC_STATUS_RX_CNT_SHIFT 6 #define AML_SDXC_STATUS_CMD (1 << 5) #define AML_SDXC_STATUS_DAT3 (1 << 4) #define AML_SDXC_STATUS_DAT2 (1 << 3) #define AML_SDXC_STATUS_DAT1 (1 << 2) #define AML_SDXC_STATUS_DAT0 (1 << 1) #define AML_SDXC_STATUS_BUSY (1 << 0) #define AML_SDXC_CLK_CNTRL_REG 16 #define AML_SDXC_CLK_CNTRL_MEM_PWR_MASK (3 << 25) #define AML_SDXC_CLK_CNTRL_MEM_PWR_OFF (3 << 25) #define AML_SDXC_CLK_CNTRL_MEM_PWR_ON (0 << 25) #define AML_SDXC_CLK_CNTRL_CLK_SEL_MASK (3 << 16) #define AML_SDXC_CLK_CNTRL_CLK_SEL_SHIFT 16 #define AML_SDXC_CLK_CNTRL_CLK_MODULE_EN (1 << 15) #define AML_SDXC_CLK_CNTRL_SD_CLK_EN (1 << 14) #define AML_SDXC_CLK_CNTRL_RX_CLK_EN (1 << 13) #define AML_SDXC_CLK_CNTRL_TX_CLK_EN (1 << 12) #define AML_SDXC_CLK_CNTRL_CLK_DIV_MASK 0x0fff #define AML_SDXC_CLK_CNTRL_CLK_DIV_SHIFT 0 #define AML_SDXC_DMA_ADDR_REG 20 #define AML_SDXC_PDMA_REG 24 #define AML_SDXC_PDMA_TX_FILL (1U << 31) #define AML_SDXC_PDMA_RX_FLUSH_NOW (1 << 30) #define AML_SDXC_PDMA_RX_FLUSH_MODE_SW (1 << 29) #define AML_SDXC_PDMA_TX_THOLD_MASK (0x3f << 22) #define AML_SDXC_PDMA_TX_THOLD_SHIFT 22 #define AML_SDXC_PDMA_RX_THOLD_MASK (0x3f << 15) #define AML_SDXC_PDMA_RX_THOLD_SHIFT 15 #define AML_SDXC_PDMA_RD_BURST_MASK (0x1f << 10) #define AML_SDXC_PDMA_RD_BURST_SHIFT 10 #define AML_SDXC_PDMA_WR_BURST_MASK (0x1f << 5) #define AML_SDXC_PDMA_WR_BURST_SHIFT 5 #define AML_SDXC_PDMA_DMA_URGENT (1 << 4) #define AML_SDXC_PDMA_RESP_INDEX_MASK (7 << 1) #define AML_SDXC_PDMA_RESP_INDEX_SHIFT 1 #define AML_SDXC_PDMA_DMA_EN (1 << 0) #define AML_SDXC_MISC_REG 28 #define AML_SDXC_MISC_TXSTART_THOLD_MASK (7U << 29) #define AML_SDXC_MISC_TXSTART_THOLD_SHIFT 29 #define AML_SDXC_MISC_MANUAL_STOP_MODE (1 << 28) #define AML_SDXC_MISC_WCRC_OK_PAT_MASK (7 << 7) #define AML_SDXC_MISC_WCRC_OK_PAT_SHIFT 7 #define AML_SDXC_MISC_WCRC_ERR_PAT_MASK (7 << 4) #define AML_SDXC_MISC_WCRC_ERR_PAT_SHIFT 4 #define AML_SDXC_DATA_REG 32 #define AML_SDXC_IRQ_ENABLE_REG 36 #define AML_SDXC_IRQ_ENABLE_TX_FIFO_EMPTY (1 << 13) #define AML_SDXC_IRQ_ENABLE_RX_FIFO_FULL (1 << 12) #define AML_SDXC_IRQ_ENABLE_DMA_DONE (1 << 11) #define AML_SDXC_IRQ_ENABLE_TRANSFER_DONE_OK (1 << 7) #define AML_SDXC_IRQ_ENABLE_A_PKG_CRC_ERR (1 << 6) #define AML_SDXC_IRQ_ENABLE_A_PKG_TIMEOUT_ERR (1 << 5) #define AML_SDXC_IRQ_ENABLE_A_PKG_DONE_OK (1 << 4) #define AML_SDXC_IRQ_ENABLE_RESP_CRC_ERR (1 << 2) #define AML_SDXC_IRQ_ENABLE_RESP_TIMEOUT_ERR (1 << 1) #define AML_SDXC_IRQ_ENABLE_RESP_OK (1 << 0) #define AML_SDXC_IRQ_ENABLE_STANDARD \ (AML_SDXC_IRQ_ENABLE_TX_FIFO_EMPTY | \ AML_SDXC_IRQ_ENABLE_RX_FIFO_FULL | \ AML_SDXC_IRQ_ENABLE_A_PKG_CRC_ERR | \ AML_SDXC_IRQ_ENABLE_A_PKG_TIMEOUT_ERR | \ AML_SDXC_IRQ_ENABLE_RESP_CRC_ERR | \ AML_SDXC_IRQ_ENABLE_RESP_TIMEOUT_ERR | \ AML_SDXC_IRQ_ENABLE_RESP_OK) #define AML_SDXC_IRQ_STATUS_REG 40 #define AML_SDXC_IRQ_STATUS_TX_FIFO_EMPTY (1 << 13) #define AML_SDXC_IRQ_STATUS_RX_FIFO_FULL (1 << 12) #define AML_SDXC_IRQ_STATUS_DMA_DONE (1 << 11) #define AML_SDXC_IRQ_STATUS_TRANSFER_DONE_OK (1 << 7) #define AML_SDXC_IRQ_STATUS_A_PKG_CRC_ERR (1 << 6) #define AML_SDXC_IRQ_STATUS_A_PKG_TIMEOUT_ERR (1 << 5) #define AML_SDXC_IRQ_STATUS_A_PKG_DONE_OK (1 << 4) #define AML_SDXC_IRQ_STATUS_RESP_CRC_ERR (1 << 2) #define AML_SDXC_IRQ_STATUS_RESP_TIMEOUT_ERR (1 << 1) #define AML_SDXC_IRQ_STATUS_RESP_OK (1 << 0) #define AML_SDXC_IRQ_STATUS_CLEAR \ (AML_SDXC_IRQ_STATUS_TX_FIFO_EMPTY | \ AML_SDXC_IRQ_STATUS_RX_FIFO_FULL | \ AML_SDXC_IRQ_STATUS_DMA_DONE | \ AML_SDXC_IRQ_STATUS_TRANSFER_DONE_OK | \ AML_SDXC_IRQ_STATUS_A_PKG_CRC_ERR | \ AML_SDXC_IRQ_STATUS_A_PKG_TIMEOUT_ERR | \ AML_SDXC_IRQ_STATUS_RESP_CRC_ERR | \ AML_SDXC_IRQ_STATUS_RESP_TIMEOUT_ERR | \ AML_SDXC_IRQ_STATUS_RESP_OK) #define AML_SDXC_SOFT_RESET_REG 44 #define AML_SDXC_SOFT_RESET_DMA (1 << 5) #define AML_SDXC_SOFT_RESET_TX_PHY (1 << 4) #define AML_SDXC_SOFT_RESET_RX_PHY (1 << 3) #define AML_SDXC_SOFT_RESET_TX_FIFO (1 << 2) #define AML_SDXC_SOFT_RESET_RX_FIFO (1 << 1) #define AML_SDXC_SOFT_RESET_MAIN (1 << 0) #define AML_SDXC_SOFT_RESET \ (AML_SDXC_SOFT_RESET_DMA | \ AML_SDXC_SOFT_RESET_TX_FIFO | \ AML_SDXC_SOFT_RESET_RX_FIFO | \ AML_SDXC_SOFT_RESET_MAIN) #define AML_SDXC_ENH_CNTRL_REG 52 #define AML_SDXC_ENH_CNTRL_TX_EMPTY_THOLD_MASK (0x7f << 25) #define AML_SDXC_ENH_CNTRL_TX_EMPTY_THOLD_SHIFT 25 #define AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_MASK (0x7f << 18) #define AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_SHIFT 18 #define AML_SDXC_ENH_CNTRL_SDIO_IRQ_PERIOD_MASK (0xff << 8) #define AML_SDXC_ENH_CNTRL_SDIO_IRQ_PERIOD_SHIFT 8 #define AML_SDXC_ENH_CNTRL_DMA_NO_WR_RESP_CHECK_M8 (1 << 17) #define AML_SDXC_ENH_CNTRL_DMA_NO_RD_RESP_CHECK_M8 (1 << 16) #define AML_SDXC_ENH_CNTRL_RX_TIMEOUT_MASK_M8 (0xff << 0) #define AML_SDXC_ENH_CNTRL_RX_TIMEOUT_SHIFT_M8 0 #define AML_SDXC_ENH_CNTRL_NO_DMA_CHECK_M8M2 (1 << 2) #define AML_SDXC_ENH_CNTRL_NO_WR_RESP_CHECK_M8M2 (1 << 1) #define AML_SDXC_ENH_CNTRL_WR_RESP_MODE_SKIP_M8M2 (1 << 0) #define AML_SDXC_CLK2_REG 56 #define AML_SDXC_CLK2_SD_PHASE_MASK (0x3ff << 12) #define AML_SDXC_CLK2_SD_PHASE_SHIFT 12 #define AML_SDXC_CLK2_RX_PHASE_MASK (0x3ff << 0) #define AML_SDXC_CLK2_RX_PHASE_SHIFT 0 #endif /* _ARM_AMLOGIC_AML8726_SDXC_M8_H */ Index: head/sys/arm/arm/cpufunc_asm_arm11.S =================================================================== --- head/sys/arm/arm/cpufunc_asm_arm11.S (revision 299068) +++ head/sys/arm/arm/cpufunc_asm_arm11.S (revision 299069) @@ -1,102 +1,102 @@ /* $NetBSD: cpufunc_asm_arm11.S,v 1.2 2005/12/11 12:16:41 christos Exp $ */ /* * Copyright (c) 2002, 2005 ARM Limited * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company may not be used to endorse or promote * products derived from this software without specific prior written * permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * ARM11 assembly functions for CPU / MMU / TLB specific operations * - * XXX We make no attempt at present to take advantage of the v6 memroy + * XXX We make no attempt at present to take advantage of the v6 memory * architecture or physically tagged cache. */ #include __FBSDID("$FreeBSD$"); /* * TLB functions */ ENTRY(arm11_tlb_flushID_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c8, c5, 1 /* flush I tlb single entry */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET END(arm11_tlb_flushID_SE) /* * Context switch. * * These is the CPU-specific parts of the context switcher cpu_switch() * These functions actually perform the TTB reload. * * NOTE: Special calling convention * r1, r4-r13 must be preserved */ ENTRY(arm11_context_switch) /* * We can assume that the caches will only contain kernel addresses * at this point. So no need to flush them again. */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ mcr p15, 0, r0, c2, c0, 0 /* set the new TTB */ mcr p15, 0, r0, c8, c7, 0 /* and flush the I+D tlbs */ /* Paranoia -- make sure the pipeline is empty. */ nop nop nop RET END(arm11_context_switch) /* * TLB functions */ ENTRY(arm11_tlb_flushID) mcr p15, 0, r0, c8, c7, 0 /* flush I+D tlb */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr END(arm11_tlb_flushID) ENTRY(arm11_tlb_flushD) mcr p15, 0, r0, c8, c6, 0 /* flush D tlb */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr END(arm11_tlb_flushD) ENTRY(arm11_tlb_flushD_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr END(arm11_tlb_flushD_SE) /* * Other functions */ ENTRY(arm11_drain_writebuf) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr END(arm11_drain_writebuf) Index: head/sys/arm/arm/exception.S =================================================================== --- head/sys/arm/arm/exception.S (revision 299068) +++ head/sys/arm/arm/exception.S (revision 299069) @@ -1,504 +1,504 @@ /* $NetBSD: exception.S,v 1.13 2003/10/31 16:30:15 scw Exp $ */ /*- * Copyright (c) 1994-1997 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * exception.S * * Low level handlers for exception vectors * * Created : 24/09/94 * * Based on kate/display/abort.s * */ #include "assym.s" #include #include #include #include #include __FBSDID("$FreeBSD$"); #ifdef KDTRACE_HOOKS .bss .align 4 .global _C_LABEL(dtrace_invop_jump_addr) _C_LABEL(dtrace_invop_jump_addr): .word 0 .word 0 #endif .text .align 2 /* * ASM macros for pushing and pulling trapframes from the stack * * These macros are used to handle the irqframe and trapframe structures * defined above. */ /* * PUSHFRAME - macro to push a trap frame on the stack in the current mode * Since the current mode is used, the SVC lr field is not defined. * * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ #if __ARM_ARCH < 6 #define PUSHFRAME \ sub sp, sp, #4; /* Align the stack */ \ str lr, [sp, #-4]!; /* Push the return address */ \ sub sp, sp, #(4*17); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]!; \ ldr r0, =ARM_RAS_START; \ mov r1, #0; \ str r1, [r0]; \ mov r1, #0xffffffff; \ str r1, [r0, #4]; #else #define PUSHFRAME \ sub sp, sp, #4; /* Align the stack */ \ str lr, [sp, #-4]!; /* Push the return address */ \ sub sp, sp, #(4*17); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]!; #endif /* * PULLFRAME - macro to pull a trap frame from the stack in the current mode * Since the current mode is used, the SVC lr field is ignored. */ #if __ARM_ARCH < 6 #define PULLFRAME \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*17); /* Adjust the stack pointer */ \ ldr lr, [sp], #4; /* Pull the return address */ \ add sp, sp, #4 /* Align the stack */ #else #define PULLFRAME \ ldr r0, [sp], #4 ; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; \ clrex; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*17); /* Adjust the stack pointer */ \ ldr lr, [sp], #4; /* Pull the return address */ \ add sp, sp, #4 /* Align the stack */ #endif /* * PUSHFRAMEINSVC - macro to push a trap frame on the stack in SVC32 mode * This should only be used if the processor is not currently in SVC32 * mode. The processor mode is switched to SVC mode and the trap frame is * stored. The SVC lr field is used to store the previous value of * lr in SVC mode. * * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ #if __ARM_ARCH < 6 #define PUSHFRAMEINSVC \ stmdb sp, {r0-r3}; /* Save 4 registers */ \ mov r0, lr; /* Save xxx32 r14 */ \ mov r1, sp; /* Save xxx32 sp */ \ mrs r3, spsr; /* Save xxx32 spsr */ \ mrs r2, cpsr; /* Get the CPSR */ \ bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ orr r2, r2, #(PSR_SVC32_MODE); \ msr cpsr_c, r2; /* Punch into SVC mode */ \ mov r2, sp; /* Save SVC sp */ \ bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ \ /* and for dtrace to emulate push/pop */ \ str r0, [sp, #-4]!; /* Push return address */ \ str lr, [sp, #-4]!; /* Push SVC lr */ \ str r2, [sp, #-4]!; /* Push SVC sp */ \ msr spsr_fsxc, r3; /* Restore correct spsr */ \ ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ ldr r5, =ARM_RAS_START; /* Check if there's any RAS */ \ ldr r4, [r5, #4]; /* reset it to point at the */ \ cmp r4, #0xffffffff; /* end of memory if necessary; */ \ movne r1, #0xffffffff; /* leave value in r4 for later */ \ - strne r1, [r5, #4]; /* comparision against PC. */ \ + strne r1, [r5, #4]; /* comparison against PC. */ \ ldr r3, [r5]; /* Retrieve global RAS_START */ \ cmp r3, #0; /* and reset it if non-zero. */ \ movne r1, #0; /* If non-zero RAS_START and */ \ strne r1, [r5]; /* PC was lower than RAS_END, */ \ ldrne r1, [r0, #16]; /* adjust the saved PC so that */ \ cmpne r4, r1; /* execution later resumes at */ \ strhi r3, [r0, #16]; /* the RAS_START location. */ \ mrs r0, spsr; \ str r0, [sp, #-4]! #else #define PUSHFRAMEINSVC \ stmdb sp, {r0-r3}; /* Save 4 registers */ \ mov r0, lr; /* Save xxx32 r14 */ \ mov r1, sp; /* Save xxx32 sp */ \ mrs r3, spsr; /* Save xxx32 spsr */ \ mrs r2, cpsr; /* Get the CPSR */ \ bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ orr r2, r2, #(PSR_SVC32_MODE); \ msr cpsr_c, r2; /* Punch into SVC mode */ \ mov r2, sp; /* Save SVC sp */ \ bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ \ /* and for dtrace to emulate push/pop */ \ str r0, [sp, #-4]!; /* Push return address */ \ str lr, [sp, #-4]!; /* Push SVC lr */ \ str r2, [sp, #-4]!; /* Push SVC sp */ \ msr spsr_fsxc, r3; /* Restore correct spsr */ \ ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]! #endif /* * PULLFRAMEFROMSVCANDEXIT - macro to pull a trap frame from the stack * in SVC32 mode and restore the saved processor mode and PC. * This should be used when the SVC lr register needs to be restored on * exit. */ #if __ARM_ARCH < 6 #define PULLFRAMEFROMSVCANDEXIT \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; /* restore SPSR */ \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*15); /* Adjust the stack pointer */ \ ldmia sp, {sp, lr, pc}^ /* Restore lr and exit */ #else #define PULLFRAMEFROMSVCANDEXIT \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; /* restore SPSR */ \ clrex; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*15); /* Adjust the stack pointer */ \ ldmia sp, {sp, lr, pc}^ /* Restore lr and exit */ #endif /* * Unwind hints so we can unwind past functions that use * PULLFRAMEFROMSVCANDEXIT. They are run in reverse order. * As the last thing we do is restore the stack pointer * we can ignore the padding at the end of struct trapframe. */ #define UNWINDSVCFRAME \ .save {r13-r15}; /* Restore sp, lr, pc */ \ .pad #(2*4); /* Skip user sp and lr */ \ .save {r0-r12}; /* Restore r0-r12 */ \ .pad #(4) /* Skip spsr */ #define DO_AST \ ldr r0, [sp]; /* Get the SPSR from stack */ \ mrs r4, cpsr; /* save CPSR */ \ orr r1, r4, #(PSR_I|PSR_F); \ msr cpsr_c, r1; /* Disable interrupts */ \ and r0, r0, #(PSR_MODE); /* Returning to USR mode? */ \ teq r0, #(PSR_USR32_MODE); \ bne 2f; /* Nope, get out now */ \ bic r4, r4, #(PSR_I|PSR_F); \ 1: GET_CURTHREAD_PTR(r5); \ ldr r1, [r5, #(TD_FLAGS)]; \ and r1, r1, #(TDF_ASTPENDING|TDF_NEEDRESCHED); \ teq r1, #0; \ beq 2f; /* Nope. Just bail */ \ msr cpsr_c, r4; /* Restore interrupts */ \ mov r0, sp; \ bl _C_LABEL(ast); /* ast(frame) */ \ orr r0, r4, #(PSR_I|PSR_F); \ msr cpsr_c, r0; \ b 1b; \ 2: /* * Entry point for a Software Interrupt (SWI). * * The hardware switches to svc32 mode on a swi, so we're already on the * right stack; just build a trapframe and call the handler. */ ASENTRY_NP(swi_entry) PUSHFRAME /* Build the trapframe on the */ mov r0, sp /* scv32 stack, pass it to the */ bl _C_LABEL(swi_handler) /* swi handler. */ /* * The fork_trampoline() code in swtch.S aranges for the MI fork_exit() * to return to swi_exit here, to return to userland. The net effect is * that a newly created thread appears to return from a SWI just like * the parent thread that created it. */ ASEENTRY_NP(swi_exit) DO_AST /* Handle pending signals. */ PULLFRAME /* Deallocate trapframe. */ movs pc, lr /* Return to userland. */ STOP_UNWINDING /* Don't unwind into user mode. */ EEND(swi_exit) END(swi_entry) /* * Standard exception exit handler. * * This is used to return from all exceptions except SWI. It uses DO_AST and * PULLFRAMEFROMSVCANDEXIT and can only be called if the exception entry code * used PUSHFRAMEINSVC. * * If the return is to user mode, this uses DO_AST to deliver any pending * signals and/or handle TDF_NEEDRESCHED first. */ ASENTRY_NP(exception_exit) DO_AST /* Handle pending signals. */ PULLFRAMEFROMSVCANDEXIT /* Return. */ UNWINDSVCFRAME /* Special unwinding for exceptions. */ END(exception_exit) /* * Entry point for a Prefetch Abort exception. * * The hardware switches to the abort mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the abort mode stack). */ ASENTRY_NP(prefetch_abort_entry) #ifdef __XSCALE__ nop /* Make absolutely sure any pending */ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ mov r1, #1 /* Type flag */ b _C_LABEL(abort_handler) END(prefetch_abort_entry) /* * Entry point for a Data Abort exception. * * The hardware switches to the abort mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the abort mode stack). */ ASENTRY_NP(data_abort_entry) #ifdef __XSCALE__ nop /* Make absolutely sure any pending */ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #8 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Exception exit routine */ mov r0, sp /* Trapframe to the handler */ mov r1, #0 /* Type flag */ b _C_LABEL(abort_handler) END(data_abort_entry) /* * Entry point for an Undefined Instruction exception. * * The hardware switches to the undefined mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the undefined mode stack). */ ASENTRY_NP(undefined_entry) PUSHFRAMEINSVC /* mode stack, build trapframe there. */ mov r4, r0 /* R0 contains SPSR */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. pass frame */ ldr r2, [sp, #(TF_PC)] /* load pc */ #if __ARM_ARCH >= 7 tst r4, #(PSR_T) /* test if PSR_T */ subne r2, r2, #(THUMB_INSN_SIZE) subeq r2, r2, #(INSN_SIZE) #else sub r2, r2, #(INSN_SIZE) /* fix pc */ #endif str r2, [sp, #TF_PC] /* store pc */ #ifdef KDTRACE_HOOKS /* Check if dtrace is enabled */ ldr r1, =_C_LABEL(dtrace_invop_jump_addr) ldr r3, [r1] cmp r3, #0 beq undefinedinstruction and r4, r4, #(PSR_MODE) /* Mask out unneeded bits */ cmp r4, #(PSR_USR32_MODE) /* Check if we came from usermode */ beq undefinedinstruction ldr r4, [r2] /* load instrution */ ldr r1, =FBT_BREAKPOINT /* load fbt inv op */ cmp r1, r4 bne undefinedinstruction bx r3 /* call invop_jump_addr */ #endif b undefinedinstruction /* call stadnard handler */ END(undefined_entry) /* * Entry point for a normal IRQ. * * The hardware switches to the IRQ mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the IRQ mode stack). */ ASENTRY_NP(irq_entry) sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ b _C_LABEL(intr_irq_handler)/* trapframe to the handler. */ END(irq_entry) /* * Entry point for an FIQ interrupt. * * We don't currently support FIQ handlers very much. Something can * install itself in the FIQ vector using code (that may or may not work * these days) in fiq.c. If nobody does that and an FIQ happens, this * default handler just disables FIQs and otherwise ignores it. */ ASENTRY_NP(fiq_entry) mrs r8, cpsr /* FIQ handling isn't supported, */ bic r8, #(PSR_F) /* just disable FIQ and return. */ msr cpsr_c, r8 /* The r8 we trash here is the */ subs pc, lr, #4 /* banked FIQ-mode r8. */ END(fiq_entry) /* * Entry point for an Address Exception exception. * This is an arm26 exception that should never happen. */ ASENTRY_NP(addr_exception_entry) mov r3, lr mrs r2, spsr mrs r1, cpsr adr r0, Laddr_exception_msg b _C_LABEL(panic) Laddr_exception_msg: .asciz "Address Exception CPSR=0x%08x SPSR=0x%08x LR=0x%08x\n" .balign 4 END(addr_exception_entry) /* * Entry point for the system Reset vector. * This should never happen, so panic. */ ASENTRY_NP(reset_entry) mov r1, lr adr r0, Lreset_panicmsg b _C_LABEL(panic) /* NOTREACHED */ Lreset_panicmsg: .asciz "Reset vector called, LR = 0x%08x" .balign 4 END(reset_entry) /* * page0 and page0_data -- An image of the ARM vectors which is copied to * the ARM vectors page (high or low) as part of CPU initialization. The * code that does the copy assumes that page0_data holds one 32-bit word * of data for each of the predefined ARM vectors. It also assumes that * page0_data follows the vectors in page0, but other stuff can appear * between the two. We currently leave room between the two for some fiq * handler code to be copied in. */ .global _C_LABEL(page0), _C_LABEL(page0_data) _C_LABEL(page0): ldr pc, .Lreset_entry ldr pc, .Lundefined_entry ldr pc, .Lswi_entry ldr pc, .Lprefetch_abort_entry ldr pc, .Ldata_abort_entry ldr pc, .Laddr_exception_entry ldr pc, .Lirq_entry .fiqv: ldr pc, .Lfiq_entry .space 256 /* room for some fiq handler code */ _C_LABEL(page0_data): .Lreset_entry: .word reset_entry .Lundefined_entry: .word undefined_entry .Lswi_entry: .word swi_entry .Lprefetch_abort_entry: .word prefetch_abort_entry .Ldata_abort_entry: .word data_abort_entry .Laddr_exception_entry: .word addr_exception_entry .Lirq_entry: .word irq_entry .Lfiq_entry: .word fiq_entry /* * These items are used by the code in fiq.c to install what it calls the * "null" handler. It's actually our default vector entry that just jumps * to the default handler which just disables FIQs and returns. */ .global _C_LABEL(fiq_nullhandler_code), _C_LABEL(fiq_nullhandler_size) _C_LABEL(fiq_nullhandler_code): .word .fiqv _C_LABEL(fiq_nullhandler_size): .word 4 Index: head/sys/arm/arm/gic.c =================================================================== --- head/sys/arm/arm/gic.c (revision 299068) +++ head/sys/arm/arm/gic.c (revision 299069) @@ -1,1570 +1,1570 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * Developed by Damjan Marion * * Based on OMAP4 GIC code by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INTRNG #include #endif #include #include #include #include #include #include #include #ifdef INTRNG #include "pic_if.h" #endif #define GIC_DEBUG_SPURIOUS /* We are using GICv2 register naming */ /* Distributor Registers */ #define GICD_CTLR 0x000 /* v1 ICDDCR */ #define GICD_TYPER 0x004 /* v1 ICDICTR */ #define GICD_IIDR 0x008 /* v1 ICDIIDR */ #define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ #define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ #define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ #define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ #define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ #define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ #define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ #define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ #define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ #define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ #define GICD_SGI_TARGET_SHIFT 16 /* CPU Registers */ #define GICC_CTLR 0x0000 /* v1 ICCICR */ #define GICC_PMR 0x0004 /* v1 ICCPMR */ #define GICC_BPR 0x0008 /* v1 ICCBPR */ #define GICC_IAR 0x000C /* v1 ICCIAR */ #define GICC_EOIR 0x0010 /* v1 ICCEOIR */ #define GICC_RPR 0x0014 /* v1 ICCRPR */ #define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ #define GICC_ABPR 0x001C /* v1 ICCABPR */ #define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ #define GIC_FIRST_SGI 0 /* Irqs 0-15 are SGIs/IPIs. */ #define GIC_LAST_SGI 15 #define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ #define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ #define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ /* First bit is a polarity bit (0 - low, 1 - high) */ #define GICD_ICFGR_POL_LOW (0 << 0) #define GICD_ICFGR_POL_HIGH (1 << 0) #define GICD_ICFGR_POL_MASK 0x1 /* Second bit is a trigger bit (0 - level, 1 - edge) */ #define GICD_ICFGR_TRIG_LVL (0 << 1) #define GICD_ICFGR_TRIG_EDGE (1 << 1) #define GICD_ICFGR_TRIG_MASK 0x2 #ifndef GIC_DEFAULT_ICFGR_INIT #define GIC_DEFAULT_ICFGR_INIT 0x00000000 #endif #ifdef INTRNG struct gic_irqsrc { struct intr_irqsrc gi_isrc; uint32_t gi_irq; enum intr_polarity gi_pol; enum intr_trigger gi_trig; #define GI_FLAG_EARLY_EOI (1 << 0) u_int gi_flags; }; static u_int gic_irq_cpu; static int arm_gic_intr(void *); static int arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc); #ifdef SMP static u_int sgi_to_ipi[GIC_LAST_SGI - GIC_FIRST_SGI + 1]; static u_int sgi_first_unused = GIC_FIRST_SGI; #endif #endif #ifdef INTRNG struct arm_gic_range { uint64_t bus; uint64_t host; uint64_t size; }; struct arm_gic_devinfo { struct ofw_bus_devinfo obdinfo; struct resource_list rl; }; #endif struct arm_gic_softc { device_t gic_dev; #ifdef INTRNG void * gic_intrhand; struct gic_irqsrc * gic_irqs; #endif struct resource * gic_res[3]; bus_space_tag_t gic_c_bst; bus_space_tag_t gic_d_bst; bus_space_handle_t gic_c_bsh; bus_space_handle_t gic_d_bsh; uint8_t ver; struct mtx mutex; uint32_t nirqs; #ifdef GIC_DEBUG_SPURIOUS uint32_t last_irq[MAXCPU]; #endif #ifdef INTRNG /* FDT child data */ pcell_t addr_cells; pcell_t size_cells; int nranges; struct arm_gic_range * ranges; #endif }; #ifdef INTRNG #define GIC_INTR_ISRC(sc, irq) (&sc->gic_irqs[irq].gi_isrc) #endif static struct resource_spec arm_gic_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ #ifdef INTRNG { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */ #endif { -1, 0 } }; static u_int arm_gic_map[MAXCPU]; static struct arm_gic_softc *gic_sc = NULL; #define gic_c_read_4(_sc, _reg) \ bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) #define gic_c_write_4(_sc, _reg, _val) \ bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) #define gic_d_read_4(_sc, _reg) \ bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) #define gic_d_write_1(_sc, _reg, _val) \ bus_space_write_1((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) #define gic_d_write_4(_sc, _reg, _val) \ bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) #ifndef INTRNG static int gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol); static void gic_post_filter(void *); #endif static struct ofw_compat_data compat_data[] = { {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ {"arm,gic-400", true}, {"arm,cortex-a15-gic", true}, {"arm,cortex-a9-gic", true}, {"arm,cortex-a7-gic", true}, {"arm,arm11mp-gic", true}, {"brcm,brahma-b15-gic", true}, {"qcom,msm-qgic2", true}, {NULL, false} }; static int arm_gic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller"); return (BUS_PROBE_DEFAULT); } #ifdef INTRNG static inline void gic_irq_unmask(struct arm_gic_softc *sc, u_int irq) { gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); } static inline void gic_irq_mask(struct arm_gic_softc *sc, u_int irq) { gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); } #endif static uint8_t gic_cpu_mask(struct arm_gic_softc *sc) { uint32_t mask; int i; /* Read the current cpuid mask by reading ITARGETSR{0..7} */ for (i = 0; i < 8; i++) { mask = gic_d_read_4(sc, GICD_ITARGETSR(i)); if (mask != 0) break; } /* No mask found, assume we are on CPU interface 0 */ if (mask == 0) return (1); /* Collect the mask in the lower byte */ mask |= mask >> 16; mask |= mask >> 8; return (mask); } #ifdef SMP #ifdef INTRNG static void arm_gic_init_secondary(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq, cpu; /* Set the mask so we can find this CPU to send it IPIs */ cpu = PCPU_GET(cpuid); arm_gic_map[cpu] = gic_cpu_mask(sc); for (irq = 0; irq < sc->nirqs; irq += 4) gic_d_write_4(sc, GICD_IPRIORITYR(irq >> 2), 0); /* Set all the interrupts to be in Group 0 (secure) */ for (irq = 0; irq < sc->nirqs; irq += 32) { gic_d_write_4(sc, GICD_IGROUPR(irq >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); /* Unmask attached SGI interrupts. */ for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu)) gic_irq_unmask(sc, irq); /* Unmask attached PPI interrupts. */ for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu)) gic_irq_unmask(sc, irq); } #else static void arm_gic_init_secondary(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); int i; /* Set the mask so we can find this CPU to send it IPIs */ arm_gic_map[PCPU_GET(cpuid)] = gic_cpu_mask(sc); for (i = 0; i < sc->nirqs; i += 4) gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); /* Set all the interrupts to be in Group 0 (secure) */ for (i = 0; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); /* * Activate the timer interrupts: virtual, secure, and non-secure. */ gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); } #endif /* INTRNG */ #endif /* SMP */ #ifndef INTRNG int gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, int *trig, int *pol) { static u_int num_intr_cells; static phandle_t self; struct ofw_compat_data *ocd; if (self == 0) { for (ocd = compat_data; ocd->ocd_str != NULL; ocd++) { if (fdt_is_compatible(iparent, ocd->ocd_str)) { self = iparent; break; } } } if (self != iparent) return (ENXIO); if (num_intr_cells == 0) { if (OF_searchencprop(OF_node_from_xref(iparent), "#interrupt-cells", &num_intr_cells, sizeof(num_intr_cells)) == -1) { num_intr_cells = 1; } } if (num_intr_cells == 1) { *interrupt = fdt32_to_cpu(intr[0]); *trig = INTR_TRIGGER_CONFORM; *pol = INTR_POLARITY_CONFORM; } else { if (fdt32_to_cpu(intr[0]) == 0) *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_SPI; else *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_PPI; /* * In intr[2], bits[3:0] are trigger type and level flags. * 1 = low-to-high edge triggered * 2 = high-to-low edge triggered * 4 = active high level-sensitive * 8 = active low level-sensitive * The hardware only supports active-high-level or rising-edge * for SPIs */ if (*interrupt >= GIC_FIRST_SPI && fdt32_to_cpu(intr[2]) & 0x0a) { printf("unsupported trigger/polarity configuration " "0x%02x\n", fdt32_to_cpu(intr[2]) & 0x0f); } *pol = INTR_POLARITY_CONFORM; if (fdt32_to_cpu(intr[2]) & 0x03) *trig = INTR_TRIGGER_EDGE; else *trig = INTR_TRIGGER_LEVEL; } return (0); } #endif #ifdef INTRNG static inline intptr_t gic_xref(device_t dev) { #ifdef FDT return (OF_xref_from_node(ofw_bus_get_node(dev))); #else return (0); #endif } static int arm_gic_register_isrcs(struct arm_gic_softc *sc, uint32_t num) { int error; uint32_t irq; struct gic_irqsrc *irqs; struct intr_irqsrc *isrc; const char *name; irqs = malloc(num * sizeof(struct gic_irqsrc), M_DEVBUF, M_WAITOK | M_ZERO); name = device_get_nameunit(sc->gic_dev); for (irq = 0; irq < num; irq++) { irqs[irq].gi_irq = irq; irqs[irq].gi_pol = INTR_POLARITY_CONFORM; irqs[irq].gi_trig = INTR_TRIGGER_CONFORM; isrc = &irqs[irq].gi_isrc; if (irq <= GIC_LAST_SGI) { error = intr_isrc_register(isrc, sc->gic_dev, INTR_ISRCF_IPI, "%s,i%u", name, irq - GIC_FIRST_SGI); } else if (irq <= GIC_LAST_PPI) { error = intr_isrc_register(isrc, sc->gic_dev, INTR_ISRCF_PPI, "%s,p%u", name, irq - GIC_FIRST_PPI); } else { error = intr_isrc_register(isrc, sc->gic_dev, 0, "%s,s%u", name, irq - GIC_FIRST_SPI); } if (error != 0) { /* XXX call intr_isrc_deregister() */ free(irqs, M_DEVBUF); return (error); } } sc->gic_irqs = irqs; sc->nirqs = num; return (0); } static int arm_gic_fill_ranges(phandle_t node, struct arm_gic_softc *sc) { pcell_t host_cells; cell_t *base_ranges; ssize_t nbase_ranges; int i, j, k; host_cells = 1; OF_getencprop(OF_parent(node), "#address-cells", &host_cells, sizeof(host_cells)); sc->addr_cells = 2; OF_getencprop(node, "#address-cells", &sc->addr_cells, sizeof(sc->addr_cells)); sc->size_cells = 2; OF_getencprop(node, "#size-cells", &sc->size_cells, sizeof(sc->size_cells)); nbase_ranges = OF_getproplen(node, "ranges"); if (nbase_ranges < 0) return (-1); sc->nranges = nbase_ranges / sizeof(cell_t) / (sc->addr_cells + host_cells + sc->size_cells); if (sc->nranges == 0) return (0); sc->ranges = malloc(sc->nranges * sizeof(sc->ranges[0]), M_DEVBUF, M_WAITOK); base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); OF_getencprop(node, "ranges", base_ranges, nbase_ranges); for (i = 0, j = 0; i < sc->nranges; i++) { sc->ranges[i].bus = 0; for (k = 0; k < sc->addr_cells; k++) { sc->ranges[i].bus <<= 32; sc->ranges[i].bus |= base_ranges[j++]; } sc->ranges[i].host = 0; for (k = 0; k < host_cells; k++) { sc->ranges[i].host <<= 32; sc->ranges[i].host |= base_ranges[j++]; } sc->ranges[i].size = 0; for (k = 0; k < sc->size_cells; k++) { sc->ranges[i].size <<= 32; sc->ranges[i].size |= base_ranges[j++]; } } free(base_ranges, M_DEVBUF); return (sc->nranges); } static bool arm_gic_add_children(device_t dev) { struct arm_gic_softc *sc; struct arm_gic_devinfo *dinfo; phandle_t child, node; device_t cdev; sc = device_get_softc(dev); node = ofw_bus_get_node(dev); /* If we have no children don't probe for them */ child = OF_child(node); if (child == 0) return (false); if (arm_gic_fill_ranges(node, sc) < 0) { device_printf(dev, "Have a child, but no ranges\n"); return (false); } for (; child != 0; child = OF_peer(child)) { dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&dinfo->obdinfo, child) != 0) { free(dinfo, M_DEVBUF); continue; } resource_list_init(&dinfo->rl); ofw_bus_reg_to_rl(dev, child, sc->addr_cells, sc->size_cells, &dinfo->rl); cdev = device_add_child(dev, NULL, -1); if (cdev == NULL) { device_printf(dev, "<%s>: device_add_child failed\n", dinfo->obdinfo.obd_name); resource_list_free(&dinfo->rl); ofw_bus_gen_destroy_devinfo(&dinfo->obdinfo); free(dinfo, M_DEVBUF); continue; } device_set_ivars(cdev, dinfo); } return (true); } #endif static int arm_gic_attach(device_t dev) { struct arm_gic_softc *sc; int i; uint32_t icciidr, mask, nirqs; #ifdef INTRNG phandle_t pxref; intptr_t xref = gic_xref(dev); #endif if (gic_sc) return (ENXIO); sc = device_get_softc(dev); if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } sc->gic_dev = dev; gic_sc = sc; /* Initialize mutex */ mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); /* Distributor Interface */ sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); /* CPU Interface */ sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); /* Disable interrupt forwarding to the CPU interface */ gic_d_write_4(sc, GICD_CTLR, 0x00); /* Get the number of interrupts */ nirqs = gic_d_read_4(sc, GICD_TYPER); nirqs = 32 * ((nirqs & 0x1f) + 1); #ifdef INTRNG if (arm_gic_register_isrcs(sc, nirqs)) { device_printf(dev, "could not register irqs\n"); goto cleanup; } #else sc->nirqs = nirqs; /* Set up function pointers */ arm_post_filter = gic_post_filter; arm_config_irq = gic_config_irq; #endif icciidr = gic_c_read_4(sc, GICC_IIDR); device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, (icciidr & 0xfff), sc->nirqs); /* Set all global interrupts to be level triggered, active low. */ for (i = 32; i < sc->nirqs; i += 16) { gic_d_write_4(sc, GICD_ICFGR(i >> 4), GIC_DEFAULT_ICFGR_INIT); } /* Disable all interrupts. */ for (i = 32; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); } /* Find the current cpu mask */ mask = gic_cpu_mask(sc); /* Set the mask so we can find this CPU to send it IPIs */ arm_gic_map[PCPU_GET(cpuid)] = mask; /* Set all four targets to this cpu */ mask |= mask << 8; mask |= mask << 16; for (i = 0; i < sc->nirqs; i += 4) { gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); if (i > 32) { gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), mask); } } /* Set all the interrupts to be in Group 0 (secure) */ for (i = 0; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); #ifndef INTRNG return (0); #else /* * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ if (intr_pic_register(dev, xref) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } /* * Controller is root if: * - doesn't have interrupt parent * - his interrupt parent is this controller */ pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); if (pxref == 0 || xref == pxref) { if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { device_printf(dev, "could not set PIC as a root\n"); intr_pic_deregister(dev, xref); goto cleanup; } } else { if (sc->gic_res[2] == NULL) { device_printf(dev, "not root PIC must have defined interrupt\n"); intr_pic_deregister(dev, xref); goto cleanup; } if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK, arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { device_printf(dev, "could not setup irq handler\n"); intr_pic_deregister(dev, xref); goto cleanup; } } OF_device_register_xref(xref, dev); /* If we have children probe and attach them */ if (arm_gic_add_children(dev)) { bus_generic_probe(dev); return (bus_generic_attach(dev)); } return (0); cleanup: /* * XXX - not implemented arm_gic_detach() should be called ! */ if (sc->gic_irqs != NULL) free(sc->gic_irqs, M_DEVBUF); bus_release_resources(dev, arm_gic_spec, sc->gic_res); return(ENXIO); #endif } #ifdef INTRNG static struct resource * arm_gic_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct arm_gic_softc *sc; struct arm_gic_devinfo *di; struct resource_list_entry *rle; int j; KASSERT(type == SYS_RES_MEMORY, ("Invalid resoure type %x", type)); sc = device_get_softc(bus); /* * Request for the default allocation with a given rid: use resource * list stored in the local device info. */ if (RMAN_IS_DEFAULT_RANGE(start, end)) { if ((di = device_get_ivars(child)) == NULL) return (NULL); if (type == SYS_RES_IOPORT) type = SYS_RES_MEMORY; rle = resource_list_find(&di->rl, type, *rid); if (rle == NULL) { if (bootverbose) device_printf(bus, "no default resources for " "rid = %d, type = %d\n", *rid, type); return (NULL); } start = rle->start; end = rle->end; count = rle->count; } /* Remap through ranges property */ for (j = 0; j < sc->nranges; j++) { if (start >= sc->ranges[j].bus && end < sc->ranges[j].bus + sc->ranges[j].size) { start -= sc->ranges[j].bus; start += sc->ranges[j].host; end -= sc->ranges[j].bus; end += sc->ranges[j].host; break; } } if (j == sc->nranges && sc->nranges != 0) { if (bootverbose) device_printf(bus, "Could not map resource " "%#jx-%#jx\n", (uintmax_t)start, (uintmax_t)end); return (NULL); } return (bus_generic_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static const struct ofw_bus_devinfo * arm_gic_ofw_get_devinfo(device_t bus __unused, device_t child) { struct arm_gic_devinfo *di; di = device_get_ivars(child); return (&di->obdinfo); } static int arm_gic_intr(void *arg) { struct arm_gic_softc *sc = arg; struct gic_irqsrc *gi; uint32_t irq_active_reg, irq; struct trapframe *tf; irq_active_reg = gic_c_read_4(sc, GICC_IAR); irq = irq_active_reg & 0x3FF; /* * 1. We do EOI here because recent read value from active interrupt * register must be used for it. Another approach is to save this * value into associated interrupt source. * 2. EOI must be done on same CPU where interrupt has fired. Thus * we must ensure that interrupted thread does not migrate to * another CPU. * 3. EOI cannot be delayed by any preemption which could happen on * critical_exit() used in MI intr code, when interrupt thread is * scheduled. See next point. * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during * an action and any use of critical_exit() could break this * assumption. See comments within smp_rendezvous_action(). * 5. We always return FILTER_HANDLED as this is an interrupt * controller dispatch function. Otherwise, in cascaded interrupt * case, the whole interrupt subtree would be masked. */ if (irq >= sc->nirqs) { #ifdef GIC_DEBUG_SPURIOUS device_printf(sc->gic_dev, "Spurious interrupt detected: last irq: %d on CPU%d\n", sc->last_irq[PCPU_GET(cpuid)], PCPU_GET(cpuid)); #endif return (FILTER_HANDLED); } tf = curthread->td_intr_frame; dispatch_irq: gi = sc->gic_irqs + irq; /* * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement * as compiler complains that comparing u_int >= 0 is always true. */ if (irq <= GIC_LAST_SGI) { #ifdef SMP /* Call EOI for all IPI before dispatch. */ gic_c_write_4(sc, GICC_EOIR, irq_active_reg); intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf); goto next_irq; #else device_printf(sc->gic_dev, "SGI %u on UP system detected\n", irq - GIC_FIRST_SGI); gic_c_write_4(sc, GICC_EOIR, irq_active_reg); goto next_irq; #endif } #ifdef GIC_DEBUG_SPURIOUS sc->last_irq[PCPU_GET(cpuid)] = irq; #endif if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI) gic_c_write_4(sc, GICC_EOIR, irq_active_reg); if (intr_isrc_dispatch(&gi->gi_isrc, tf) != 0) { gic_irq_mask(sc, irq); if ((gi->gi_flags & GI_FLAG_EARLY_EOI) != GI_FLAG_EARLY_EOI) gic_c_write_4(sc, GICC_EOIR, irq_active_reg); device_printf(sc->gic_dev, "Stray irq %u disabled\n", irq); } next_irq: arm_irq_memory_barrier(irq); irq_active_reg = gic_c_read_4(sc, GICC_IAR); irq = irq_active_reg & 0x3FF; if (irq < sc->nirqs) goto dispatch_irq; return (FILTER_HANDLED); } static void gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig, enum intr_polarity pol) { uint32_t reg; uint32_t mask; if (irq < GIC_FIRST_SPI) return; mtx_lock_spin(&sc->mutex); reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); mask = (reg >> 2*(irq % 16)) & 0x3; if (pol == INTR_POLARITY_LOW) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_LOW; } else if (pol == INTR_POLARITY_HIGH) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_HIGH; } if (trig == INTR_TRIGGER_LEVEL) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_LVL; } else if (trig == INTR_TRIGGER_EDGE) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_EDGE; } /* Set mask */ reg = reg & ~(0x3 << 2*(irq % 16)); reg = reg | (mask << 2*(irq % 16)); gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); mtx_unlock_spin(&sc->mutex); } static int gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus) { uint32_t cpu, end, mask; end = min(mp_ncpus, 8); for (cpu = end; cpu < MAXCPU; cpu++) if (CPU_ISSET(cpu, cpus)) return (EINVAL); for (mask = 0, cpu = 0; cpu < end; cpu++) if (CPU_ISSET(cpu, cpus)) mask |= 1 << cpu; gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask); return (0); } #ifdef FDT static int gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { if (ncells == 1) { *irqp = cells[0]; *polp = INTR_POLARITY_CONFORM; *trigp = INTR_TRIGGER_CONFORM; return (0); } if (ncells == 3) { u_int irq, tripol; /* * The 1st cell is the interrupt type: * 0 = SPI * 1 = PPI * The 2nd cell contains the interrupt number: * [0 - 987] for SPI * [0 - 15] for PPI * The 3rd cell is the flags, encoded as follows: * bits[3:0] trigger type and level flags * 1 = low-to-high edge triggered * 2 = high-to-low edge triggered * 4 = active high level-sensitive * 8 = active low level-sensitive * bits[15:8] PPI interrupt cpu mask * Each bit corresponds to each of the 8 possible cpus * attached to the GIC. A bit set to '1' indicated * the interrupt is wired to that CPU. */ switch (cells[0]) { case 0: irq = GIC_FIRST_SPI + cells[1]; /* SPI irq is checked later. */ break; case 1: irq = GIC_FIRST_PPI + cells[1]; if (irq > GIC_LAST_PPI) { device_printf(dev, "unsupported PPI interrupt " "number %u\n", cells[1]); return (EINVAL); } break; default: device_printf(dev, "unsupported interrupt type " "configuration %u\n", cells[0]); return (EINVAL); } tripol = cells[2] & 0xff; if (tripol & 0xf0 || (tripol & 0x0a && cells[0] == 0)) device_printf(dev, "unsupported trigger/polarity " "configuration 0x%02x\n", tripol); *irqp = irq; *polp = INTR_POLARITY_CONFORM; *trigp = tripol & 0x03 ? INTR_TRIGGER_EDGE : INTR_TRIGGER_LEVEL; return (0); } return (EINVAL); } #endif static int gic_map_intr(device_t dev, struct intr_map_data *data, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { u_int irq; enum intr_polarity pol; enum intr_trigger trig; struct arm_gic_softc *sc; sc = device_get_softc(dev); switch (data->type) { #ifdef FDT case INTR_MAP_DATA_FDT: if (gic_map_fdt(dev, data->fdt.ncells, data->fdt.cells, &irq, &pol, &trig) != 0) return (EINVAL); break; #endif default: return (EINVAL); } if (irq >= sc->nirqs) return (EINVAL); if (pol != INTR_POLARITY_CONFORM && pol != INTR_POLARITY_LOW && pol != INTR_POLARITY_HIGH) return (EINVAL); if (trig != INTR_TRIGGER_CONFORM && trig != INTR_TRIGGER_EDGE && trig != INTR_TRIGGER_LEVEL) return (EINVAL); *irqp = irq; if (polp != NULL) *polp = pol; if (trigp != NULL) *trigp = trig; return (0); } static int arm_gic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { int error; u_int irq; struct arm_gic_softc *sc; error = gic_map_intr(dev, data, &irq, NULL, NULL); if (error == 0) { sc = device_get_softc(dev); *isrcp = GIC_INTR_ISRC(sc, irq); } return (error); } static int arm_gic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; u_int irq; enum intr_trigger trig; enum intr_polarity pol; if (data == NULL) return (ENOTSUP); /* Get config for resource. */ if (gic_map_intr(dev, data, &irq, &pol, &trig)) return (EINVAL); if (gi->gi_irq != irq) return (EINVAL); /* Compare config if this is not first setup. */ if (isrc->isrc_handlers != 0) { if ((pol != INTR_POLARITY_CONFORM && pol != gi->gi_pol) || (trig != INTR_TRIGGER_CONFORM && trig != gi->gi_trig)) return (EINVAL); else return (0); } if (pol == INTR_POLARITY_CONFORM) pol = INTR_POLARITY_LOW; /* just pick some */ if (trig == INTR_TRIGGER_CONFORM) trig = INTR_TRIGGER_EDGE; /* just pick some */ gi->gi_pol = pol; gi->gi_trig = trig; /* Edge triggered interrupts need an early EOI sent */ if (gi->gi_pol == INTR_TRIGGER_EDGE) gi->gi_flags |= GI_FLAG_EARLY_EOI; /* * XXX - In case that per CPU interrupt is going to be enabled in time * when SMP is already started, we need some IPI call which * enables it on others CPUs. Further, it's more complicated as * pic_enable_source() and pic_disable_source() should act on * per CPU basis only. Thus, it should be solved here somehow. */ if (isrc->isrc_flags & INTR_ISRCF_PPI) CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); gic_config(sc, gi->gi_irq, trig, pol); arm_gic_bind_intr(dev, isrc); return (0); } static int arm_gic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; if (isrc->isrc_handlers == 0) { gi->gi_pol = INTR_POLARITY_CONFORM; gi->gi_trig = INTR_TRIGGER_CONFORM; } return (0); } static void arm_gic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; arm_irq_memory_barrier(gi->gi_irq); gic_irq_unmask(sc, gi->gi_irq); } static void arm_gic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; gic_irq_mask(sc, gi->gi_irq); } static void arm_gic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; arm_gic_disable_intr(dev, isrc); gic_c_write_4(sc, GICC_EOIR, gi->gi_irq); } static void arm_gic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { arm_irq_memory_barrier(0); arm_gic_enable_intr(dev, isrc); } static void arm_gic_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; /* EOI for edge-triggered done earlier. */ if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI) return; arm_irq_memory_barrier(0); gic_c_write_4(sc, GICC_EOIR, gi->gi_irq); } static int arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; if (gi->gi_irq < GIC_FIRST_SPI) return (EINVAL); if (CPU_EMPTY(&isrc->isrc_cpu)) { gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); } return (gic_bind(sc, gi->gi_irq, &isrc->isrc_cpu)); } #ifdef SMP static void arm_gic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, u_int ipi) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; uint32_t val = 0, i; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; gic_d_write_4(sc, GICD_SGIR(0), val | gi->gi_irq); } static int arm_gic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) { struct intr_irqsrc *isrc; struct arm_gic_softc *sc = device_get_softc(dev); if (sgi_first_unused > GIC_LAST_SGI) return (ENOSPC); isrc = GIC_INTR_ISRC(sc, sgi_first_unused); sgi_to_ipi[sgi_first_unused++] = ipi; CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); *isrcp = isrc; return (0); } #endif #else static int arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq) { uint32_t active_irq; active_irq = gic_c_read_4(sc, GICC_IAR); /* - * Immediatly EOIR the SGIs, because doing so requires the other + * Immediately EOIR the SGIs, because doing so requires the other * bits (ie CPU number), not just the IRQ number, and we do not * have this information later. */ if ((active_irq & 0x3ff) <= GIC_LAST_SGI) gic_c_write_4(sc, GICC_EOIR, active_irq); active_irq &= 0x3FF; if (active_irq == 0x3FF) { if (last_irq == -1) device_printf(sc->gic_dev, "Spurious interrupt detected\n"); return -1; } return active_irq; } static int arm_gic_config(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t reg; uint32_t mask; /* Function is public-accessible, so validate input arguments */ if ((irq < 0) || (irq >= sc->nirqs)) goto invalid_args; if ((trig != INTR_TRIGGER_EDGE) && (trig != INTR_TRIGGER_LEVEL) && (trig != INTR_TRIGGER_CONFORM)) goto invalid_args; if ((pol != INTR_POLARITY_HIGH) && (pol != INTR_POLARITY_LOW) && (pol != INTR_POLARITY_CONFORM)) goto invalid_args; mtx_lock_spin(&sc->mutex); reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); mask = (reg >> 2*(irq % 16)) & 0x3; if (pol == INTR_POLARITY_LOW) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_LOW; } else if (pol == INTR_POLARITY_HIGH) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_HIGH; } if (trig == INTR_TRIGGER_LEVEL) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_LVL; } else if (trig == INTR_TRIGGER_EDGE) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_EDGE; } /* Set mask */ reg = reg & ~(0x3 << 2*(irq % 16)); reg = reg | (mask << 2*(irq % 16)); gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); mtx_unlock_spin(&sc->mutex); return (0); invalid_args: device_printf(dev, "gic_config_irg, invalid parameters\n"); return (EINVAL); } static void arm_gic_mask(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); gic_c_write_4(sc, GICC_EOIR, irq); /* XXX - not allowed */ } static void arm_gic_unmask(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); if (irq > GIC_LAST_SGI) arm_irq_memory_barrier(irq); gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); } #ifdef SMP static void arm_gic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t val = 0, i; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; gic_d_write_4(sc, GICD_SGIR(0), val | ipi); } static int arm_gic_ipi_read(device_t dev, int i) { if (i != -1) { /* * The intr code will automagically give the frame pointer * if the interrupt argument is 0. */ if ((unsigned int)i > 16) return (0); return (i); } return (0x3ff); } static void arm_gic_ipi_clear(device_t dev, int ipi) { /* no-op */ } #endif static void gic_post_filter(void *arg) { struct arm_gic_softc *sc = gic_sc; uintptr_t irq = (uintptr_t) arg; if (irq > GIC_LAST_SGI) arm_irq_memory_barrier(irq); gic_c_write_4(sc, GICC_EOIR, irq); } static int gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol) { return (arm_gic_config(gic_sc->gic_dev, irq, trig, pol)); } void arm_mask_irq(uintptr_t nb) { arm_gic_mask(gic_sc->gic_dev, nb); } void arm_unmask_irq(uintptr_t nb) { arm_gic_unmask(gic_sc->gic_dev, nb); } int arm_get_next_irq(int last_irq) { return (arm_gic_next_irq(gic_sc, last_irq)); } #ifdef SMP void intr_pic_init_secondary(void) { arm_gic_init_secondary(gic_sc->gic_dev); } void pic_ipi_send(cpuset_t cpus, u_int ipi) { arm_gic_ipi_send(gic_sc->gic_dev, cpus, ipi); } int pic_ipi_read(int i) { return (arm_gic_ipi_read(gic_sc->gic_dev, i)); } void pic_ipi_clear(int ipi) { arm_gic_ipi_clear(gic_sc->gic_dev, ipi); } #endif #endif /* INTRNG */ static device_method_t arm_gic_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gic_probe), DEVMETHOD(device_attach, arm_gic_attach), #ifdef INTRNG /* Bus interface */ DEVMETHOD(bus_add_child, bus_generic_add_child), DEVMETHOD(bus_alloc_resource, arm_gic_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_activate_resource,bus_generic_activate_resource), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, arm_gic_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, arm_gic_disable_intr), DEVMETHOD(pic_enable_intr, arm_gic_enable_intr), DEVMETHOD(pic_map_intr, arm_gic_map_intr), DEVMETHOD(pic_setup_intr, arm_gic_setup_intr), DEVMETHOD(pic_teardown_intr, arm_gic_teardown_intr), DEVMETHOD(pic_post_filter, arm_gic_post_filter), DEVMETHOD(pic_post_ithread, arm_gic_post_ithread), DEVMETHOD(pic_pre_ithread, arm_gic_pre_ithread), #ifdef SMP DEVMETHOD(pic_bind_intr, arm_gic_bind_intr), DEVMETHOD(pic_init_secondary, arm_gic_init_secondary), DEVMETHOD(pic_ipi_send, arm_gic_ipi_send), DEVMETHOD(pic_ipi_setup, arm_gic_ipi_setup), #endif #endif { 0, 0 } }; static driver_t arm_gic_driver = { "gic", arm_gic_methods, sizeof(struct arm_gic_softc), }; static devclass_t arm_gic_devclass; EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_driver, arm_gic_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_driver, arm_gic_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); #ifdef INTRNG /* * GICv2m support -- the GICv2 MSI/MSI-X controller. */ #define GICV2M_MSI_TYPER 0x008 #define MSI_TYPER_SPI_BASE(x) (((x) >> 16) & 0x3ff) #define MSI_TYPER_SPI_COUNT(x) (((x) >> 0) & 0x3ff) #define GICv2M_MSI_SETSPI_NS 0x040 #define GICV2M_MSI_IIDR 0xFCC struct arm_gicv2m_softc { struct resource *sc_mem; struct mtx sc_mutex; u_int sc_spi_start; u_int sc_spi_count; u_int sc_spi_offset; }; static struct ofw_compat_data gicv2m_compat_data[] = { {"arm,gic-v2m-frame", true}, {NULL, false} }; static int arm_gicv2m_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, gicv2m_compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller MSI/MSIX"); return (BUS_PROBE_DEFAULT); } static int arm_gicv2m_attach(device_t dev) { struct arm_gicv2m_softc *sc; uint32_t typer; int rid; sc = device_get_softc(dev); rid = 0; sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem == NULL) { device_printf(dev, "Unable to allocate resources\n"); return (ENXIO); } typer = bus_read_4(sc->sc_mem, GICV2M_MSI_TYPER); sc->sc_spi_start = MSI_TYPER_SPI_BASE(typer); sc->sc_spi_count = MSI_TYPER_SPI_COUNT(typer); mtx_init(&sc->sc_mutex, "GICv2m lock", "", MTX_DEF); if (bootverbose) device_printf(dev, "using spi %u to %u\n", sc->sc_spi_start, sc->sc_spi_start + sc->sc_spi_count - 1); return (0); } static device_method_t arm_gicv2m_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gicv2m_probe), DEVMETHOD(device_attach, arm_gicv2m_attach), /* End */ DEVMETHOD_END }; DEFINE_CLASS_0(gicv2m, arm_gicv2m_driver, arm_gicv2m_methods, sizeof(struct arm_gicv2m_softc)); static devclass_t arm_gicv2m_devclass; EARLY_DRIVER_MODULE(gicv2m, gic, arm_gicv2m_driver, arm_gicv2m_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); #endif Index: head/sys/arm/arm/machdep.c =================================================================== --- head/sys/arm/arm/machdep.c (revision 299068) +++ head/sys/arm/arm/machdep.c (revision 299069) @@ -1,2015 +1,2015 @@ /* $NetBSD: arm32_machdep.c,v 1.44 2004/03/24 15:34:47 atatat Exp $ */ /*- * Copyright (c) 2004 Olivier Houchard * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe * for the NetBSD Project. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * Created : 17/09/94 * Updated : 18/04/01 updated for new wscons */ #include "opt_compat.h" #include "opt_ddb.h" #include "opt_kstack_pages.h" #include "opt_platform.h" #include "opt_sched.h" #include "opt_timer.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #ifdef DDB #include #if __ARM_ARCH >= 6 DB_SHOW_COMMAND(cp15, db_show_cp15) { u_int reg; reg = cp15_midr_get(); db_printf("Cpu ID: 0x%08x\n", reg); reg = cp15_ctr_get(); db_printf("Current Cache Lvl ID: 0x%08x\n",reg); reg = cp15_sctlr_get(); db_printf("Ctrl: 0x%08x\n",reg); reg = cp15_actlr_get(); db_printf("Aux Ctrl: 0x%08x\n",reg); reg = cp15_id_pfr0_get(); db_printf("Processor Feat 0: 0x%08x\n", reg); reg = cp15_id_pfr1_get(); db_printf("Processor Feat 1: 0x%08x\n", reg); reg = cp15_id_dfr0_get(); db_printf("Debug Feat 0: 0x%08x\n", reg); reg = cp15_id_afr0_get(); db_printf("Auxiliary Feat 0: 0x%08x\n", reg); reg = cp15_id_mmfr0_get(); db_printf("Memory Model Feat 0: 0x%08x\n", reg); reg = cp15_id_mmfr1_get(); db_printf("Memory Model Feat 1: 0x%08x\n", reg); reg = cp15_id_mmfr2_get(); db_printf("Memory Model Feat 2: 0x%08x\n", reg); reg = cp15_id_mmfr3_get(); db_printf("Memory Model Feat 3: 0x%08x\n", reg); reg = cp15_ttbr_get(); db_printf("TTB0: 0x%08x\n", reg); } DB_SHOW_COMMAND(vtop, db_show_vtop) { u_int reg; if (have_addr) { cp15_ats1cpr_set(addr); reg = cp15_par_get(); db_printf("Physical address reg: 0x%08x\n",reg); } else db_printf("show vtop \n"); } #endif /* __ARM_ARCH >= 6 */ #endif /* DDB */ #ifdef DEBUG #define debugf(fmt, args...) printf(fmt, ##args) #else #define debugf(fmt, args...) #endif #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7) || \ defined(COMPAT_FREEBSD9) #error FreeBSD/arm doesn't provide compatibility with releases prior to 10 #endif struct pcpu __pcpu[MAXCPU]; struct pcpu *pcpup = &__pcpu[0]; static struct trapframe proc0_tf; uint32_t cpu_reset_address = 0; int cold = 1; vm_offset_t vector_page; int (*_arm_memcpy)(void *, void *, int, int) = NULL; int (*_arm_bzero)(void *, int, int) = NULL; int _min_memcpy_size = 0; int _min_bzero_size = 0; extern int *end; #ifdef FDT static char *loader_envp; vm_paddr_t pmap_pa; #if __ARM_ARCH >= 6 vm_offset_t systempage; vm_offset_t irqstack; vm_offset_t undstack; vm_offset_t abtstack; #else /* * This is the number of L2 page tables required for covering max * (hypothetical) memsize of 4GB and all kernel mappings (vectors, msgbuf, * stacks etc.), uprounded to be divisible by 4. */ #define KERNEL_PT_MAX 78 static struct pv_addr kernel_pt_table[KERNEL_PT_MAX]; struct pv_addr systempage; static struct pv_addr msgbufpv; struct pv_addr irqstack; struct pv_addr undstack; struct pv_addr abtstack; static struct pv_addr kernelstack; #endif #endif #if defined(LINUX_BOOT_ABI) #define LBABI_MAX_BANKS 10 #define CMDLINE_GUARD "FreeBSD:" uint32_t board_id; struct arm_lbabi_tag *atag_list; char linux_command_line[LBABI_MAX_COMMAND_LINE + 1]; char atags[LBABI_MAX_COMMAND_LINE * 2]; uint32_t memstart[LBABI_MAX_BANKS]; uint32_t memsize[LBABI_MAX_BANKS]; uint32_t membanks; #endif #ifdef MULTIDELAY static delay_func *delay_impl; static void *delay_arg; #endif static uint32_t board_revision; /* hex representation of uint64_t */ static char board_serial[32]; SYSCTL_NODE(_hw, OID_AUTO, board, CTLFLAG_RD, 0, "Board attributes"); SYSCTL_UINT(_hw_board, OID_AUTO, revision, CTLFLAG_RD, &board_revision, 0, "Board revision"); SYSCTL_STRING(_hw_board, OID_AUTO, serial, CTLFLAG_RD, board_serial, 0, "Board serial"); int vfp_exists; SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, &vfp_exists, 0, "Floating point support enabled"); void board_set_serial(uint64_t serial) { snprintf(board_serial, sizeof(board_serial)-1, "%016jx", serial); } void board_set_revision(uint32_t revision) { board_revision = revision; } void sendsig(catcher, ksi, mask) sig_t catcher; ksiginfo_t *ksi; sigset_t *mask; { struct thread *td; struct proc *p; struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; struct sysentvec *sysent; int onstack; int sig; int code; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; code = ksi->ksi_code; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; onstack = sigonstack(tf->tf_usr_sp); CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, catcher, sig); /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !(onstack) && SIGISMEMBER(psp->ps_sigonstack, sig)) { fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); #if defined(COMPAT_43) td->td_sigstk.ss_flags |= SS_ONSTACK; #endif } else fp = (struct sigframe *)td->td_frame->tf_usr_sp; /* make room on the stack */ fp--; /* make the stack aligned */ fp = (struct sigframe *)STACKALIGN(fp); /* Populate the siginfo frame. */ get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); frame.sf_si = ksi->ksi_info; frame.sf_uc.uc_sigmask = *mask; frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK ) ? ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE; frame.sf_uc.uc_stack = td->td_sigstk; mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); PROC_LOCK(p); sigexit(td, SIGILL); } /* * Build context to run handler in. We invoke the handler * directly, only returning via the trampoline. Note the * trampoline version numbers are coordinated with machine- * dependent code in libc. */ tf->tf_r0 = sig; tf->tf_r1 = (register_t)&fp->sf_si; tf->tf_r2 = (register_t)&fp->sf_uc; /* the trampoline uses r5 as the uc address */ tf->tf_r5 = (register_t)&fp->sf_uc; tf->tf_pc = (register_t)catcher; tf->tf_usr_sp = (register_t)fp; sysent = p->p_sysent; if (sysent->sv_sigcode_base != 0) tf->tf_usr_lr = (register_t)sysent->sv_sigcode_base; else tf->tf_usr_lr = (register_t)(sysent->sv_psstrings - *(sysent->sv_szsigcode)); /* Set the mode to enter in the signal handler */ #if __ARM_ARCH >= 7 if ((register_t)catcher & 1) tf->tf_spsr |= PSR_T; else tf->tf_spsr &= ~PSR_T; #endif CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_usr_lr, tf->tf_usr_sp); PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } struct kva_md_info kmi; /* * arm32_vector_init: * * Initialize the vector page, and select whether or not to * relocate the vectors. * * NOTE: We expect the vector page to be mapped at its expected * destination. */ extern unsigned int page0[], page0_data[]; void arm_vector_init(vm_offset_t va, int which) { unsigned int *vectors = (int *) va; unsigned int *vectors_data = vectors + (page0_data - page0); int vec; /* * Loop through the vectors we're taking over, and copy the * vector's insn and data word. */ for (vec = 0; vec < ARM_NVEC; vec++) { if ((which & (1 << vec)) == 0) { /* Don't want to take over this vector. */ continue; } vectors[vec] = page0[vec]; vectors_data[vec] = page0_data[vec]; } /* Now sync the vectors. */ icache_sync(va, (ARM_NVEC * 2) * sizeof(u_int)); vector_page = va; if (va == ARM_VECTORS_HIGH) { /* * Assume the MD caller knows what it's doing here, and * really does want the vector page relocated. * * Note: This has to be done here (and not just in * cpu_setup()) because the vector page needs to be * accessible *before* cpu_startup() is called. * Think ddb(9) ... * * NOTE: If the CPU control register is not readable, * this will totally fail! We'll just assume that * any system that has high vector support has a * readable CPU control register, for now. If we * ever encounter one that does not, we'll have to * rethink this. */ cpu_control(CPU_CONTROL_VECRELOC, CPU_CONTROL_VECRELOC); } } static void cpu_startup(void *dummy) { struct pcb *pcb = thread0.td_pcb; const unsigned int mbyte = 1024 * 1024; #if __ARM_ARCH < 6 && !defined(ARM_CACHE_LOCK_ENABLE) vm_page_t m; #endif identify_arm_cpu(); vm_ksubmap_init(&kmi); /* * Display the RAM layout. */ printf("real memory = %ju (%ju MB)\n", (uintmax_t)arm32_ptob(realmem), (uintmax_t)arm32_ptob(realmem) / mbyte); printf("avail memory = %ju (%ju MB)\n", (uintmax_t)arm32_ptob(vm_cnt.v_free_count), (uintmax_t)arm32_ptob(vm_cnt.v_free_count) / mbyte); if (bootverbose) { arm_physmem_print_tables(); devmap_print_table(); } bufinit(); vm_pager_bufferinit(); pcb->pcb_regs.sf_sp = (u_int)thread0.td_kstack + USPACE_SVC_STACK_TOP; pmap_set_pcb_pagedir(kernel_pmap, pcb); #if __ARM_ARCH < 6 vector_page_setprot(VM_PROT_READ); pmap_postinit(); #ifdef ARM_CACHE_LOCK_ENABLE pmap_kenter_user(ARM_TP_ADDRESS, ARM_TP_ADDRESS); arm_lock_cache_line(ARM_TP_ADDRESS); #else m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_ZERO); pmap_kenter_user(ARM_TP_ADDRESS, VM_PAGE_TO_PHYS(m)); #endif *(uint32_t *)ARM_RAS_START = 0; *(uint32_t *)ARM_RAS_END = 0xffffffff; #endif } SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); /* * Flush the D-cache for non-DMA I/O so that the I-cache can * be made coherent later. */ void cpu_flush_dcache(void *ptr, size_t len) { dcache_wb_poc((vm_offset_t)ptr, (vm_paddr_t)vtophys(ptr), len); } /* Get current clock frequency for the given cpu id. */ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { return (ENXIO); } void cpu_idle(int busy) { CTR2(KTR_SPARE2, "cpu_idle(%d) at %d", busy, curcpu); spinlock_enter(); #ifndef NO_EVENTTIMERS if (!busy) cpu_idleclock(); #endif if (!sched_runnable()) cpu_sleep(0); #ifndef NO_EVENTTIMERS if (!busy) cpu_activeclock(); #endif spinlock_exit(); CTR2(KTR_SPARE2, "cpu_idle(%d) at %d done", busy, curcpu); } int cpu_idle_wakeup(int cpu) { return (0); } /* * Most ARM platforms don't need to do anything special to init their clocks * (they get intialized during normal device attachment), and by not defining a * cpu_initclocks() function they get this generic one. Any platform that needs * to do something special can just provide their own implementation, which will * override this one due to the weak linkage. */ void arm_generic_initclocks(void) { #ifndef NO_EVENTTIMERS #ifdef SMP if (PCPU_GET(cpuid) == 0) cpu_initclocks_bsp(); else cpu_initclocks_ap(); #else cpu_initclocks_bsp(); #endif #endif } __weak_reference(arm_generic_initclocks, cpu_initclocks); #ifdef MULTIDELAY void arm_set_delay(delay_func *impl, void *arg) { KASSERT(impl != NULL, ("No DELAY implementation")); delay_impl = impl; delay_arg = arg; } void DELAY(int usec) { delay_impl(usec, delay_arg); } #endif int fill_regs(struct thread *td, struct reg *regs) { struct trapframe *tf = td->td_frame; bcopy(&tf->tf_r0, regs->r, sizeof(regs->r)); regs->r_sp = tf->tf_usr_sp; regs->r_lr = tf->tf_usr_lr; regs->r_pc = tf->tf_pc; regs->r_cpsr = tf->tf_spsr; return (0); } int fill_fpregs(struct thread *td, struct fpreg *regs) { bzero(regs, sizeof(*regs)); return (0); } int set_regs(struct thread *td, struct reg *regs) { struct trapframe *tf = td->td_frame; bcopy(regs->r, &tf->tf_r0, sizeof(regs->r)); tf->tf_usr_sp = regs->r_sp; tf->tf_usr_lr = regs->r_lr; tf->tf_pc = regs->r_pc; tf->tf_spsr &= ~PSR_FLAGS; tf->tf_spsr |= regs->r_cpsr & PSR_FLAGS; return (0); } int set_fpregs(struct thread *td, struct fpreg *regs) { return (0); } int fill_dbregs(struct thread *td, struct dbreg *regs) { return (0); } int set_dbregs(struct thread *td, struct dbreg *regs) { return (0); } static int ptrace_read_int(struct thread *td, vm_offset_t addr, uint32_t *v) { if (proc_readmem(td, td->td_proc, addr, v, sizeof(*v)) != sizeof(*v)) return (ENOMEM); return (0); } static int ptrace_write_int(struct thread *td, vm_offset_t addr, uint32_t v) { if (proc_writemem(td, td->td_proc, addr, &v, sizeof(v)) != sizeof(v)) return (ENOMEM); return (0); } static u_int ptrace_get_usr_reg(void *cookie, int reg) { int ret; struct thread *td = cookie; KASSERT(((reg >= 0) && (reg <= ARM_REG_NUM_PC)), ("reg is outside range")); switch(reg) { case ARM_REG_NUM_PC: ret = td->td_frame->tf_pc; break; case ARM_REG_NUM_LR: ret = td->td_frame->tf_usr_lr; break; case ARM_REG_NUM_SP: ret = td->td_frame->tf_usr_sp; break; default: ret = *((register_t*)&td->td_frame->tf_r0 + reg); break; } return (ret); } static u_int ptrace_get_usr_int(void* cookie, vm_offset_t offset, u_int* val) { struct thread *td = cookie; u_int error; error = ptrace_read_int(td, offset, val); return (error); } /** * This function parses current instruction opcode and decodes * any possible jump (change in PC) which might occur after * the instruction is executed. * * @param td Thread structure of analysed task * @param cur_instr Currently executed instruction * @param alt_next_address Pointer to the variable where * the destination address of the * jump instruction shall be stored. * * @return <0> when jump is possible * otherwise */ static int ptrace_get_alternative_next(struct thread *td, uint32_t cur_instr, uint32_t *alt_next_address) { int error; if (inst_branch(cur_instr) || inst_call(cur_instr) || inst_return(cur_instr)) { error = arm_predict_branch(td, cur_instr, td->td_frame->tf_pc, alt_next_address, ptrace_get_usr_reg, ptrace_get_usr_int); return (error); } return (EINVAL); } int ptrace_single_step(struct thread *td) { struct proc *p; int error, error_alt; uint32_t cur_instr, alt_next = 0; /* TODO: This needs to be updated for Thumb-2 */ if ((td->td_frame->tf_spsr & PSR_T) != 0) return (EINVAL); KASSERT(td->td_md.md_ptrace_instr == 0, ("Didn't clear single step")); KASSERT(td->td_md.md_ptrace_instr_alt == 0, ("Didn't clear alternative single step")); p = td->td_proc; PROC_UNLOCK(p); error = ptrace_read_int(td, td->td_frame->tf_pc, &cur_instr); if (error) goto out; error = ptrace_read_int(td, td->td_frame->tf_pc + INSN_SIZE, &td->td_md.md_ptrace_instr); if (error == 0) { error = ptrace_write_int(td, td->td_frame->tf_pc + INSN_SIZE, PTRACE_BREAKPOINT); if (error) { td->td_md.md_ptrace_instr = 0; } else { td->td_md.md_ptrace_addr = td->td_frame->tf_pc + INSN_SIZE; } } error_alt = ptrace_get_alternative_next(td, cur_instr, &alt_next); if (error_alt == 0) { error_alt = ptrace_read_int(td, alt_next, &td->td_md.md_ptrace_instr_alt); if (error_alt) { td->td_md.md_ptrace_instr_alt = 0; } else { error_alt = ptrace_write_int(td, alt_next, PTRACE_BREAKPOINT); if (error_alt) td->td_md.md_ptrace_instr_alt = 0; else td->td_md.md_ptrace_addr_alt = alt_next; } } out: PROC_LOCK(p); return ((error != 0) && (error_alt != 0)); } int ptrace_clear_single_step(struct thread *td) { struct proc *p; /* TODO: This needs to be updated for Thumb-2 */ if ((td->td_frame->tf_spsr & PSR_T) != 0) return (EINVAL); if (td->td_md.md_ptrace_instr != 0) { p = td->td_proc; PROC_UNLOCK(p); ptrace_write_int(td, td->td_md.md_ptrace_addr, td->td_md.md_ptrace_instr); PROC_LOCK(p); td->td_md.md_ptrace_instr = 0; } if (td->td_md.md_ptrace_instr_alt != 0) { p = td->td_proc; PROC_UNLOCK(p); ptrace_write_int(td, td->td_md.md_ptrace_addr_alt, td->td_md.md_ptrace_instr_alt); PROC_LOCK(p); td->td_md.md_ptrace_instr_alt = 0; } return (0); } int ptrace_set_pc(struct thread *td, unsigned long addr) { td->td_frame->tf_pc = addr; return (0); } void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { } void spinlock_enter(void) { struct thread *td; register_t cspr; td = curthread; if (td->td_md.md_spinlock_count == 0) { cspr = disable_interrupts(PSR_I | PSR_F); td->td_md.md_spinlock_count = 1; td->td_md.md_saved_cspr = cspr; } else td->td_md.md_spinlock_count++; critical_enter(); } void spinlock_exit(void) { struct thread *td; register_t cspr; td = curthread; critical_exit(); cspr = td->td_md.md_saved_cspr; td->td_md.md_spinlock_count--; if (td->td_md.md_spinlock_count == 0) restore_interrupts(cspr); } /* * Clear registers on exec */ void exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) { struct trapframe *tf = td->td_frame; memset(tf, 0, sizeof(*tf)); tf->tf_usr_sp = stack; tf->tf_usr_lr = imgp->entry_addr; tf->tf_svc_lr = 0x77777777; tf->tf_pc = imgp->entry_addr; tf->tf_spsr = PSR_USR32_MODE; } /* * Get machine context. */ int get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) { struct trapframe *tf = td->td_frame; __greg_t *gr = mcp->__gregs; if (clear_ret & GET_MC_CLEAR_RET) { gr[_REG_R0] = 0; gr[_REG_CPSR] = tf->tf_spsr & ~PSR_C; } else { gr[_REG_R0] = tf->tf_r0; gr[_REG_CPSR] = tf->tf_spsr; } gr[_REG_R1] = tf->tf_r1; gr[_REG_R2] = tf->tf_r2; gr[_REG_R3] = tf->tf_r3; gr[_REG_R4] = tf->tf_r4; gr[_REG_R5] = tf->tf_r5; gr[_REG_R6] = tf->tf_r6; gr[_REG_R7] = tf->tf_r7; gr[_REG_R8] = tf->tf_r8; gr[_REG_R9] = tf->tf_r9; gr[_REG_R10] = tf->tf_r10; gr[_REG_R11] = tf->tf_r11; gr[_REG_R12] = tf->tf_r12; gr[_REG_SP] = tf->tf_usr_sp; gr[_REG_LR] = tf->tf_usr_lr; gr[_REG_PC] = tf->tf_pc; return (0); } /* * Set machine context. * * However, we don't set any but the user modifiable flags, and we won't * touch the cs selector. */ int set_mcontext(struct thread *td, mcontext_t *mcp) { struct trapframe *tf = td->td_frame; const __greg_t *gr = mcp->__gregs; tf->tf_r0 = gr[_REG_R0]; tf->tf_r1 = gr[_REG_R1]; tf->tf_r2 = gr[_REG_R2]; tf->tf_r3 = gr[_REG_R3]; tf->tf_r4 = gr[_REG_R4]; tf->tf_r5 = gr[_REG_R5]; tf->tf_r6 = gr[_REG_R6]; tf->tf_r7 = gr[_REG_R7]; tf->tf_r8 = gr[_REG_R8]; tf->tf_r9 = gr[_REG_R9]; tf->tf_r10 = gr[_REG_R10]; tf->tf_r11 = gr[_REG_R11]; tf->tf_r12 = gr[_REG_R12]; tf->tf_usr_sp = gr[_REG_SP]; tf->tf_usr_lr = gr[_REG_LR]; tf->tf_pc = gr[_REG_PC]; tf->tf_spsr = gr[_REG_CPSR]; return (0); } /* * MPSAFE */ int sys_sigreturn(td, uap) struct thread *td; struct sigreturn_args /* { const struct __ucontext *sigcntxp; } */ *uap; { ucontext_t uc; int spsr; if (uap == NULL) return (EFAULT); if (copyin(uap->sigcntxp, &uc, sizeof(uc))) return (EFAULT); /* * Make sure the processor mode has not been tampered with and * interrupts have not been disabled. */ spsr = uc.uc_mcontext.__gregs[_REG_CPSR]; if ((spsr & PSR_MODE) != PSR_USR32_MODE || (spsr & (PSR_I | PSR_F)) != 0) return (EINVAL); /* Restore register context. */ set_mcontext(td, &uc.uc_mcontext); /* Restore signal mask. */ kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); return (EJUSTRETURN); } /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter * the debugger. We have the context in the trapframe, but base the trace * on the PCB. The PCB doesn't have to be perfect, as long as it contains * enough for a backtrace. */ void makectx(struct trapframe *tf, struct pcb *pcb) { pcb->pcb_regs.sf_r4 = tf->tf_r4; pcb->pcb_regs.sf_r5 = tf->tf_r5; pcb->pcb_regs.sf_r6 = tf->tf_r6; pcb->pcb_regs.sf_r7 = tf->tf_r7; pcb->pcb_regs.sf_r8 = tf->tf_r8; pcb->pcb_regs.sf_r9 = tf->tf_r9; pcb->pcb_regs.sf_r10 = tf->tf_r10; pcb->pcb_regs.sf_r11 = tf->tf_r11; pcb->pcb_regs.sf_r12 = tf->tf_r12; pcb->pcb_regs.sf_pc = tf->tf_pc; pcb->pcb_regs.sf_lr = tf->tf_usr_lr; pcb->pcb_regs.sf_sp = tf->tf_usr_sp; } /* * Fake up a boot descriptor table */ vm_offset_t fake_preload_metadata(struct arm_boot_params *abp __unused, void *dtb_ptr, size_t dtb_size) { #ifdef DDB vm_offset_t zstart = 0, zend = 0; #endif vm_offset_t lastaddr; int i = 0; static uint32_t fake_preload[35]; fake_preload[i++] = MODINFO_NAME; fake_preload[i++] = strlen("kernel") + 1; strcpy((char*)&fake_preload[i++], "kernel"); i += 1; fake_preload[i++] = MODINFO_TYPE; fake_preload[i++] = strlen("elf kernel") + 1; strcpy((char*)&fake_preload[i++], "elf kernel"); i += 2; fake_preload[i++] = MODINFO_ADDR; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = KERNVIRTADDR; fake_preload[i++] = MODINFO_SIZE; fake_preload[i++] = sizeof(uint32_t); fake_preload[i++] = (uint32_t)&end - KERNVIRTADDR; #ifdef DDB if (*(uint32_t *)KERNVIRTADDR == MAGIC_TRAMP_NUMBER) { fake_preload[i++] = MODINFO_METADATA|MODINFOMD_SSYM; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = *(uint32_t *)(KERNVIRTADDR + 4); fake_preload[i++] = MODINFO_METADATA|MODINFOMD_ESYM; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = *(uint32_t *)(KERNVIRTADDR + 8); lastaddr = *(uint32_t *)(KERNVIRTADDR + 8); zend = lastaddr; zstart = *(uint32_t *)(KERNVIRTADDR + 4); db_fetch_ksymtab(zstart, zend); } else #endif lastaddr = (vm_offset_t)&end; if (dtb_ptr != NULL) { /* Copy DTB to KVA space and insert it into module chain. */ lastaddr = roundup(lastaddr, sizeof(int)); fake_preload[i++] = MODINFO_METADATA | MODINFOMD_DTBP; fake_preload[i++] = sizeof(uint32_t); fake_preload[i++] = (uint32_t)lastaddr; memmove((void *)lastaddr, dtb_ptr, dtb_size); lastaddr += dtb_size; lastaddr = roundup(lastaddr, sizeof(int)); } fake_preload[i++] = 0; fake_preload[i] = 0; preload_metadata = (void *)fake_preload; init_static_kenv(NULL, 0); return (lastaddr); } void pcpu0_init(void) { #if __ARM_ARCH >= 6 set_curthread(&thread0); #endif pcpu_init(pcpup, 0, sizeof(struct pcpu)); PCPU_SET(curthread, &thread0); } #if defined(LINUX_BOOT_ABI) /* Convert the U-Boot command line into FreeBSD kenv and boot options. */ static void cmdline_set_env(char *cmdline, const char *guard) { char *cmdline_next, *env; size_t size, guard_len; int i; size = strlen(cmdline); /* Skip leading spaces. */ for (; isspace(*cmdline) && (size > 0); cmdline++) size--; /* Test and remove guard. */ if (guard != NULL && guard[0] != '\0') { guard_len = strlen(guard); if (strncasecmp(cmdline, guard, guard_len) != 0) return; cmdline += guard_len; size -= guard_len; } /* Skip leading spaces. */ for (; isspace(*cmdline) && (size > 0); cmdline++) size--; /* Replace ',' with '\0'. */ /* TODO: implement escaping for ',' character. */ cmdline_next = cmdline; while(strsep(&cmdline_next, ",") != NULL) ; init_static_kenv(cmdline, 0); /* Parse boothowto. */ for (i = 0; howto_names[i].ev != NULL; i++) { env = kern_getenv(howto_names[i].ev); if (env != NULL) { if (strtoul(env, NULL, 10) != 0) boothowto |= howto_names[i].mask; freeenv(env); } } } vm_offset_t linux_parse_boot_param(struct arm_boot_params *abp) { struct arm_lbabi_tag *walker; uint32_t revision; uint64_t serial; int size; vm_offset_t lastaddr; #ifdef FDT struct fdt_header *dtb_ptr; uint32_t dtb_size; #endif /* * Linux boot ABI: r0 = 0, r1 is the board type (!= 0) and r2 * is atags or dtb pointer. If all of these aren't satisfied, * then punt. Unfortunately, it looks like DT enabled kernels * doesn't uses board type and U-Boot delivers 0 in r1 for them. */ if (abp->abp_r0 != 0 || abp->abp_r2 == 0) return (0); #ifdef FDT /* Test if r2 point to valid DTB. */ dtb_ptr = (struct fdt_header *)abp->abp_r2; if (fdt_check_header(dtb_ptr) == 0) { dtb_size = fdt_totalsize(dtb_ptr); return (fake_preload_metadata(abp, dtb_ptr, dtb_size)); } #endif board_id = abp->abp_r1; walker = (struct arm_lbabi_tag *)abp->abp_r2; if (ATAG_TAG(walker) != ATAG_CORE) return 0; atag_list = walker; while (ATAG_TAG(walker) != ATAG_NONE) { switch (ATAG_TAG(walker)) { case ATAG_CORE: break; case ATAG_MEM: arm_physmem_hardware_region(walker->u.tag_mem.start, walker->u.tag_mem.size); break; case ATAG_INITRD2: break; case ATAG_SERIAL: serial = walker->u.tag_sn.high; serial <<= 32; serial |= walker->u.tag_sn.low; board_set_serial(serial); break; case ATAG_REVISION: revision = walker->u.tag_rev.rev; board_set_revision(revision); break; case ATAG_CMDLINE: size = ATAG_SIZE(walker) - sizeof(struct arm_lbabi_header); size = min(size, LBABI_MAX_COMMAND_LINE); strncpy(linux_command_line, walker->u.tag_cmd.command, size); linux_command_line[size] = '\0'; break; default: break; } walker = ATAG_NEXT(walker); } /* Save a copy for later */ bcopy(atag_list, atags, (char *)walker - (char *)atag_list + ATAG_SIZE(walker)); lastaddr = fake_preload_metadata(abp, NULL, 0); cmdline_set_env(linux_command_line, CMDLINE_GUARD); return lastaddr; } #endif #if defined(FREEBSD_BOOT_LOADER) vm_offset_t freebsd_parse_boot_param(struct arm_boot_params *abp) { vm_offset_t lastaddr = 0; void *mdp; void *kmdp; #ifdef DDB vm_offset_t ksym_start; vm_offset_t ksym_end; #endif /* * Mask metadata pointer: it is supposed to be on page boundary. If * the first argument (mdp) doesn't point to a valid address the * bootloader must have passed us something else than the metadata * ptr, so we give up. Also give up if we cannot find metadta section * the loader creates that we get all this data out of. */ if ((mdp = (void *)(abp->abp_r0 & ~PAGE_MASK)) == NULL) return 0; preload_metadata = mdp; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) return 0; boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); loader_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *); init_static_kenv(loader_envp, 0); lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); #ifdef DDB ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t); ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t); db_fetch_ksymtab(ksym_start, ksym_end); #endif return lastaddr; } #endif vm_offset_t default_parse_boot_param(struct arm_boot_params *abp) { vm_offset_t lastaddr; #if defined(LINUX_BOOT_ABI) if ((lastaddr = linux_parse_boot_param(abp)) != 0) return lastaddr; #endif #if defined(FREEBSD_BOOT_LOADER) if ((lastaddr = freebsd_parse_boot_param(abp)) != 0) return lastaddr; #endif /* Fall back to hardcoded metadata. */ lastaddr = fake_preload_metadata(abp, NULL, 0); return lastaddr; } /* * Stub version of the boot parameter parsing routine. We are * called early in initarm, before even VM has been initialized. * This routine needs to preserve any data that the boot loader * has passed in before the kernel starts to grow past the end * of the BSS, traditionally the place boot-loaders put this data. * * Since this is called so early, things that depend on the vm system * being setup (including access to some SoC's serial ports), about * all that can be done in this routine is to copy the arguments. * * This is the default boot parameter parsing routine. Individual * kernels/boards can override this weak function with one of their * own. We just fake metadata... */ __weak_reference(default_parse_boot_param, parse_boot_param); /* * Initialize proc0 */ void init_proc0(vm_offset_t kstack) { proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_pcb = (struct pcb *) (thread0.td_kstack + kstack_pages * PAGE_SIZE) - 1; thread0.td_pcb->pcb_flags = 0; thread0.td_pcb->pcb_vfpcpu = -1; thread0.td_pcb->pcb_vfpstate.fpscr = VFPSCR_DN; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; } int arm_predict_branch(void *cookie, u_int insn, register_t pc, register_t *new_pc, u_int (*fetch_reg)(void*, int), u_int (*read_int)(void*, vm_offset_t, u_int*)) { u_int addr, nregs, offset = 0; int error = 0; switch ((insn >> 24) & 0xf) { case 0x2: /* add pc, reg1, #value */ case 0x0: /* add pc, reg1, reg2, lsl #offset */ addr = fetch_reg(cookie, (insn >> 16) & 0xf); if (((insn >> 16) & 0xf) == 15) addr += 8; if (insn & 0x0200000) { offset = (insn >> 7) & 0x1e; offset = (insn & 0xff) << (32 - offset) | (insn & 0xff) >> offset; } else { offset = fetch_reg(cookie, insn & 0x0f); if ((insn & 0x0000ff0) != 0x00000000) { if (insn & 0x10) nregs = fetch_reg(cookie, (insn >> 8) & 0xf); else nregs = (insn >> 7) & 0x1f; switch ((insn >> 5) & 3) { case 0: /* lsl */ offset = offset << nregs; break; case 1: /* lsr */ offset = offset >> nregs; break; default: break; /* XXX */ } } *new_pc = addr + offset; return (0); } case 0xa: /* b ... */ case 0xb: /* bl ... */ addr = ((insn << 2) & 0x03ffffff); if (addr & 0x02000000) addr |= 0xfc000000; *new_pc = (pc + 8 + addr); return (0); case 0x7: /* ldr pc, [pc, reg, lsl #2] */ addr = fetch_reg(cookie, insn & 0xf); addr = pc + 8 + (addr << 2); error = read_int(cookie, addr, &addr); *new_pc = addr; return (error); case 0x1: /* mov pc, reg */ *new_pc = fetch_reg(cookie, insn & 0xf); return (0); case 0x4: case 0x5: /* ldr pc, [reg] */ addr = fetch_reg(cookie, (insn >> 16) & 0xf); /* ldr pc, [reg, #offset] */ if (insn & (1 << 24)) offset = insn & 0xfff; if (insn & 0x00800000) addr += offset; else addr -= offset; error = read_int(cookie, addr, &addr); *new_pc = addr; return (error); case 0x8: /* ldmxx reg, {..., pc} */ case 0x9: addr = fetch_reg(cookie, (insn >> 16) & 0xf); nregs = (insn & 0x5555) + ((insn >> 1) & 0x5555); nregs = (nregs & 0x3333) + ((nregs >> 2) & 0x3333); nregs = (nregs + (nregs >> 4)) & 0x0f0f; nregs = (nregs + (nregs >> 8)) & 0x001f; switch ((insn >> 23) & 0x3) { case 0x0: /* ldmda */ addr = addr - 0; break; case 0x1: /* ldmia */ addr = addr + 0 + ((nregs - 1) << 2); break; case 0x2: /* ldmdb */ addr = addr - 4; break; case 0x3: /* ldmib */ addr = addr + 4 + ((nregs - 1) << 2); break; } error = read_int(cookie, addr, &addr); *new_pc = addr; return (error); default: return (EINVAL); } } #if __ARM_ARCH >= 6 void set_stackptrs(int cpu) { set_stackptr(PSR_IRQ32_MODE, irqstack + ((IRQ_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); set_stackptr(PSR_ABT32_MODE, abtstack + ((ABT_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); set_stackptr(PSR_UND32_MODE, undstack + ((UND_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); } #else void set_stackptrs(int cpu) { set_stackptr(PSR_IRQ32_MODE, irqstack.pv_va + ((IRQ_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); set_stackptr(PSR_ABT32_MODE, abtstack.pv_va + ((ABT_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); set_stackptr(PSR_UND32_MODE, undstack.pv_va + ((UND_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); } #endif #ifdef EFI #define efi_next_descriptor(ptr, size) \ ((struct efi_md *)(((uint8_t *) ptr) + size)) static void add_efi_map_entries(struct efi_map_header *efihdr, struct mem_region *mr, int *mrcnt) { struct efi_md *map, *p; const char *type; size_t efisz, memory_size; int ndesc, i, j; static const char *types[] = { "Reserved", "LoaderCode", "LoaderData", "BootServicesCode", "BootServicesData", "RuntimeServicesCode", "RuntimeServicesData", "ConventionalMemory", "UnusableMemory", "ACPIReclaimMemory", "ACPIMemoryNVS", "MemoryMappedIO", "MemoryMappedIOPortSpace", "PalCode" }; *mrcnt = 0; /* * Memory map data provided by UEFI via the GetMemoryMap * Boot Services API. */ efisz = roundup2(sizeof(struct efi_map_header), 0x10); map = (struct efi_md *)((uint8_t *)efihdr + efisz); if (efihdr->descriptor_size == 0) return; ndesc = efihdr->memory_size / efihdr->descriptor_size; if (boothowto & RB_VERBOSE) printf("%23s %12s %12s %8s %4s\n", "Type", "Physical", "Virtual", "#Pages", "Attr"); memory_size = 0; for (i = 0, j = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p, efihdr->descriptor_size)) { if (boothowto & RB_VERBOSE) { if (p->md_type <= EFI_MD_TYPE_PALCODE) type = types[p->md_type]; else type = ""; printf("%23s %012llx %12p %08llx ", type, p->md_phys, p->md_virt, p->md_pages); if (p->md_attr & EFI_MD_ATTR_UC) printf("UC "); if (p->md_attr & EFI_MD_ATTR_WC) printf("WC "); if (p->md_attr & EFI_MD_ATTR_WT) printf("WT "); if (p->md_attr & EFI_MD_ATTR_WB) printf("WB "); if (p->md_attr & EFI_MD_ATTR_UCE) printf("UCE "); if (p->md_attr & EFI_MD_ATTR_WP) printf("WP "); if (p->md_attr & EFI_MD_ATTR_RP) printf("RP "); if (p->md_attr & EFI_MD_ATTR_XP) printf("XP "); if (p->md_attr & EFI_MD_ATTR_RT) printf("RUNTIME"); printf("\n"); } switch (p->md_type) { case EFI_MD_TYPE_CODE: case EFI_MD_TYPE_DATA: case EFI_MD_TYPE_BS_CODE: case EFI_MD_TYPE_BS_DATA: case EFI_MD_TYPE_FREE: /* * We're allowed to use any entry with these types. */ break; default: continue; } j++; if (j >= FDT_MEM_REGIONS) break; mr[j].mr_start = p->md_phys; mr[j].mr_size = p->md_pages * PAGE_SIZE; memory_size += mr[j].mr_size; } *mrcnt = j; } #endif /* EFI */ #ifdef FDT static char * kenv_next(char *cp) { if (cp != NULL) { while (*cp != 0) cp++; cp++; if (*cp == 0) cp = NULL; } return (cp); } static void print_kenv(void) { char *cp; debugf("loader passed (static) kenv:\n"); if (loader_envp == NULL) { debugf(" no env, null ptr\n"); return; } debugf(" loader_envp = 0x%08x\n", (uint32_t)loader_envp); for (cp = loader_envp; cp != NULL; cp = kenv_next(cp)) debugf(" %x %s\n", (uint32_t)cp, cp); } #if __ARM_ARCH < 6 void * initarm(struct arm_boot_params *abp) { struct mem_region mem_regions[FDT_MEM_REGIONS]; struct pv_addr kernel_l1pt; struct pv_addr dpcpu; vm_offset_t dtbp, freemempos, l2_start, lastaddr; uint64_t memsize; uint32_t l2size; char *env; void *kmdp; u_int l1pagetable; int i, j, err_devmap, mem_regions_sz; lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; memsize = 0; cpuinfo_init(); set_cpufuncs(); /* * Find the dtb passed in by the boot loader. */ kmdp = preload_search_by_type("elf kernel"); if (kmdp != NULL) dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); else dtbp = (vm_offset_t)NULL; #if defined(FDT_DTB_STATIC) /* * In case the device tree blob was not retrieved (from metadata) try * to use the statically embedded one. */ if (dtbp == (vm_offset_t)NULL) dtbp = (vm_offset_t)&fdt_static_dtb; #endif if (OF_install(OFW_FDT, 0) == FALSE) panic("Cannot install FDT"); if (OF_init((void *)dtbp) != 0) panic("OF_init failed with the found device tree"); /* Grab physical memory regions information from device tree. */ if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, &memsize) != 0) panic("Cannot get physical memory regions"); arm_physmem_hardware_regions(mem_regions, mem_regions_sz); /* Grab reserved memory regions information from device tree. */ if (fdt_get_reserved_regions(mem_regions, &mem_regions_sz) == 0) arm_physmem_exclude_regions(mem_regions, mem_regions_sz, EXFLAG_NODUMP | EXFLAG_NOALLOC); /* Platform-specific initialisation */ platform_probe_and_attach(); pcpu0_init(); /* Do basic tuning, hz etc */ init_param1(); /* Calculate number of L2 tables needed for mapping vm_page_array */ l2size = (memsize / PAGE_SIZE) * sizeof(struct vm_page); l2size = (l2size >> L1_S_SHIFT) + 1; /* * Add one table for end of kernel map, one for stacks, msgbuf and * L1 and L2 tables map and one for vectors map. */ l2size += 3; /* Make it divisible by 4 */ l2size = (l2size + 3) & ~3; freemempos = (lastaddr + PAGE_MASK) & ~PAGE_MASK; /* Define a macro to simplify memory allocation */ #define valloc_pages(var, np) \ alloc_pages((var).pv_va, (np)); \ (var).pv_pa = (var).pv_va + (abp->abp_physaddr - KERNVIRTADDR); #define alloc_pages(var, np) \ (var) = freemempos; \ freemempos += (np * PAGE_SIZE); \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos += PAGE_SIZE; valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); for (i = 0, j = 0; i < l2size; ++i) { if (!(i % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[i], L2_TABLE_SIZE / PAGE_SIZE); j = i; } else { kernel_pt_table[i].pv_va = kernel_pt_table[j].pv_va + L2_TABLE_SIZE_REAL * (i - j); kernel_pt_table[i].pv_pa = kernel_pt_table[i].pv_va - KERNVIRTADDR + abp->abp_physaddr; } } /* * Allocate a page for the system page mapped to 0x00000000 * or 0xffff0000. This page will just contain the system vectors * and can be shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE * MAXCPU); valloc_pages(abtstack, ABT_STACK_SIZE * MAXCPU); valloc_pages(undstack, UND_STACK_SIZE * MAXCPU); valloc_pages(kernelstack, kstack_pages * MAXCPU); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Now we start construction of the L1 page table * We start by mapping the L2 page tables into the L1. * This means that we can replace L1 mappings later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* * Try to map as much as possible of kernel text and data using * 1MB section mapping and for the rest of initial kernel address * space use L2 coarse tables. * * Link L2 tables for mapping remainder of kernel (modulo 1MB) * and kernel structures */ l2_start = lastaddr & ~(L1_S_OFFSET); for (i = 0 ; i < l2size - 1; i++) pmap_link_l2pt(l1pagetable, l2_start + i * L1_S_SIZE, &kernel_pt_table[i]); pmap_curmaxkvaddr = l2_start + (l2size - 1) * L1_S_SIZE; /* Map kernel code and data */ pmap_map_chunk(l1pagetable, KERNVIRTADDR, abp->abp_physaddr, (((uint32_t)(lastaddr) - KERNVIRTADDR) + PAGE_MASK) & ~PAGE_MASK, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map L1 directory and allocated L2 page tables */ pmap_map_chunk(l1pagetable, kernel_l1pt.pv_va, kernel_l1pt.pv_pa, L1_TABLE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, kernel_pt_table[0].pv_va, kernel_pt_table[0].pv_pa, L2_TABLE_SIZE_REAL * l2size, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); /* Map allocated DPCPU, stacks and msgbuf */ pmap_map_chunk(l1pagetable, dpcpu.pv_va, dpcpu.pv_pa, freemempos - dpcpu.pv_va, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Link and map the vector page */ pmap_link_l2pt(l1pagetable, ARM_VECTORS_HIGH, &kernel_pt_table[l2size - 1]); pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, PTE_CACHE); /* Establish static device mappings. */ err_devmap = platform_devmap_init(); devmap_bootstrap(l1pagetable, NULL); vm_max_kernel_address = platform_lastaddr(); cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | DOMAIN_CLIENT); pmap_pa = kernel_l1pt.pv_pa; cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)); /* * Now that proper page tables are installed, call cpu_setup() to enable * instruction and data caches and other chip-specific features. */ cpu_setup(); /* * Only after the SOC registers block is mapped we can perform device * tree fixups, as they may attempt to read parameters from hardware. */ OF_interpret("perform-fixup", 0); platform_gpio_init(); cninit(); debugf("initarm: console initialized\n"); debugf(" arg1 kmdp = 0x%08x\n", (uint32_t)kmdp); debugf(" boothowto = 0x%08x\n", boothowto); debugf(" dtbp = 0x%08x\n", (uint32_t)dtbp); print_kenv(); env = kern_getenv("kernelname"); if (env != NULL) { strlcpy(kernelname, env, sizeof(kernelname)); freeenv(env); } if (err_devmap != 0) printf("WARNING: could not fully configure devmap, error=%d\n", err_devmap); platform_late_init(); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ cpu_control(CPU_CONTROL_MMU_ENABLE, CPU_CONTROL_MMU_ENABLE); set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); undefined_init(); init_proc0(kernelstack.pv_va); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_bootstrap(freemempos, &kernel_l1pt); msgbufp = (void *)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Exclude the kernel (and all the things we allocated which immediately * follow the kernel) from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_exclude_region(abp->abp_physaddr, (virtual_avail - KERNVIRTADDR), EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); dbg_monitor_init(); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); } #else /* __ARM_ARCH < 6 */ void * initarm(struct arm_boot_params *abp) { struct mem_region mem_regions[FDT_MEM_REGIONS]; vm_paddr_t lastaddr; vm_offset_t dtbp, kernelstack, dpcpu; char *env; void *kmdp; int err_devmap, mem_regions_sz; #ifdef EFI struct efi_map_header *efihdr; #endif /* get last allocated physical address */ arm_physmem_kernaddr = abp->abp_physaddr; lastaddr = parse_boot_param(abp) - KERNVIRTADDR + arm_physmem_kernaddr; set_cpufuncs(); cpuinfo_init(); /* * Find the dtb passed in by the boot loader. */ kmdp = preload_search_by_type("elf kernel"); dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); #if defined(FDT_DTB_STATIC) /* * In case the device tree blob was not retrieved (from metadata) try * to use the statically embedded one. */ if (dtbp == (vm_offset_t)NULL) dtbp = (vm_offset_t)&fdt_static_dtb; #endif if (OF_install(OFW_FDT, 0) == FALSE) panic("Cannot install FDT"); if (OF_init((void *)dtbp) != 0) panic("OF_init failed with the found device tree"); #if defined(LINUX_BOOT_ABI) if (loader_envp == NULL && fdt_get_chosen_bootargs(linux_command_line, LBABI_MAX_COMMAND_LINE) == 0) cmdline_set_env(linux_command_line, CMDLINE_GUARD); #endif #ifdef EFI efihdr = (struct efi_map_header *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_MAP); if (efihdr != NULL) { add_efi_map_entries(efihdr, mem_regions, &mem_regions_sz); } else #endif { /* Grab physical memory regions information from device tree. */ if (fdt_get_mem_regions(mem_regions, &mem_regions_sz,NULL) != 0) panic("Cannot get physical memory regions"); } arm_physmem_hardware_regions(mem_regions, mem_regions_sz); /* Grab reserved memory regions information from device tree. */ if (fdt_get_reserved_regions(mem_regions, &mem_regions_sz) == 0) arm_physmem_exclude_regions(mem_regions, mem_regions_sz, EXFLAG_NODUMP | EXFLAG_NOALLOC); /* * Set TEX remapping registers. * Setup kernel page tables and switch to kernel L1 page table. */ pmap_set_tex(); pmap_bootstrap_prepare(lastaddr); /* * Now that proper page tables are installed, call cpu_setup() to enable * instruction and data caches and other chip-specific features. */ cpu_setup(); /* Platform-specific initialisation */ platform_probe_and_attach(); pcpu0_init(); /* Do basic tuning, hz etc */ init_param1(); /* * Allocate a page for the system page mapped to 0xffff0000 * This page will just contain the system vectors and can be * shared by all processes. */ systempage = pmap_preboot_get_pages(1); /* Map the vector page. */ pmap_preboot_map_pages(systempage, ARM_VECTORS_HIGH, 1); if (virtual_end >= ARM_VECTORS_HIGH) virtual_end = ARM_VECTORS_HIGH - 1; /* Allocate dynamic per-cpu area. */ dpcpu = pmap_preboot_get_vpages(DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu, 0); /* Allocate stacks for all modes */ irqstack = pmap_preboot_get_vpages(IRQ_STACK_SIZE * MAXCPU); abtstack = pmap_preboot_get_vpages(ABT_STACK_SIZE * MAXCPU); undstack = pmap_preboot_get_vpages(UND_STACK_SIZE * MAXCPU ); kernelstack = pmap_preboot_get_vpages(kstack_pages * MAXCPU); /* Allocate message buffer. */ msgbufp = (void *)pmap_preboot_get_vpages( round_page(msgbufsize) / PAGE_SIZE); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ set_stackptrs(0); mutex_init(); /* Establish static device mappings. */ err_devmap = platform_devmap_init(); devmap_bootstrap(0, NULL); vm_max_kernel_address = platform_lastaddr(); /* * Only after the SOC registers block is mapped we can perform device * tree fixups, as they may attempt to read parameters from hardware. */ OF_interpret("perform-fixup", 0); platform_gpio_init(); cninit(); debugf("initarm: console initialized\n"); debugf(" arg1 kmdp = 0x%08x\n", (uint32_t)kmdp); debugf(" boothowto = 0x%08x\n", boothowto); debugf(" dtbp = 0x%08x\n", (uint32_t)dtbp); debugf(" lastaddr1: 0x%08x\n", lastaddr); print_kenv(); env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); if (err_devmap != 0) printf("WARNING: could not fully configure devmap, error=%d\n", err_devmap); platform_late_init(); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ /* Set stack for exception handlers */ undefined_init(); init_proc0(kernelstack); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); enable_interrupts(PSR_A); pmap_bootstrap(0); /* Exclude the kernel (and all the things we allocated which immediately * follow the kernel) from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_exclude_region(abp->abp_physaddr, pmap_preboot_get_pages(0) - abp->abp_physaddr, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); /* Init message buffer. */ msgbufinit(msgbufp, msgbufsize); dbg_monitor_init(); kdb_init(); return ((void *)STACKALIGN(thread0.td_pcb)); } #endif /* __ARM_ARCH < 6 */ #endif /* FDT */ uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, struct timecounter *); uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) { return (arm_cpu_fill_vdso_timehands != NULL ? arm_cpu_fill_vdso_timehands(vdso_th, tc) : 0); } Index: head/sys/arm/arm/mpcore_timer.c =================================================================== --- head/sys/arm/arm/mpcore_timer.c (revision 299068) +++ head/sys/arm/arm/mpcore_timer.c (revision 299069) @@ -1,546 +1,546 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * Developed by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /** * The ARM Cortex-A9 core can support a global timer plus a private and * watchdog timer per core. This driver reserves memory and interrupt * resources for accessing both timer register sets, these resources are * stored globally and used to setup the timecount and eventtimer. * * The timecount timer uses the global 64-bit counter, whereas the * per-CPU eventtimer uses the private 32-bit counters. * * * REF: ARM Cortex-A9 MPCore, Technical Reference Manual (rev. r2p2) */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Private (per-CPU) timer register map */ #define PRV_TIMER_LOAD 0x0000 #define PRV_TIMER_COUNT 0x0004 #define PRV_TIMER_CTRL 0x0008 #define PRV_TIMER_INTR 0x000C #define PRV_TIMER_CTR_PRESCALER_SHIFT 8 #define PRV_TIMER_CTRL_IRQ_ENABLE (1UL << 2) #define PRV_TIMER_CTRL_AUTO_RELOAD (1UL << 1) #define PRV_TIMER_CTRL_TIMER_ENABLE (1UL << 0) #define PRV_TIMER_INTR_EVENT (1UL << 0) /* Global timer register map */ #define GBL_TIMER_COUNT_LOW 0x0000 #define GBL_TIMER_COUNT_HIGH 0x0004 #define GBL_TIMER_CTRL 0x0008 #define GBL_TIMER_INTR 0x000C #define GBL_TIMER_CTR_PRESCALER_SHIFT 8 #define GBL_TIMER_CTRL_AUTO_INC (1UL << 3) #define GBL_TIMER_CTRL_IRQ_ENABLE (1UL << 2) #define GBL_TIMER_CTRL_COMP_ENABLE (1UL << 1) #define GBL_TIMER_CTRL_TIMER_ENABLE (1UL << 0) #define GBL_TIMER_INTR_EVENT (1UL << 0) struct arm_tmr_softc { device_t dev; int irqrid; int memrid; struct resource * gbl_mem; struct resource * prv_mem; struct resource * prv_irq; uint64_t clkfreq; struct eventtimer et; }; static struct eventtimer *arm_tmr_et; static struct timecounter *arm_tmr_tc; static uint64_t arm_tmr_freq; static boolean_t arm_tmr_freq_varies; #define tmr_prv_read_4(sc, reg) bus_read_4((sc)->prv_mem, reg) #define tmr_prv_write_4(sc, reg, val) bus_write_4((sc)->prv_mem, reg, val) #define tmr_gbl_read_4(sc, reg) bus_read_4((sc)->gbl_mem, reg) #define tmr_gbl_write_4(sc, reg, val) bus_write_4((sc)->gbl_mem, reg, val) static timecounter_get_t arm_tmr_get_timecount; static struct timecounter arm_tmr_timecount = { .tc_name = "MPCore", .tc_get_timecount = arm_tmr_get_timecount, .tc_poll_pps = NULL, .tc_counter_mask = ~0u, .tc_frequency = 0, .tc_quality = 800, }; #define TMR_GBL 0x01 #define TMR_PRV 0x02 #define TMR_BOTH (TMR_GBL | TMR_PRV) #define TMR_NONE 0 static struct ofw_compat_data compat_data[] = { {"arm,mpcore-timers", TMR_BOTH}, /* Non-standard, FreeBSD. */ {"arm,cortex-a9-global-timer", TMR_GBL}, {"arm,cortex-a5-global-timer", TMR_GBL}, {"arm,cortex-a9-twd-timer", TMR_PRV}, {"arm,cortex-a5-twd-timer", TMR_PRV}, {"arm,arm11mp-twd-timer", TMR_PRV}, {NULL, TMR_NONE} }; /** * arm_tmr_get_timecount - reads the timecount (global) timer * @tc: pointer to arm_tmr_timecount struct * * We only read the lower 32-bits, the timecount stuff only uses 32-bits * so (for now?) ignore the upper 32-bits. * * RETURNS * The lower 32-bits of the counter. */ static unsigned arm_tmr_get_timecount(struct timecounter *tc) { struct arm_tmr_softc *sc; sc = tc->tc_priv; return (tmr_gbl_read_4(sc, GBL_TIMER_COUNT_LOW)); } /** * arm_tmr_start - starts the eventtimer (private) timer * @et: pointer to eventtimer struct * @first: the number of seconds and fractional sections to trigger in * @period: the period (in seconds and fractional sections) to set * * If the eventtimer is required to be in oneshot mode, period will be * NULL and first will point to the time to trigger. If in periodic mode * period will contain the time period and first may optionally contain * the time for the first period. * * RETURNS * Always returns 0 */ static int arm_tmr_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { struct arm_tmr_softc *sc; uint32_t load, count; uint32_t ctrl; sc = et->et_priv; tmr_prv_write_4(sc, PRV_TIMER_CTRL, 0); tmr_prv_write_4(sc, PRV_TIMER_INTR, PRV_TIMER_INTR_EVENT); ctrl = PRV_TIMER_CTRL_IRQ_ENABLE | PRV_TIMER_CTRL_TIMER_ENABLE; if (period != 0) { load = ((uint32_t)et->et_frequency * period) >> 32; ctrl |= PRV_TIMER_CTRL_AUTO_RELOAD; } else load = 0; if (first != 0) count = (uint32_t)((et->et_frequency * first) >> 32); else count = load; tmr_prv_write_4(sc, PRV_TIMER_LOAD, load); tmr_prv_write_4(sc, PRV_TIMER_COUNT, count); tmr_prv_write_4(sc, PRV_TIMER_CTRL, ctrl); return (0); } /** * arm_tmr_stop - stops the eventtimer (private) timer * @et: pointer to eventtimer struct * * Simply stops the private timer by clearing all bits in the ctrl register. * * RETURNS * Always returns 0 */ static int arm_tmr_stop(struct eventtimer *et) { struct arm_tmr_softc *sc; sc = et->et_priv; tmr_prv_write_4(sc, PRV_TIMER_CTRL, 0); tmr_prv_write_4(sc, PRV_TIMER_INTR, PRV_TIMER_INTR_EVENT); return (0); } /** * arm_tmr_intr - ISR for the eventtimer (private) timer * @arg: pointer to arm_tmr_softc struct * * Clears the event register and then calls the eventtimer callback. * * RETURNS * Always returns FILTER_HANDLED */ static int arm_tmr_intr(void *arg) { struct arm_tmr_softc *sc; sc = arg; tmr_prv_write_4(sc, PRV_TIMER_INTR, PRV_TIMER_INTR_EVENT); if (sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); return (FILTER_HANDLED); } /** * arm_tmr_probe - timer probe routine * @dev: new device * * The probe function returns success when probed with the fdt compatible * string set to "arm,mpcore-timers". * * RETURNS * BUS_PROBE_DEFAULT if the fdt device is compatible, otherwise ENXIO. */ static int arm_tmr_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == TMR_NONE) return (ENXIO); device_set_desc(dev, "ARM MPCore Timers"); return (BUS_PROBE_DEFAULT); } static int attach_tc(struct arm_tmr_softc *sc) { int rid; if (arm_tmr_tc != NULL) return (EBUSY); rid = sc->memrid; sc->gbl_mem = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->gbl_mem == NULL) { device_printf(sc->dev, "could not allocate gbl mem resources\n"); return (ENXIO); } tmr_gbl_write_4(sc, GBL_TIMER_CTRL, 0x00000000); arm_tmr_timecount.tc_frequency = sc->clkfreq; arm_tmr_timecount.tc_priv = sc; tc_init(&arm_tmr_timecount); arm_tmr_tc = &arm_tmr_timecount; tmr_gbl_write_4(sc, GBL_TIMER_CTRL, GBL_TIMER_CTRL_TIMER_ENABLE); return (0); } static int attach_et(struct arm_tmr_softc *sc) { void *ihl; int irid, mrid; if (arm_tmr_et != NULL) return (EBUSY); mrid = sc->memrid; sc->prv_mem = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &mrid, RF_ACTIVE); if (sc->prv_mem == NULL) { device_printf(sc->dev, "could not allocate prv mem resources\n"); return (ENXIO); } tmr_prv_write_4(sc, PRV_TIMER_CTRL, 0x00000000); irid = sc->irqrid; sc->prv_irq = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irid, RF_ACTIVE); if (sc->prv_irq == NULL) { bus_release_resource(sc->dev, SYS_RES_MEMORY, mrid, sc->prv_mem); device_printf(sc->dev, "could not allocate prv irq resources\n"); return (ENXIO); } if (bus_setup_intr(sc->dev, sc->prv_irq, INTR_TYPE_CLK, arm_tmr_intr, NULL, sc, &ihl) != 0) { bus_release_resource(sc->dev, SYS_RES_MEMORY, mrid, sc->prv_mem); bus_release_resource(sc->dev, SYS_RES_IRQ, irid, sc->prv_irq); device_printf(sc->dev, "unable to setup the et irq handler.\n"); return (ENXIO); } /* * Setup and register the eventtimer. Most event timers set their min * and max period values to some value calculated from the clock * frequency. We might not know yet what our runtime clock frequency * will be, so we just use some safe values. A max of 2 seconds ensures * that even if our base clock frequency is 2GHz (meaning a 4GHz CPU), * we won't overflow our 32-bit timer count register. A min of 20 * nanoseconds is pretty much completely arbitrary. */ sc->et.et_name = "MPCore"; sc->et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; sc->et.et_quality = 1000; sc->et.et_frequency = sc->clkfreq; sc->et.et_min_period = 20 * SBT_1NS; sc->et.et_max_period = 2 * SBT_1S; sc->et.et_start = arm_tmr_start; sc->et.et_stop = arm_tmr_stop; sc->et.et_priv = sc; et_register(&sc->et); arm_tmr_et = &sc->et; return (0); } /** * arm_tmr_attach - attaches the timer to the simplebus * @dev: new device * * Reserves memory and interrupt resources, stores the softc structure * globally and registers both the timecount and eventtimer objects. * * RETURNS - * Zero on sucess or ENXIO if an error occuried. + * Zero on success or ENXIO if an error occuried. */ static int arm_tmr_attach(device_t dev) { struct arm_tmr_softc *sc; phandle_t node; pcell_t clock; int et_err, tc_err, tmrtype; sc = device_get_softc(dev); sc->dev = dev; if (arm_tmr_freq_varies) { sc->clkfreq = arm_tmr_freq; } else { if (arm_tmr_freq != 0) { sc->clkfreq = arm_tmr_freq; } else { /* Get the base clock frequency */ node = ofw_bus_get_node(dev); if ((OF_getencprop(node, "clock-frequency", &clock, sizeof(clock))) <= 0) { device_printf(dev, "missing clock-frequency " "attribute in FDT\n"); return (ENXIO); } sc->clkfreq = clock; } } tmrtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data; tc_err = ENXIO; et_err = ENXIO; /* * If we're handling the global timer and it is fixed-frequency, set it * up to use as a timecounter. If it's variable frequency it won't work * as a timecounter. We also can't use it for DELAY(), so hopefully the * platform provides its own implementation. If it doesn't, ours will * get used, but since the frequency isn't set, it will only use the * bogus loop counter. */ if (tmrtype & TMR_GBL) { if (!arm_tmr_freq_varies) tc_err = attach_tc(sc); else if (bootverbose) device_printf(sc->dev, "not using variable-frequency device as timecounter"); sc->memrid++; sc->irqrid++; } /* If we are handling the private timer, set it up as an eventtimer. */ if (tmrtype & TMR_PRV) { et_err = attach_et(sc); } /* * If we didn't successfully set up a timecounter or eventtimer then we * didn't actually attach at all, return error. */ if (tc_err != 0 && et_err != 0) { return (ENXIO); } return (0); } static device_method_t arm_tmr_methods[] = { DEVMETHOD(device_probe, arm_tmr_probe), DEVMETHOD(device_attach, arm_tmr_attach), { 0, 0 } }; static driver_t arm_tmr_driver = { "mp_tmr", arm_tmr_methods, sizeof(struct arm_tmr_softc), }; static devclass_t arm_tmr_devclass; EARLY_DRIVER_MODULE(mp_tmr, simplebus, arm_tmr_driver, arm_tmr_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(mp_tmr, ofwbus, arm_tmr_driver, arm_tmr_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); /* * Handle a change in clock frequency. The mpcore timer runs at half the CPU * frequency. When the CPU frequency changes due to power-saving or thermal - * managment, the platform-specific code that causes the frequency change calls + * management, the platform-specific code that causes the frequency change calls * this routine to inform the clock driver, and we in turn inform the event * timer system, which actually updates the value in et->frequency for us and * reschedules the current event(s) in a way that's atomic with respect to * start/stop/intr code that may be running on various CPUs at the time of the * call. * * This routine can also be called by a platform's early init code. If the * value passed is ARM_TMR_FREQUENCY_VARIES, that will cause the attach() code * to register as an eventtimer, but not a timecounter. If the value passed in * is any other non-zero value it is used as the fixed frequency for the timer. */ void arm_tmr_change_frequency(uint64_t newfreq) { if (newfreq == ARM_TMR_FREQUENCY_VARIES) { arm_tmr_freq_varies = true; return; } arm_tmr_freq = newfreq; if (arm_tmr_et != NULL) et_change_frequency(arm_tmr_et, newfreq); } /** * DELAY - Delay for at least usec microseconds. * @usec: number of microseconds to delay by * * This function is called all over the kernel and is suppose to provide a * consistent delay. This function may also be called before the console * is setup so no printf's can be called here. * * RETURNS: * nothing */ static void __used /* Must emit function code for the weak ref below. */ arm_tmr_DELAY(int usec) { struct arm_tmr_softc *sc; int32_t counts_per_usec; int32_t counts; uint32_t first, last; /* Check the timers are setup, if not just use a for loop for the meantime */ if (arm_tmr_tc == NULL || arm_tmr_timecount.tc_frequency == 0) { for (; usec > 0; usec--) for (counts = 200; counts > 0; counts--) cpufunc_nullop(); /* Prevent gcc from optimizing * out the loop */ return; } sc = arm_tmr_tc->tc_priv; /* Get the number of times to count */ counts_per_usec = ((arm_tmr_timecount.tc_frequency / 1000000) + 1); /* * Clamp the timeout at a maximum value (about 32 seconds with * a 66MHz clock). *Nobody* should be delay()ing for anywhere * near that length of time and if they are, they should be hung * out to dry. */ if (usec >= (0x80000000U / counts_per_usec)) counts = (0x80000000U / counts_per_usec) - 1; else counts = usec * counts_per_usec; first = tmr_gbl_read_4(sc, GBL_TIMER_COUNT_LOW); while (counts > 0) { last = tmr_gbl_read_4(sc, GBL_TIMER_COUNT_LOW); counts -= (int32_t)(last - first); first = last; } } /* * Supply a DELAY() implementation via weak linkage. A platform may want to use * the mpcore per-cpu eventtimers but provide its own DELAY() routine, * especially when the core frequency can change on the fly. */ __weak_reference(arm_tmr_DELAY, DELAY); Index: head/sys/arm/arm/pmap-v4.c =================================================================== --- head/sys/arm/arm/pmap-v4.c (revision 299068) +++ head/sys/arm/arm/pmap-v4.c (revision 299069) @@ -1,4874 +1,4874 @@ /* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */ /*- * Copyright 2004 Olivier Houchard. * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2002-2003 Wasabi Systems, Inc. * Copyright (c) 2001 Richard Earnshaw * Copyright (c) 2001-2002 Christopher Gilbert * All rights reserved. * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /*- * Copyright (c) 1999 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Charles M. Hannum. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * * RiscBSD kernel project * * pmap.c * - * Machine dependant vm stuff + * Machine dependent vm stuff * * Created : 20/09/94 */ /* * Special compilation symbols * PMAP_DEBUG - Build in pmap_debug_level code * * Note that pmap_mapdev() and pmap_unmapdev() are implemented in arm/devmap.c */ /* Include header files */ #include "opt_vm.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef PMAP_DEBUG #define PDEBUG(_lev_,_stat_) \ if (pmap_debug_level >= (_lev_)) \ ((_stat_)) #define dprintf printf int pmap_debug_level = 0; #define PMAP_INLINE #else /* PMAP_DEBUG */ #define PDEBUG(_lev_,_stat_) /* Nothing */ #define dprintf(x, arg...) #define PMAP_INLINE __inline #endif /* PMAP_DEBUG */ extern struct pv_addr systempage; extern int last_fault_code; /* * Internal function prototypes */ static void pmap_free_pv_entry (pv_entry_t); static pv_entry_t pmap_get_pv_entry(void); static int pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int); static vm_paddr_t pmap_extract_locked(pmap_t pmap, vm_offset_t va); static void pmap_fix_cache(struct vm_page *, pmap_t, vm_offset_t); static void pmap_alloc_l1(pmap_t); static void pmap_free_l1(pmap_t); static int pmap_clearbit(struct vm_page *, u_int); static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t); static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t); static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); static vm_offset_t kernel_pt_lookup(vm_paddr_t); static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1"); vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ vm_offset_t pmap_curmaxkvaddr; vm_paddr_t kernel_l1pa; vm_offset_t kernel_vm_end = 0; vm_offset_t vm_max_kernel_address; struct pmap kernel_pmap_store; static pt_entry_t *csrc_pte, *cdst_pte; static vm_offset_t csrcp, cdstp, qmap_addr; static struct mtx cmtx, qmap_mtx; static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); /* * These routines are called when the CPU type is identified to set up * the PTE prototypes, cache modes, etc. * * The variables are always here, just in case LKMs need to reference * them (though, they shouldn't). */ pt_entry_t pte_l1_s_cache_mode; pt_entry_t pte_l1_s_cache_mode_pt; pt_entry_t pte_l1_s_cache_mask; pt_entry_t pte_l2_l_cache_mode; pt_entry_t pte_l2_l_cache_mode_pt; pt_entry_t pte_l2_l_cache_mask; pt_entry_t pte_l2_s_cache_mode; pt_entry_t pte_l2_s_cache_mode_pt; pt_entry_t pte_l2_s_cache_mask; pt_entry_t pte_l2_s_prot_u; pt_entry_t pte_l2_s_prot_w; pt_entry_t pte_l2_s_prot_mask; pt_entry_t pte_l1_s_proto; pt_entry_t pte_l1_c_proto; pt_entry_t pte_l2_s_proto; void (*pmap_copy_page_func)(vm_paddr_t, vm_paddr_t); void (*pmap_copy_page_offs_func)(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); void (*pmap_zero_page_func)(vm_paddr_t, int, int); struct msgbuf *msgbufp = NULL; /* * Crashdump maps. */ static caddr_t crashdumpmap; extern void bcopy_page(vm_offset_t, vm_offset_t); extern void bzero_page(vm_offset_t); extern vm_offset_t alloc_firstaddr; char *_tmppt; /* * Metadata for L1 translation tables. */ struct l1_ttable { /* Entry on the L1 Table list */ SLIST_ENTRY(l1_ttable) l1_link; /* Entry on the L1 Least Recently Used list */ TAILQ_ENTRY(l1_ttable) l1_lru; /* Track how many domains are allocated from this L1 */ volatile u_int l1_domain_use_count; /* * A free-list of domain numbers for this L1. * We avoid using ffs() and a bitmap to track domains since ffs() * is slow on ARM. */ u_int8_t l1_domain_first; u_int8_t l1_domain_free[PMAP_DOMAINS]; /* Physical address of this L1 page table */ vm_paddr_t l1_physaddr; /* KVA of this L1 page table */ pd_entry_t *l1_kva; }; /* * Convert a virtual address into its L1 table index. That is, the * index used to locate the L2 descriptor table pointer in an L1 table. * This is basically used to index l1->l1_kva[]. * * Each L2 descriptor table represents 1MB of VA space. */ #define L1_IDX(va) (((vm_offset_t)(va)) >> L1_S_SHIFT) /* * L1 Page Tables are tracked using a Least Recently Used list. * - New L1s are allocated from the HEAD. * - Freed L1s are added to the TAIl. * - Recently accessed L1s (where an 'access' is some change to one of * the userland pmaps which owns this L1) are moved to the TAIL. */ static TAILQ_HEAD(, l1_ttable) l1_lru_list; /* * A list of all L1 tables */ static SLIST_HEAD(, l1_ttable) l1_list; static struct mtx l1_lru_lock; /* * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. * * This is normally 16MB worth L2 page descriptors for any given pmap. * Reference counts are maintained for L2 descriptors so they can be * freed when empty. */ struct l2_dtable { /* The number of L2 page descriptors allocated to this l2_dtable */ u_int l2_occupancy; /* List of L2 page descriptors */ struct l2_bucket { pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ vm_paddr_t l2b_phys; /* Physical address of same */ u_short l2b_l1idx; /* This L2 table's L1 index */ u_short l2b_occupancy; /* How many active descriptors */ } l2_bucket[L2_BUCKET_SIZE]; }; /* pmap_kenter_internal flags */ #define KENTER_CACHE 0x1 #define KENTER_USER 0x2 /* * Given an L1 table index, calculate the corresponding l2_dtable index * and bucket index within the l2_dtable. */ #define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \ (L2_SIZE - 1)) #define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1)) /* * Given a virtual address, this macro returns the * virtual address required to drop into the next L2 bucket. */ #define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE) /* * We try to map the page tables write-through, if possible. However, not * all CPUs have a write-through cache mode, so on those we have to sync * the cache when we frob page tables. * * We try to evaluate this at compile time, if possible. However, it's * not always possible to do that, hence this run-time var. */ int pmap_needs_pte_sync; /* * Macro to determine if a mapping might be resident in the * instruction cache and/or TLB */ #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) /* * Macro to determine if a mapping might be resident in the * data cache and/or TLB */ #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif #define pmap_is_current(pm) ((pm) == kernel_pmap || \ curproc->p_vmspace->vm_map.pmap == (pm)) static uma_zone_t pvzone = NULL; uma_zone_t l2zone; static uma_zone_t l2table_zone; static vm_offset_t pmap_kernel_l2dtable_kva; static vm_offset_t pmap_kernel_l2ptp_kva; static vm_paddr_t pmap_kernel_l2ptp_phys; static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; static struct rwlock pvh_global_lock; void pmap_copy_page_offs_generic(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); #if ARM_MMU_XSCALE == 1 void pmap_copy_page_offs_xscale(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); #endif /* * This list exists for the benefit of pmap_map_chunk(). It keeps track * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can * find them as necessary. * * Note that the data on this list MUST remain valid after initarm() returns, * as pmap_bootstrap() uses it to contruct L2 table metadata. */ SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); static void pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) { int i; l1->l1_kva = l1pt; l1->l1_domain_use_count = 0; l1->l1_domain_first = 0; for (i = 0; i < PMAP_DOMAINS; i++) l1->l1_domain_free[i] = i + 1; /* * Copy the kernel's L1 entries to each new L1. */ if (l1pt != kernel_pmap->pm_l1->l1_kva) memcpy(l1pt, kernel_pmap->pm_l1->l1_kva, L1_TABLE_SIZE); if ((l1->l1_physaddr = pmap_extract(kernel_pmap, (vm_offset_t)l1pt)) == 0) panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); SLIST_INSERT_HEAD(&l1_list, l1, l1_link); TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); } static vm_offset_t kernel_pt_lookup(vm_paddr_t pa) { struct pv_addr *pv; SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { if (pv->pv_pa == pa) return (pv->pv_va); } return (0); } #if ARM_MMU_GENERIC != 0 void pmap_pte_init_generic(void) { pte_l1_s_cache_mode = L1_S_B|L1_S_C; pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; pte_l2_l_cache_mode = L2_B|L2_C; pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; pte_l2_s_cache_mode = L2_B|L2_C; pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; /* * If we have a write-through cache, set B and C. If * we have a write-back cache, then we assume setting * only C will make those pages write-through. */ if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop) { pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; pte_l2_l_cache_mode_pt = L2_B|L2_C; pte_l2_s_cache_mode_pt = L2_B|L2_C; } else { pte_l1_s_cache_mode_pt = L1_S_C; pte_l2_l_cache_mode_pt = L2_C; pte_l2_s_cache_mode_pt = L2_C; } pte_l2_s_prot_u = L2_S_PROT_U_generic; pte_l2_s_prot_w = L2_S_PROT_W_generic; pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; pte_l1_s_proto = L1_S_PROTO_generic; pte_l1_c_proto = L1_C_PROTO_generic; pte_l2_s_proto = L2_S_PROTO_generic; pmap_copy_page_func = pmap_copy_page_generic; pmap_copy_page_offs_func = pmap_copy_page_offs_generic; pmap_zero_page_func = pmap_zero_page_generic; } #endif /* ARM_MMU_GENERIC != 0 */ #if ARM_MMU_XSCALE == 1 #if (ARM_NMMUS > 1) || defined (CPU_XSCALE_CORE3) static u_int xscale_use_minidata; #endif void pmap_pte_init_xscale(void) { uint32_t auxctl; int write_through = 0; pte_l1_s_cache_mode = L1_S_B|L1_S_C|L1_S_XSCALE_P; pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; pte_l2_l_cache_mode = L2_B|L2_C; pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; pte_l2_s_cache_mode = L2_B|L2_C; pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; pte_l1_s_cache_mode_pt = L1_S_C; pte_l2_l_cache_mode_pt = L2_C; pte_l2_s_cache_mode_pt = L2_C; #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE /* * The XScale core has an enhanced mode where writes that * miss the cache cause a cache line to be allocated. This * is significantly faster than the traditional, write-through * behavior of this case. */ pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ #ifdef XSCALE_CACHE_WRITE_THROUGH /* * Some versions of the XScale core have various bugs in * their cache units, the work-around for which is to run * the cache in write-through mode. Unfortunately, this * has a major (negative) impact on performance. So, we * go ahead and run fast-and-loose, in the hopes that we * don't line up the planets in a way that will trip the * bugs. * * However, we give you the option to be slow-but-correct. */ write_through = 1; #elif defined(XSCALE_CACHE_WRITE_BACK) /* force write back cache mode */ write_through = 0; #elif defined(CPU_XSCALE_PXA2X0) /* * Intel PXA2[15]0 processors are known to have a bug in * write-back cache on revision 4 and earlier (stepping * A[01] and B[012]). Fixed for C0 and later. */ { uint32_t id, type; id = cpu_ident(); type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { if ((id & CPU_ID_REVISION_MASK) < 5) { /* write through for stepping A0-1 and B0-2 */ write_through = 1; } } } #endif /* XSCALE_CACHE_WRITE_THROUGH */ if (write_through) { pte_l1_s_cache_mode = L1_S_C; pte_l2_l_cache_mode = L2_C; pte_l2_s_cache_mode = L2_C; } #if (ARM_NMMUS > 1) xscale_use_minidata = 1; #endif pte_l2_s_prot_u = L2_S_PROT_U_xscale; pte_l2_s_prot_w = L2_S_PROT_W_xscale; pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; pte_l1_s_proto = L1_S_PROTO_xscale; pte_l1_c_proto = L1_C_PROTO_xscale; pte_l2_s_proto = L2_S_PROTO_xscale; #ifdef CPU_XSCALE_CORE3 pmap_copy_page_func = pmap_copy_page_generic; pmap_copy_page_offs_func = pmap_copy_page_offs_generic; pmap_zero_page_func = pmap_zero_page_generic; xscale_use_minidata = 0; /* Make sure it is L2-cachable */ pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_T); pte_l1_s_cache_mode_pt = pte_l1_s_cache_mode &~ L1_S_XSCALE_P; pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_T) ; pte_l2_l_cache_mode_pt = pte_l1_s_cache_mode; pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_T); pte_l2_s_cache_mode_pt = pte_l2_s_cache_mode; #else pmap_copy_page_func = pmap_copy_page_xscale; pmap_copy_page_offs_func = pmap_copy_page_offs_xscale; pmap_zero_page_func = pmap_zero_page_xscale; #endif /* * Disable ECC protection of page table access, for now. */ __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); auxctl &= ~XSCALE_AUXCTL_P; __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); } /* * xscale_setup_minidata: * * Set up the mini-data cache clean area. We require the * caller to allocate the right amount of physically and * virtually contiguous space. */ extern vm_offset_t xscale_minidata_clean_addr; extern vm_size_t xscale_minidata_clean_size; /* already initialized */ void xscale_setup_minidata(vm_offset_t l1pt, vm_offset_t va, vm_paddr_t pa) { pd_entry_t *pde = (pd_entry_t *) l1pt; pt_entry_t *pte; vm_size_t size; uint32_t auxctl; xscale_minidata_clean_addr = va; /* Round it to page size. */ size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; for (; size != 0; va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { pte = (pt_entry_t *) kernel_pt_lookup( pde[L1_IDX(va)] & L1_C_ADDR_MASK); if (pte == NULL) panic("xscale_setup_minidata: can't find L2 table for " "VA 0x%08x", (u_int32_t) va); pte[l2pte_index(va)] = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); } /* * Configure the mini-data cache for write-back with * read/write-allocate. * * NOTE: In order to reconfigure the mini-data cache, we must * make sure it contains no valid data! In order to do that, * we must issue a global data cache invalidate command! * * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! * THIS IS VERY IMPORTANT! */ /* Invalidate data and mini-data. */ __asm __volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); } #endif /* * Allocate an L1 translation table for the specified pmap. * This is called at pmap creation time. */ static void pmap_alloc_l1(pmap_t pm) { struct l1_ttable *l1; u_int8_t domain; /* * Remove the L1 at the head of the LRU list */ mtx_lock(&l1_lru_lock); l1 = TAILQ_FIRST(&l1_lru_list); TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); /* * Pick the first available domain number, and update * the link to the next number. */ domain = l1->l1_domain_first; l1->l1_domain_first = l1->l1_domain_free[domain]; /* * If there are still free domain numbers in this L1, * put it back on the TAIL of the LRU list. */ if (++l1->l1_domain_use_count < PMAP_DOMAINS) TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); mtx_unlock(&l1_lru_lock); /* * Fix up the relevant bits in the pmap structure */ pm->pm_l1 = l1; pm->pm_domain = domain + 1; } /* * Free an L1 translation table. * This is called at pmap destruction time. */ static void pmap_free_l1(pmap_t pm) { struct l1_ttable *l1 = pm->pm_l1; mtx_lock(&l1_lru_lock); /* * If this L1 is currently on the LRU list, remove it. */ if (l1->l1_domain_use_count < PMAP_DOMAINS) TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); /* * Free up the domain number which was allocated to the pmap */ l1->l1_domain_free[pm->pm_domain - 1] = l1->l1_domain_first; l1->l1_domain_first = pm->pm_domain - 1; l1->l1_domain_use_count--; /* * The L1 now must have at least 1 free domain, so add * it back to the LRU list. If the use count is zero, * put it at the head of the list, otherwise it goes * to the tail. */ if (l1->l1_domain_use_count == 0) { TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); } else TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); mtx_unlock(&l1_lru_lock); } /* * Returns a pointer to the L2 bucket associated with the specified pmap * and VA, or NULL if no L2 bucket exists for the address. */ static PMAP_INLINE struct l2_bucket * pmap_get_l2_bucket(pmap_t pm, vm_offset_t va) { struct l2_dtable *l2; struct l2_bucket *l2b; u_short l1idx; l1idx = L1_IDX(va); if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) return (NULL); return (l2b); } /* * Returns a pointer to the L2 bucket associated with the specified pmap * and VA. * * If no L2 bucket exists, perform the necessary allocations to put an L2 * bucket/page table in place. * * Note that if a new L2 bucket/page was allocated, the caller *must* * increment the bucket occupancy counter appropriately *before* * releasing the pmap's lock to ensure no other thread or cpu deallocates * the bucket/page in the meantime. */ static struct l2_bucket * pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) { struct l2_dtable *l2; struct l2_bucket *l2b; u_short l1idx; l1idx = L1_IDX(va); PMAP_ASSERT_LOCKED(pm); rw_assert(&pvh_global_lock, RA_WLOCKED); if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { /* * No mapping at this address, as there is * no entry in the L1 table. * Need to allocate a new l2_dtable. */ PMAP_UNLOCK(pm); rw_wunlock(&pvh_global_lock); if ((l2 = uma_zalloc(l2table_zone, M_NOWAIT)) == NULL) { rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); return (NULL); } rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); if (pm->pm_l2[L2_IDX(l1idx)] != NULL) { /* * Someone already allocated the l2_dtable while * we were doing the same. */ uma_zfree(l2table_zone, l2); l2 = pm->pm_l2[L2_IDX(l1idx)]; } else { bzero(l2, sizeof(*l2)); /* * Link it into the parent pmap */ pm->pm_l2[L2_IDX(l1idx)] = l2; } } l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; /* * Fetch pointer to the L2 page table associated with the address. */ if (l2b->l2b_kva == NULL) { pt_entry_t *ptep; /* * No L2 page table has been allocated. Chances are, this * is because we just allocated the l2_dtable, above. */ l2->l2_occupancy++; PMAP_UNLOCK(pm); rw_wunlock(&pvh_global_lock); ptep = uma_zalloc(l2zone, M_NOWAIT); rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); if (l2b->l2b_kva != NULL) { /* We lost the race. */ l2->l2_occupancy--; uma_zfree(l2zone, ptep); return (l2b); } l2b->l2b_phys = vtophys(ptep); if (ptep == NULL) { /* * Oops, no more L2 page tables available at this * time. We may need to deallocate the l2_dtable * if we allocated a new one above. */ l2->l2_occupancy--; if (l2->l2_occupancy == 0) { pm->pm_l2[L2_IDX(l1idx)] = NULL; uma_zfree(l2table_zone, l2); } return (NULL); } l2b->l2b_kva = ptep; l2b->l2b_l1idx = l1idx; } return (l2b); } static PMAP_INLINE void #ifndef PMAP_INCLUDE_PTE_SYNC pmap_free_l2_ptp(pt_entry_t *l2) #else pmap_free_l2_ptp(boolean_t need_sync, pt_entry_t *l2) #endif { #ifdef PMAP_INCLUDE_PTE_SYNC /* * Note: With a write-back cache, we may need to sync this * L2 table before re-using it. * This is because it may have belonged to a non-current * pmap, in which case the cache syncs would have been * skipped when the pages were being unmapped. If the * L2 table were then to be immediately re-allocated to * the *current* pmap, it may well contain stale mappings * which have not yet been cleared by a cache write-back * and so would still be visible to the mmu. */ if (need_sync) PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); #endif uma_zfree(l2zone, l2); } /* * One or more mappings in the specified L2 descriptor table have just been * invalidated. * * Garbage collect the metadata and descriptor table itself if necessary. * * The pmap lock must be acquired when this is called (not necessary * for the kernel pmap). */ static void pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) { struct l2_dtable *l2; pd_entry_t *pl1pd, l1pd; pt_entry_t *ptep; u_short l1idx; /* * Update the bucket's reference count according to how many * PTEs the caller has just invalidated. */ l2b->l2b_occupancy -= count; /* * Note: * * Level 2 page tables allocated to the kernel pmap are never freed * as that would require checking all Level 1 page tables and * removing any references to the Level 2 page table. See also the * comment elsewhere about never freeing bootstrap L2 descriptors. * * We make do with just invalidating the mapping in the L2 table. * * This isn't really a big deal in practice and, in fact, leads * to a performance win over time as we don't need to continually * alloc/free. */ if (l2b->l2b_occupancy > 0 || pm == kernel_pmap) return; /* * There are no more valid mappings in this level 2 page table. * Go ahead and NULL-out the pointer in the bucket, then * free the page table. */ l1idx = l2b->l2b_l1idx; ptep = l2b->l2b_kva; l2b->l2b_kva = NULL; pl1pd = &pm->pm_l1->l1_kva[l1idx]; /* * If the L1 slot matches the pmap's domain * number, then invalidate it. */ l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { *pl1pd = 0; PTE_SYNC(pl1pd); } /* * Release the L2 descriptor table back to the pool cache. */ #ifndef PMAP_INCLUDE_PTE_SYNC pmap_free_l2_ptp(ptep); #else pmap_free_l2_ptp(!pmap_is_current(pm), ptep); #endif /* * Update the reference count in the associated l2_dtable */ l2 = pm->pm_l2[L2_IDX(l1idx)]; if (--l2->l2_occupancy > 0) return; /* * There are no more valid mappings in any of the Level 1 * slots managed by this l2_dtable. Go ahead and NULL-out * the pointer in the parent pmap and free the l2_dtable. */ pm->pm_l2[L2_IDX(l1idx)] = NULL; uma_zfree(l2table_zone, l2); } /* * Pool cache constructors for L2 descriptor tables, metadata and pmap * structures. */ static int pmap_l2ptp_ctor(void *mem, int size, void *arg, int flags) { #ifndef PMAP_INCLUDE_PTE_SYNC struct l2_bucket *l2b; pt_entry_t *ptep, pte; vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK; /* * The mappings for these page tables were initially made using * pmap_kenter() by the pool subsystem. Therefore, the cache- * mode will not be right for page table mappings. To avoid * polluting the pmap_kenter() code with a special case for * page tables, we simply fix up the cache-mode here if it's not * correct. */ l2b = pmap_get_l2_bucket(kernel_pmap, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; pte = *ptep; if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { /* * Page tables must have the cache-mode set to * Write-Thru. */ *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; PTE_SYNC(ptep); cpu_tlb_flushD_SE(va); cpu_cpwait(); } #endif memset(mem, 0, L2_TABLE_SIZE_REAL); PTE_SYNC_RANGE(mem, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); return (0); } /* * A bunch of routines to conditionally flush the caches/TLB depending * on whether the specified pmap actually needs to be flushed at any * given time. */ static PMAP_INLINE void pmap_tlb_flushID_SE(pmap_t pm, vm_offset_t va) { if (pmap_is_current(pm)) cpu_tlb_flushID_SE(va); } static PMAP_INLINE void pmap_tlb_flushD_SE(pmap_t pm, vm_offset_t va) { if (pmap_is_current(pm)) cpu_tlb_flushD_SE(va); } static PMAP_INLINE void pmap_tlb_flushID(pmap_t pm) { if (pmap_is_current(pm)) cpu_tlb_flushID(); } static PMAP_INLINE void pmap_tlb_flushD(pmap_t pm) { if (pmap_is_current(pm)) cpu_tlb_flushD(); } static int pmap_has_valid_mapping(pmap_t pm, vm_offset_t va) { pd_entry_t *pde; pt_entry_t *ptep; if (pmap_get_pde_pte(pm, va, &pde, &ptep) && ptep && ((*ptep & L2_TYPE_MASK) != L2_TYPE_INV)) return (1); return (0); } static PMAP_INLINE void pmap_idcache_wbinv_range(pmap_t pm, vm_offset_t va, vm_size_t len) { vm_size_t rest; CTR4(KTR_PMAP, "pmap_dcache_wbinv_range: pmap %p is_kernel %d va 0x%08x" " len 0x%x ", pm, pm == kernel_pmap, va, len); if (pmap_is_current(pm) || pm == kernel_pmap) { rest = MIN(PAGE_SIZE - (va & PAGE_MASK), len); while (len > 0) { if (pmap_has_valid_mapping(pm, va)) { cpu_idcache_wbinv_range(va, rest); cpu_l2cache_wbinv_range(va, rest); } len -= rest; va += rest; rest = MIN(PAGE_SIZE, len); } } } static PMAP_INLINE void pmap_dcache_wb_range(pmap_t pm, vm_offset_t va, vm_size_t len, boolean_t do_inv, boolean_t rd_only) { vm_size_t rest; CTR4(KTR_PMAP, "pmap_dcache_wb_range: pmap %p is_kernel %d va 0x%08x " "len 0x%x ", pm, pm == kernel_pmap, va, len); CTR2(KTR_PMAP, " do_inv %d rd_only %d", do_inv, rd_only); if (pmap_is_current(pm)) { rest = MIN(PAGE_SIZE - (va & PAGE_MASK), len); while (len > 0) { if (pmap_has_valid_mapping(pm, va)) { if (do_inv && rd_only) { cpu_dcache_inv_range(va, rest); cpu_l2cache_inv_range(va, rest); } else if (do_inv) { cpu_dcache_wbinv_range(va, rest); cpu_l2cache_wbinv_range(va, rest); } else if (!rd_only) { cpu_dcache_wb_range(va, rest); cpu_l2cache_wb_range(va, rest); } } len -= rest; va += rest; rest = MIN(PAGE_SIZE, len); } } } static PMAP_INLINE void pmap_idcache_wbinv_all(pmap_t pm) { if (pmap_is_current(pm)) { cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); } } #ifdef notyet static PMAP_INLINE void pmap_dcache_wbinv_all(pmap_t pm) { if (pmap_is_current(pm)) { cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); } } #endif /* * PTE_SYNC_CURRENT: * * Make sure the pte is written out to RAM. * We need to do this for one of two cases: * - We're dealing with the kernel pmap * - There is no pmap active in the cache/tlb. * - The specified pmap is 'active' in the cache/tlb. */ #ifdef PMAP_INCLUDE_PTE_SYNC #define PTE_SYNC_CURRENT(pm, ptep) \ do { \ if (PMAP_NEEDS_PTE_SYNC && \ pmap_is_current(pm)) \ PTE_SYNC(ptep); \ } while (/*CONSTCOND*/0) #else #define PTE_SYNC_CURRENT(pm, ptep) /* nothing */ #endif /* * cacheable == -1 means we must make the entry uncacheable, 1 means * cacheable; */ static __inline void pmap_set_cache_entry(pv_entry_t pv, pmap_t pm, vm_offset_t va, int cacheable) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; if (cacheable == 1) { pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode; if (l2pte_valid(pte)) { if (PV_BEEN_EXECD(pv->pv_flags)) { pmap_tlb_flushID_SE(pv->pv_pmap, pv->pv_va); } else if (PV_BEEN_REFD(pv->pv_flags)) { pmap_tlb_flushD_SE(pv->pv_pmap, pv->pv_va); } } } else { pte = *ptep &~ L2_S_CACHE_MASK; if ((va != pv->pv_va || pm != pv->pv_pmap) && l2pte_valid(pte)) { if (PV_BEEN_EXECD(pv->pv_flags)) { pmap_idcache_wbinv_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE); pmap_tlb_flushID_SE(pv->pv_pmap, pv->pv_va); } else if (PV_BEEN_REFD(pv->pv_flags)) { pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, (pv->pv_flags & PVF_WRITE) == 0); pmap_tlb_flushD_SE(pv->pv_pmap, pv->pv_va); } } } *ptep = pte; PTE_SYNC_CURRENT(pv->pv_pmap, ptep); } static void pmap_fix_cache(struct vm_page *pg, pmap_t pm, vm_offset_t va) { int pmwc = 0; int writable = 0, kwritable = 0, uwritable = 0; int entries = 0, kentries = 0, uentries = 0; struct pv_entry *pv; rw_assert(&pvh_global_lock, RA_WLOCKED); /* the cache gets written back/invalidated on context switch. * therefore, if a user page shares an entry in the same page or * with the kernel map and at least one is writable, then the * cache entry must be set write-through. */ TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { /* generate a count of the pv_entry uses */ if (pv->pv_flags & PVF_WRITE) { if (pv->pv_pmap == kernel_pmap) kwritable++; else if (pv->pv_pmap == pm) uwritable++; writable++; } if (pv->pv_pmap == kernel_pmap) kentries++; else { if (pv->pv_pmap == pm) uentries++; entries++; } } /* * check if the user duplicate mapping has * been removed. */ if ((pm != kernel_pmap) && (((uentries > 1) && uwritable) || (uwritable > 1))) pmwc = 1; TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { /* check for user uncachable conditions - order is important */ if (pm != kernel_pmap && (pv->pv_pmap == pm || pv->pv_pmap == kernel_pmap)) { if ((uentries > 1 && uwritable) || uwritable > 1) { /* user duplicate mapping */ if (pv->pv_pmap != kernel_pmap) pv->pv_flags |= PVF_MWC; if (!(pv->pv_flags & PVF_NC)) { pv->pv_flags |= PVF_NC; pmap_set_cache_entry(pv, pm, va, -1); } continue; } else /* no longer a duplicate user */ pv->pv_flags &= ~PVF_MWC; } /* * check for kernel uncachable conditions * kernel writable or kernel readable with writable user entry */ if ((kwritable && (entries || kentries > 1)) || (kwritable > 1) || ((kwritable != writable) && kentries && (pv->pv_pmap == kernel_pmap || (pv->pv_flags & PVF_WRITE) || (pv->pv_flags & PVF_MWC)))) { if (!(pv->pv_flags & PVF_NC)) { pv->pv_flags |= PVF_NC; pmap_set_cache_entry(pv, pm, va, -1); } continue; } /* kernel and user are cachable */ if ((pm == kernel_pmap) && !(pv->pv_flags & PVF_MWC) && (pv->pv_flags & PVF_NC)) { pv->pv_flags &= ~PVF_NC; if (pg->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) pmap_set_cache_entry(pv, pm, va, 1); continue; } /* user is no longer sharable and writable */ if (pm != kernel_pmap && (pv->pv_pmap == pm || pv->pv_pmap == kernel_pmap) && !pmwc && (pv->pv_flags & PVF_NC)) { pv->pv_flags &= ~(PVF_NC | PVF_MWC); if (pg->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) pmap_set_cache_entry(pv, pm, va, 1); } } if ((kwritable == 0) && (writable == 0)) { pg->md.pvh_attrs &= ~PVF_MOD; vm_page_aflag_clear(pg, PGA_WRITEABLE); return; } } /* * Modify pte bits for all ptes corresponding to the given physical address. * We use `maskbits' rather than `clearbits' because we're always passing * constants and the latter would require an extra inversion at run-time. */ static int pmap_clearbit(struct vm_page *pg, u_int maskbits) { struct l2_bucket *l2b; struct pv_entry *pv; pt_entry_t *ptep, npte, opte; pmap_t pm; vm_offset_t va; u_int oflags; int count = 0; rw_wlock(&pvh_global_lock); if (maskbits & PVF_WRITE) maskbits |= PVF_MOD; /* * Clear saved attributes (modify, reference) */ pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); if (TAILQ_EMPTY(&pg->md.pv_list)) { rw_wunlock(&pvh_global_lock); return (0); } /* * Loop over all current mappings setting/clearing as appropos */ TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { va = pv->pv_va; pm = pv->pv_pmap; oflags = pv->pv_flags; if (!(oflags & maskbits)) { if ((maskbits & PVF_WRITE) && (pv->pv_flags & PVF_NC)) { if (pg->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) { PMAP_LOCK(pm); l2b = pmap_get_l2_bucket(pm, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; *ptep |= pte_l2_s_cache_mode; PTE_SYNC(ptep); PMAP_UNLOCK(pm); } pv->pv_flags &= ~(PVF_NC | PVF_MWC); } continue; } pv->pv_flags &= ~maskbits; PMAP_LOCK(pm); l2b = pmap_get_l2_bucket(pm, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; npte = opte = *ptep; if (maskbits & (PVF_WRITE|PVF_MOD)) { if ((pv->pv_flags & PVF_NC)) { /* * Entry is not cacheable: * * Don't turn caching on again if this is a * modified emulation. This would be - * inconsitent with the settings created by + * inconsistent with the settings created by * pmap_fix_cache(). Otherwise, it's safe - * to re-enable cacheing. + * to re-enable caching. * * There's no need to call pmap_fix_cache() * here: all pages are losing their write * permission. */ if (maskbits & PVF_WRITE) { if (pg->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) npte |= pte_l2_s_cache_mode; pv->pv_flags &= ~(PVF_NC | PVF_MWC); } } else if (opte & L2_S_PROT_W) { vm_page_dirty(pg); /* * Entry is writable/cacheable: check if pmap * is current if it is flush it, otherwise it * won't be in the cache */ if (PV_BEEN_EXECD(oflags)) pmap_idcache_wbinv_range(pm, pv->pv_va, PAGE_SIZE); else if (PV_BEEN_REFD(oflags)) pmap_dcache_wb_range(pm, pv->pv_va, PAGE_SIZE, (maskbits & PVF_REF) ? TRUE : FALSE, FALSE); } /* make the pte read only */ npte &= ~L2_S_PROT_W; } if (maskbits & PVF_REF) { if ((pv->pv_flags & PVF_NC) == 0 && (maskbits & (PVF_WRITE|PVF_MOD)) == 0) { /* * Check npte here; we may have already * done the wbinv above, and the validity * of the PTE is the same for opte and * npte. */ if (npte & L2_S_PROT_W) { if (PV_BEEN_EXECD(oflags)) pmap_idcache_wbinv_range(pm, pv->pv_va, PAGE_SIZE); else if (PV_BEEN_REFD(oflags)) pmap_dcache_wb_range(pm, pv->pv_va, PAGE_SIZE, TRUE, FALSE); } else if ((npte & L2_TYPE_MASK) != L2_TYPE_INV) { /* XXXJRT need idcache_inv_range */ if (PV_BEEN_EXECD(oflags)) pmap_idcache_wbinv_range(pm, pv->pv_va, PAGE_SIZE); else if (PV_BEEN_REFD(oflags)) pmap_dcache_wb_range(pm, pv->pv_va, PAGE_SIZE, TRUE, TRUE); } } /* * Make the PTE invalid so that we will take a * page fault the next time the mapping is * referenced. */ npte &= ~L2_TYPE_MASK; npte |= L2_TYPE_INV; } if (npte != opte) { count++; *ptep = npte; PTE_SYNC(ptep); /* Flush the TLB entry if a current pmap. */ if (PV_BEEN_EXECD(oflags)) pmap_tlb_flushID_SE(pm, pv->pv_va); else if (PV_BEEN_REFD(oflags)) pmap_tlb_flushD_SE(pm, pv->pv_va); } PMAP_UNLOCK(pm); } if (maskbits & PVF_WRITE) vm_page_aflag_clear(pg, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); return (count); } /* * main pv_entry manipulation functions: * pmap_enter_pv: enter a mapping onto a vm_page list * pmap_remove_pv: remove a mappiing from a vm_page list * * NOTE: pmap_enter_pv expects to lock the pvh itself * pmap_remove_pv expects the caller to lock the pvh before calling */ /* * pmap_enter_pv: enter a mapping onto a vm_page's PV list * * => caller should hold the proper lock on pvh_global_lock * => caller should have pmap locked * => we will (someday) gain the lock on the vm_page's PV list * => caller should adjust ptp's wire_count before calling * => caller should not adjust pmap's wire_count */ static void pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, vm_offset_t va, u_int flags) { rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_ASSERT_LOCKED(pm); if (pg->md.pv_kva != 0) { pve->pv_pmap = kernel_pmap; pve->pv_va = pg->md.pv_kva; pve->pv_flags = PVF_WRITE | PVF_UNMAN; if (pm != kernel_pmap) PMAP_LOCK(kernel_pmap); TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); TAILQ_INSERT_HEAD(&kernel_pmap->pm_pvlist, pve, pv_plist); if (pm != kernel_pmap) PMAP_UNLOCK(kernel_pmap); pg->md.pv_kva = 0; if ((pve = pmap_get_pv_entry()) == NULL) panic("pmap_kenter_pv: no pv entries"); } pve->pv_pmap = pm; pve->pv_va = va; pve->pv_flags = flags; TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD); if (pve->pv_flags & PVF_WIRED) ++pm->pm_stats.wired_count; vm_page_aflag_set(pg, PGA_REFERENCED); } /* * * pmap_find_pv: Find a pv entry * * => caller should hold lock on vm_page */ static PMAP_INLINE struct pv_entry * pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) { struct pv_entry *pv; rw_assert(&pvh_global_lock, RA_WLOCKED); TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) if (pm == pv->pv_pmap && va == pv->pv_va) break; return (pv); } /* * vector_page_setprot: * * Manipulate the protection of the vector page. */ void vector_page_setprot(int prot) { struct l2_bucket *l2b; pt_entry_t *ptep; l2b = pmap_get_l2_bucket(kernel_pmap, vector_page); ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; *ptep = (*ptep & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); PTE_SYNC(ptep); cpu_tlb_flushD_SE(vector_page); cpu_cpwait(); } /* * pmap_remove_pv: try to remove a mapping from a pv_list * * => caller should hold proper lock on pmap_main_lock * => pmap should be locked * => caller should hold lock on vm_page [so that attrs can be adjusted] * => caller should adjust ptp's wire_count and free PTP if needed * => caller should NOT adjust pmap's wire_count * => we return the removed pve */ static void pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve) { struct pv_entry *pv; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_ASSERT_LOCKED(pm); TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist); if (pve->pv_flags & PVF_WIRED) --pm->pm_stats.wired_count; if (pg->md.pvh_attrs & PVF_MOD) vm_page_dirty(pg); if (TAILQ_FIRST(&pg->md.pv_list) == NULL) pg->md.pvh_attrs &= ~PVF_REF; else vm_page_aflag_set(pg, PGA_REFERENCED); if ((pve->pv_flags & PVF_NC) && ((pm == kernel_pmap) || (pve->pv_flags & PVF_WRITE) || !(pve->pv_flags & PVF_MWC))) pmap_fix_cache(pg, pm, 0); else if (pve->pv_flags & PVF_WRITE) { TAILQ_FOREACH(pve, &pg->md.pv_list, pv_list) if (pve->pv_flags & PVF_WRITE) break; if (!pve) { pg->md.pvh_attrs &= ~PVF_MOD; vm_page_aflag_clear(pg, PGA_WRITEABLE); } } pv = TAILQ_FIRST(&pg->md.pv_list); if (pv != NULL && (pv->pv_flags & PVF_UNMAN) && TAILQ_NEXT(pv, pv_list) == NULL) { pm = kernel_pmap; pg->md.pv_kva = pv->pv_va; /* a recursive pmap_nuke_pv */ TAILQ_REMOVE(&pg->md.pv_list, pv, pv_list); TAILQ_REMOVE(&pm->pm_pvlist, pv, pv_plist); if (pv->pv_flags & PVF_WIRED) --pm->pm_stats.wired_count; pg->md.pvh_attrs &= ~PVF_REF; pg->md.pvh_attrs &= ~PVF_MOD; vm_page_aflag_clear(pg, PGA_WRITEABLE); pmap_free_pv_entry(pv); } } static struct pv_entry * pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) { struct pv_entry *pve; rw_assert(&pvh_global_lock, RA_WLOCKED); pve = TAILQ_FIRST(&pg->md.pv_list); while (pve) { if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ pmap_nuke_pv(pg, pm, pve); break; } pve = TAILQ_NEXT(pve, pv_list); } if (pve == NULL && pg->md.pv_kva == va) pg->md.pv_kva = 0; return(pve); /* return removed pve */ } /* * * pmap_modify_pv: Update pv flags * * => caller should hold lock on vm_page [so that attrs can be adjusted] * => caller should NOT adjust pmap's wire_count * => we return the old flags * * Modify a physical-virtual mapping in the pv table */ static u_int pmap_modify_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va, u_int clr_mask, u_int set_mask) { struct pv_entry *npv; u_int flags, oflags; PMAP_ASSERT_LOCKED(pm); rw_assert(&pvh_global_lock, RA_WLOCKED); if ((npv = pmap_find_pv(pg, pm, va)) == NULL) return (0); /* * There is at least one VA mapping this page. */ if (clr_mask & (PVF_REF | PVF_MOD)) pg->md.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); oflags = npv->pv_flags; npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; if ((flags ^ oflags) & PVF_WIRED) { if (flags & PVF_WIRED) ++pm->pm_stats.wired_count; else --pm->pm_stats.wired_count; } if ((flags ^ oflags) & PVF_WRITE) pmap_fix_cache(pg, pm, 0); return (oflags); } /* Function to set the debug level of the pmap code */ #ifdef PMAP_DEBUG void pmap_debug(int level) { pmap_debug_level = level; dprintf("pmap_debug: level=%d\n", pmap_debug_level); } #endif /* PMAP_DEBUG */ void pmap_pinit0(struct pmap *pmap) { PDEBUG(1, printf("pmap_pinit0: pmap = %08x\n", (u_int32_t) pmap)); bcopy(kernel_pmap, pmap, sizeof(*pmap)); bzero(&pmap->pm_mtx, sizeof(pmap->pm_mtx)); PMAP_LOCK_INIT(pmap); } /* * Initialize a vm_page's machine-dependent fields. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.pv_list); m->md.pv_memattr = VM_MEMATTR_DEFAULT; } /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ void pmap_init(void) { int shpgperproc = PMAP_SHPGPERPROC; l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); /* * Initialize the PV entry allocator. */ pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count; uma_zone_reserve_kva(pvzone, pv_entry_max); pv_entry_high_water = 9 * (pv_entry_max / 10); /* * Now it is safe to enable pv_table recording. */ PDEBUG(1, printf("pmap_init: done!\n")); } int pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user) { struct l2_dtable *l2; struct l2_bucket *l2b; pd_entry_t *pl1pd, l1pd; pt_entry_t *ptep, pte; vm_paddr_t pa; u_int l1idx; int rv = 0; l1idx = L1_IDX(va); rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); /* * If there is no l2_dtable for this address, then the process * has no business accessing it. * * Note: This will catch userland processes trying to access * kernel addresses. */ l2 = pm->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL) goto out; /* * Likewise if there is no L2 descriptor table */ l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; if (l2b->l2b_kva == NULL) goto out; /* * Check the PTE itself. */ ptep = &l2b->l2b_kva[l2pte_index(va)]; pte = *ptep; if (pte == 0) goto out; /* * Catch a userland access to the vector page mapped at 0x0 */ if (user && (pte & L2_S_PROT_U) == 0) goto out; if (va == vector_page) goto out; pa = l2pte_pa(pte); if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) { /* * This looks like a good candidate for "page modified" * emulation... */ struct pv_entry *pv; struct vm_page *pg; /* Extract the physical address of the page */ if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { goto out; } /* Get the current flags for this page. */ pv = pmap_find_pv(pg, pm, va); if (pv == NULL) { goto out; } /* * Do the flags say this page is writable? If not then it * is a genuine write fault. If yes then the write fault is * our fault as we did not reflect the write access in the * PTE. Now we know a write has occurred we can correct this * and also set the modified bit */ if ((pv->pv_flags & PVF_WRITE) == 0) { goto out; } pg->md.pvh_attrs |= PVF_REF | PVF_MOD; vm_page_dirty(pg); pv->pv_flags |= PVF_REF | PVF_MOD; /* * Re-enable write permissions for the page. No need to call * pmap_fix_cache(), since this is just a * modified-emulation fault, and the PVF_WRITE bit isn't * changing. We've already set the cacheable bits based on * the assumption that we can write to this page. */ *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; PTE_SYNC(ptep); rv = 1; } else if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { /* * This looks like a good candidate for "page referenced" * emulation. */ struct pv_entry *pv; struct vm_page *pg; /* Extract the physical address of the page */ if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) goto out; /* Get the current flags for this page. */ pv = pmap_find_pv(pg, pm, va); if (pv == NULL) goto out; pg->md.pvh_attrs |= PVF_REF; pv->pv_flags |= PVF_REF; *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; PTE_SYNC(ptep); rv = 1; } /* * We know there is a valid mapping here, so simply * fix up the L1 if necessary. */ pl1pd = &pm->pm_l1->l1_kva[l1idx]; l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; if (*pl1pd != l1pd) { *pl1pd = l1pd; PTE_SYNC(pl1pd); rv = 1; } #ifdef DEBUG /* * If 'rv == 0' at this point, it generally indicates that there is a * stale TLB entry for the faulting address. This happens when two or * more processes are sharing an L1. Since we don't flush the TLB on * a context switch between such processes, we can take domain faults * for mappings which exist at the same VA in both processes. EVEN IF * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for * example. * * This is extremely likely to happen if pmap_enter() updated the L1 * entry for a recently entered mapping. In this case, the TLB is * flushed for the new mapping, but there may still be TLB entries for * other mappings belonging to other processes in the 1MB range * covered by the L1 entry. * * Since 'rv == 0', we know that the L1 already contains the correct * value, so the fault must be due to a stale TLB entry. * * Since we always need to flush the TLB anyway in the case where we * fixed up the L1, or frobbed the L2 PTE, we effectively deal with * stale TLB entries dynamically. * * However, the above condition can ONLY happen if the current L1 is * being shared. If it happens when the L1 is unshared, it indicates * that other parts of the pmap are not doing their job WRT managing * the TLB. */ if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", pm, (u_long)va, ftype); printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", l2, l2b, ptep, pl1pd); printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", pte, l1pd, last_fault_code); #ifdef DDB Debugger(); #endif } #endif cpu_tlb_flushID_SE(va); cpu_cpwait(); rv = 1; out: rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); return (rv); } void pmap_postinit(void) { struct l2_bucket *l2b; struct l1_ttable *l1; pd_entry_t *pl1pt; pt_entry_t *ptep, pte; vm_offset_t va, eva; u_int loop, needed; needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); needed -= 1; l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK); for (loop = 0; loop < needed; loop++, l1++) { /* Allocate a L1 page table */ va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, M_VMPMAP, 0, 0x0, 0xffffffff, L1_TABLE_SIZE, 0); if (va == 0) panic("Cannot allocate L1 KVM"); eva = va + L1_TABLE_SIZE; pl1pt = (pd_entry_t *)va; while (va < eva) { l2b = pmap_get_l2_bucket(kernel_pmap, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; pte = *ptep; pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; *ptep = pte; PTE_SYNC(ptep); cpu_tlb_flushD_SE(va); va += PAGE_SIZE; } pmap_init_l1(l1, pl1pt); } #ifdef DEBUG printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", needed); #endif } /* * This is used to stuff certain critical values into the PCB where they * can be accessed quickly from cpu_switch() et al. */ void pmap_set_pcb_pagedir(pmap_t pm, struct pcb *pcb) { struct l2_bucket *l2b; pcb->pcb_pagedir = pm->pm_l1->l1_physaddr; pcb->pcb_dacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | (DOMAIN_CLIENT << (pm->pm_domain * 2)); if (vector_page < KERNBASE) { pcb->pcb_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; l2b = pmap_get_l2_bucket(pm, vector_page); pcb->pcb_l1vec = l2b->l2b_phys | L1_C_PROTO | L1_C_DOM(pm->pm_domain) | L1_C_DOM(PMAP_DOMAIN_KERNEL); } else pcb->pcb_pl1vec = NULL; } void pmap_activate(struct thread *td) { pmap_t pm; struct pcb *pcb; pm = vmspace_pmap(td->td_proc->p_vmspace); pcb = td->td_pcb; critical_enter(); pmap_set_pcb_pagedir(pm, pcb); if (td == curthread) { u_int cur_dacr, cur_ttb; __asm __volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(cur_ttb)); __asm __volatile("mrc p15, 0, %0, c3, c0, 0" : "=r"(cur_dacr)); cur_ttb &= ~(L1_TABLE_SIZE - 1); if (cur_ttb == (u_int)pcb->pcb_pagedir && cur_dacr == pcb->pcb_dacr) { /* * No need to switch address spaces. */ critical_exit(); return; } /* * We MUST, I repeat, MUST fix up the L1 entry corresponding * to 'vector_page' in the incoming L1 table before switching * to it otherwise subsequent interrupts/exceptions (including * domain faults!) will jump into hyperspace. */ if (pcb->pcb_pl1vec) { *pcb->pcb_pl1vec = pcb->pcb_l1vec; /* * Don't need to PTE_SYNC() at this point since * cpu_setttb() is about to flush both the cache * and the TLB. */ } cpu_domains(pcb->pcb_dacr); cpu_setttb(pcb->pcb_pagedir); } critical_exit(); } static int pmap_set_pt_cache_mode(pd_entry_t *kl1, vm_offset_t va) { pd_entry_t *pdep, pde; pt_entry_t *ptep, pte; vm_offset_t pa; int rv = 0; /* * Make sure the descriptor itself has the correct cache mode */ pdep = &kl1[L1_IDX(va)]; pde = *pdep; if (l1pte_section_p(pde)) { if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { *pdep = (pde & ~L1_S_CACHE_MASK) | pte_l1_s_cache_mode_pt; PTE_SYNC(pdep); cpu_dcache_wbinv_range((vm_offset_t)pdep, sizeof(*pdep)); cpu_l2cache_wbinv_range((vm_offset_t)pdep, sizeof(*pdep)); rv = 1; } } else { pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); ptep = (pt_entry_t *)kernel_pt_lookup(pa); if (ptep == NULL) panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); ptep = &ptep[l2pte_index(va)]; pte = *ptep; if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; PTE_SYNC(ptep); cpu_dcache_wbinv_range((vm_offset_t)ptep, sizeof(*ptep)); cpu_l2cache_wbinv_range((vm_offset_t)ptep, sizeof(*ptep)); rv = 1; } } return (rv); } static void pmap_alloc_specials(vm_offset_t *availp, int pages, vm_offset_t *vap, pt_entry_t **ptep) { vm_offset_t va = *availp; struct l2_bucket *l2b; if (ptep) { l2b = pmap_get_l2_bucket(kernel_pmap, va); if (l2b == NULL) panic("pmap_alloc_specials: no l2b for 0x%x", va); *ptep = &l2b->l2b_kva[l2pte_index(va)]; } *vap = va; *availp = va + (PAGE_SIZE * pages); } /* * Bootstrap the system enough to run with virtual memory. * * On the arm this is called after mapping has already been enabled * and just syncs the pmap module with what has already been done. * [We can't call it easily with mapping off since the kernel is not * mapped with PA == VA, hence we would have to relocate every address * from the linked base (virtual) address "KERNBASE" to the actual * (physical) address starting relative to 0] */ #define PMAP_STATIC_L2_SIZE 16 void pmap_bootstrap(vm_offset_t firstaddr, struct pv_addr *l1pt) { static struct l1_ttable static_l1; static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; struct l1_ttable *l1 = &static_l1; struct l2_dtable *l2; struct l2_bucket *l2b; pd_entry_t pde; pd_entry_t *kernel_l1pt = (pd_entry_t *)l1pt->pv_va; pt_entry_t *ptep; pt_entry_t *qmap_pte; vm_paddr_t pa; vm_offset_t va; vm_size_t size; int l1idx, l2idx, l2next = 0; PDEBUG(1, printf("firstaddr = %08x, lastaddr = %08x\n", firstaddr, vm_max_kernel_address)); virtual_avail = firstaddr; kernel_pmap->pm_l1 = l1; kernel_l1pa = l1pt->pv_pa; /* * Scan the L1 translation table created by initarm() and create * the required metadata for all valid mappings found in it. */ for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { pde = kernel_l1pt[l1idx]; /* * We're only interested in Coarse mappings. * pmap_extract() can deal with section mappings without * recourse to checking L2 metadata. */ if ((pde & L1_TYPE_MASK) != L1_TYPE_C) continue; /* * Lookup the KVA of this L2 descriptor table */ pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); ptep = (pt_entry_t *)kernel_pt_lookup(pa); if (ptep == NULL) { panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", (u_int)l1idx << L1_S_SHIFT, (long unsigned int)pa); } /* * Fetch the associated L2 metadata structure. * Allocate a new one if necessary. */ if ((l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]) == NULL) { if (l2next == PMAP_STATIC_L2_SIZE) panic("pmap_bootstrap: out of static L2s"); kernel_pmap->pm_l2[L2_IDX(l1idx)] = l2 = &static_l2[l2next++]; } /* * One more L1 slot tracked... */ l2->l2_occupancy++; /* * Fill in the details of the L2 descriptor in the * appropriate bucket. */ l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; l2b->l2b_kva = ptep; l2b->l2b_phys = pa; l2b->l2b_l1idx = l1idx; /* * Establish an initial occupancy count for this descriptor */ for (l2idx = 0; l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); l2idx++) { if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { l2b->l2b_occupancy++; } } /* * Make sure the descriptor itself has the correct cache mode. * If not, fix it, but whine about the problem. Port-meisters * should consider this a clue to fix up their initarm() * function. :) */ if (pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)ptep)) { printf("pmap_bootstrap: WARNING! wrong cache mode for " "L2 pte @ %p\n", ptep); } } /* * Ensure the primary (kernel) L1 has the correct cache mode for * a page table. Bitch if it is not correctly set. */ for (va = (vm_offset_t)kernel_l1pt; va < ((vm_offset_t)kernel_l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { if (pmap_set_pt_cache_mode(kernel_l1pt, va)) printf("pmap_bootstrap: WARNING! wrong cache mode for " "primary L1 @ 0x%x\n", va); } cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); cpu_tlb_flushID(); cpu_cpwait(); PMAP_LOCK_INIT(kernel_pmap); CPU_FILL(&kernel_pmap->pm_active); kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; TAILQ_INIT(&kernel_pmap->pm_pvlist); /* * Initialize the global pv list lock. */ rw_init_flags(&pvh_global_lock, "pmap pv global", RW_RECURSE); /* * Reserve some special page table entries/VA space for temporary * mapping of pages. */ pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte); pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)csrc_pte); pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte); pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)cdst_pte); pmap_alloc_specials(&virtual_avail, 1, &qmap_addr, &qmap_pte); pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)qmap_pte); size = ((vm_max_kernel_address - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; pmap_alloc_specials(&virtual_avail, round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, &pmap_kernel_l2ptp_kva, NULL); size = howmany(size, L2_BUCKET_SIZE); pmap_alloc_specials(&virtual_avail, round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, &pmap_kernel_l2dtable_kva, NULL); pmap_alloc_specials(&virtual_avail, 1, (vm_offset_t*)&_tmppt, NULL); pmap_alloc_specials(&virtual_avail, MAXDUMPPGS, (vm_offset_t *)&crashdumpmap, NULL); SLIST_INIT(&l1_list); TAILQ_INIT(&l1_lru_list); mtx_init(&l1_lru_lock, "l1 list lock", NULL, MTX_DEF); pmap_init_l1(l1, kernel_l1pt); cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); virtual_avail = round_page(virtual_avail); virtual_end = vm_max_kernel_address; kernel_vm_end = pmap_curmaxkvaddr; mtx_init(&cmtx, "TMP mappings mtx", NULL, MTX_DEF); mtx_init(&qmap_mtx, "quick mapping mtx", NULL, MTX_DEF); pmap_set_pcb_pagedir(kernel_pmap, thread0.td_pcb); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pmap) { struct pcb *pcb; pmap_idcache_wbinv_all(pmap); cpu_l2cache_wbinv_all(); pmap_tlb_flushID(pmap); cpu_cpwait(); if (vector_page < KERNBASE) { struct pcb *curpcb = PCPU_GET(curpcb); pcb = thread0.td_pcb; if (pmap_is_current(pmap)) { /* * Frob the L1 entry corresponding to the vector * page so that it contains the kernel pmap's domain * number. This will ensure pmap_remove() does not * pull the current vector page out from under us. */ critical_enter(); *pcb->pcb_pl1vec = pcb->pcb_l1vec; cpu_domains(pcb->pcb_dacr); cpu_setttb(pcb->pcb_pagedir); critical_exit(); } pmap_remove(pmap, vector_page, vector_page + PAGE_SIZE); /* * Make sure cpu_switch(), et al, DTRT. This is safe to do * since this process has no remaining mappings of its own. */ curpcb->pcb_pl1vec = pcb->pcb_pl1vec; curpcb->pcb_l1vec = pcb->pcb_l1vec; curpcb->pcb_dacr = pcb->pcb_dacr; curpcb->pcb_pagedir = pcb->pcb_pagedir; } pmap_free_l1(pmap); dprintf("pmap_release()\n"); } /* * Helper function for pmap_grow_l2_bucket() */ static __inline int pmap_grow_map(vm_offset_t va, pt_entry_t cache_mode, vm_paddr_t *pap) { struct l2_bucket *l2b; pt_entry_t *ptep; vm_paddr_t pa; struct vm_page *pg; pg = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (pg == NULL) return (1); pa = VM_PAGE_TO_PHYS(pg); if (pap) *pap = pa; l2b = pmap_get_l2_bucket(kernel_pmap, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; *ptep = L2_S_PROTO | pa | cache_mode | L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); PTE_SYNC(ptep); return (0); } /* * This is the same as pmap_alloc_l2_bucket(), except that it is only * used by pmap_growkernel(). */ static __inline struct l2_bucket * pmap_grow_l2_bucket(pmap_t pm, vm_offset_t va) { struct l2_dtable *l2; struct l2_bucket *l2b; struct l1_ttable *l1; pd_entry_t *pl1pd; u_short l1idx; vm_offset_t nva; l1idx = L1_IDX(va); if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { /* * No mapping at this address, as there is * no entry in the L1 table. * Need to allocate a new l2_dtable. */ nva = pmap_kernel_l2dtable_kva; if ((nva & PAGE_MASK) == 0) { /* * Need to allocate a backing page */ if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) return (NULL); } l2 = (struct l2_dtable *)nva; nva += sizeof(struct l2_dtable); if ((nva & PAGE_MASK) < (pmap_kernel_l2dtable_kva & PAGE_MASK)) { /* * The new l2_dtable straddles a page boundary. * Map in another page to cover it. */ if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) return (NULL); } pmap_kernel_l2dtable_kva = nva; /* * Link it into the parent pmap */ pm->pm_l2[L2_IDX(l1idx)] = l2; memset(l2, 0, sizeof(*l2)); } l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; /* * Fetch pointer to the L2 page table associated with the address. */ if (l2b->l2b_kva == NULL) { pt_entry_t *ptep; /* * No L2 page table has been allocated. Chances are, this * is because we just allocated the l2_dtable, above. */ nva = pmap_kernel_l2ptp_kva; ptep = (pt_entry_t *)nva; if ((nva & PAGE_MASK) == 0) { /* * Need to allocate a backing page */ if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt, &pmap_kernel_l2ptp_phys)) return (NULL); PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); } memset(ptep, 0, L2_TABLE_SIZE_REAL); l2->l2_occupancy++; l2b->l2b_kva = ptep; l2b->l2b_l1idx = l1idx; l2b->l2b_phys = pmap_kernel_l2ptp_phys; pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; } /* Distribute new L1 entry to all other L1s */ SLIST_FOREACH(l1, &l1_list, l1_link) { pl1pd = &l1->l1_kva[L1_IDX(va)]; *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; PTE_SYNC(pl1pd); } return (l2b); } /* * grow the number of kernel page table entries, if needed */ void pmap_growkernel(vm_offset_t addr) { pmap_t kpm = kernel_pmap; if (addr <= pmap_curmaxkvaddr) return; /* we are OK */ /* * whoops! we need to add kernel PTPs */ /* Map 1MB at a time */ for (; pmap_curmaxkvaddr < addr; pmap_curmaxkvaddr += L1_S_SIZE) pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); /* * flush out the cache, expensive but growkernel will happen so * rarely */ cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); cpu_tlb_flushD(); cpu_cpwait(); kernel_vm_end = pmap_curmaxkvaddr; } /* * Remove all pages from specified address space * this aids process exit speeds. Also, this code * is special cased for current process only, but * can have the more generic (and slightly slower) * mode enabled. This is much faster than pmap_remove * in the case of running down an entire address space. */ void pmap_remove_pages(pmap_t pmap) { struct pv_entry *pv, *npv; struct l2_bucket *l2b = NULL; vm_page_t m; pt_entry_t *pt; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { if (pv->pv_flags & PVF_WIRED || pv->pv_flags & PVF_UNMAN) { /* Cannot remove wired or unmanaged pages now. */ npv = TAILQ_NEXT(pv, pv_plist); continue; } pmap->pm_stats.resident_count--; l2b = pmap_get_l2_bucket(pmap, pv->pv_va); KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; m = PHYS_TO_VM_PAGE(*pt & L2_S_FRAME); KASSERT((vm_offset_t)m >= KERNBASE, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt)); *pt = 0; PTE_SYNC(pt); npv = TAILQ_NEXT(pv, pv_plist); pmap_nuke_pv(m, pmap, pv); if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); pmap_free_pv_entry(pv); pmap_free_l2_bucket(pmap, l2b, 1); } rw_wunlock(&pvh_global_lock); cpu_tlb_flushID(); cpu_cpwait(); PMAP_UNLOCK(pmap); } /*************************************************** * Low level mapping routines..... ***************************************************/ #ifdef ARM_HAVE_SUPERSECTIONS /* Map a super section into the KVA. */ void pmap_kenter_supersection(vm_offset_t va, uint64_t pa, int flags) { pd_entry_t pd = L1_S_PROTO | L1_S_SUPERSEC | (pa & L1_SUP_FRAME) | (((pa >> 32) & 0xf) << 20) | L1_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); struct l1_ttable *l1; vm_offset_t va0, va_end; KASSERT(((va | pa) & L1_SUP_OFFSET) == 0, ("Not a valid super section mapping")); if (flags & SECTION_CACHE) pd |= pte_l1_s_cache_mode; else if (flags & SECTION_PT) pd |= pte_l1_s_cache_mode_pt; va0 = va & L1_SUP_FRAME; va_end = va + L1_SUP_SIZE; SLIST_FOREACH(l1, &l1_list, l1_link) { va = va0; for (; va < va_end; va += L1_S_SIZE) { l1->l1_kva[L1_IDX(va)] = pd; PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); } } } #endif /* Map a section into the KVA. */ void pmap_kenter_section(vm_offset_t va, vm_offset_t pa, int flags) { pd_entry_t pd = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); struct l1_ttable *l1; KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("Not a valid section mapping")); if (flags & SECTION_CACHE) pd |= pte_l1_s_cache_mode; else if (flags & SECTION_PT) pd |= pte_l1_s_cache_mode_pt; SLIST_FOREACH(l1, &l1_list, l1_link) { l1->l1_kva[L1_IDX(va)] = pd; PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); } } /* * Make a temporary mapping for a physical address. This is only intended * to be used for panic dumps. */ void * pmap_kenter_temporary(vm_paddr_t pa, int i) { vm_offset_t va; va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); pmap_kenter(va, pa); return ((void *)crashdumpmap); } /* * add a wired page to the kva * note that in order for the mapping to take effect -- you * should do a invltlb after doing the pmap_kenter... */ static PMAP_INLINE void pmap_kenter_internal(vm_offset_t va, vm_offset_t pa, int flags) { struct l2_bucket *l2b; pt_entry_t *pte; pt_entry_t opte; struct pv_entry *pve; vm_page_t m; PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n", (uint32_t) va, (uint32_t) pa)); l2b = pmap_get_l2_bucket(kernel_pmap, va); if (l2b == NULL) l2b = pmap_grow_l2_bucket(kernel_pmap, va); KASSERT(l2b != NULL, ("No L2 Bucket")); pte = &l2b->l2b_kva[l2pte_index(va)]; opte = *pte; PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", (uint32_t) pte, opte, *pte)); if (l2pte_valid(opte)) { pmap_kremove(va); } else { if (opte == 0) l2b->l2b_occupancy++; } *pte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); if (flags & KENTER_CACHE) *pte |= pte_l2_s_cache_mode; if (flags & KENTER_USER) *pte |= L2_S_PROT_U; PTE_SYNC(pte); /* * A kernel mapping may not be the page's only mapping, so create a PV * entry to ensure proper caching. * * The existence test for the pvzone is used to delay the recording of * kernel mappings until the VM system is fully initialized. * * This expects the physical memory to have a vm_page_array entry. */ if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa)) != NULL) { rw_wlock(&pvh_global_lock); if (!TAILQ_EMPTY(&m->md.pv_list) || m->md.pv_kva != 0) { if ((pve = pmap_get_pv_entry()) == NULL) panic("pmap_kenter_internal: no pv entries"); PMAP_LOCK(kernel_pmap); pmap_enter_pv(m, pve, kernel_pmap, va, PVF_WRITE | PVF_UNMAN); pmap_fix_cache(m, kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } else { m->md.pv_kva = va; } rw_wunlock(&pvh_global_lock); } } void pmap_kenter(vm_offset_t va, vm_paddr_t pa) { pmap_kenter_internal(va, pa, KENTER_CACHE); } void pmap_kenter_nocache(vm_offset_t va, vm_paddr_t pa) { pmap_kenter_internal(va, pa, 0); } void pmap_kenter_device(vm_offset_t va, vm_size_t size, vm_paddr_t pa) { vm_offset_t sva; KASSERT((size & PAGE_MASK) == 0, ("%s: device mapping not page-sized", __func__)); sva = va; while (size != 0) { pmap_kenter_internal(va, pa, 0); va += PAGE_SIZE; pa += PAGE_SIZE; size -= PAGE_SIZE; } } void pmap_kremove_device(vm_offset_t va, vm_size_t size) { vm_offset_t sva; KASSERT((size & PAGE_MASK) == 0, ("%s: device mapping not page-sized", __func__)); sva = va; while (size != 0) { pmap_kremove(va); va += PAGE_SIZE; size -= PAGE_SIZE; } } void pmap_kenter_user(vm_offset_t va, vm_paddr_t pa) { pmap_kenter_internal(va, pa, KENTER_CACHE|KENTER_USER); /* * Call pmap_fault_fixup now, to make sure we'll have no exception * at the first use of the new address, or bad things will happen, * as we use one of these addresses in the exception handlers. */ pmap_fault_fixup(kernel_pmap, va, VM_PROT_READ|VM_PROT_WRITE, 1); } vm_paddr_t pmap_kextract(vm_offset_t va) { return (pmap_extract_locked(kernel_pmap, va)); } /* * remove a page from the kernel pagetables */ void pmap_kremove(vm_offset_t va) { struct l2_bucket *l2b; pt_entry_t *pte, opte; struct pv_entry *pve; vm_page_t m; vm_offset_t pa; l2b = pmap_get_l2_bucket(kernel_pmap, va); if (!l2b) return; KASSERT(l2b != NULL, ("No L2 Bucket")); pte = &l2b->l2b_kva[l2pte_index(va)]; opte = *pte; if (l2pte_valid(opte)) { /* pa = vtophs(va) taken from pmap_extract() */ if ((opte & L2_TYPE_MASK) == L2_TYPE_L) pa = (opte & L2_L_FRAME) | (va & L2_L_OFFSET); else pa = (opte & L2_S_FRAME) | (va & L2_S_OFFSET); /* note: should never have to remove an allocation * before the pvzone is initialized. */ rw_wlock(&pvh_global_lock); PMAP_LOCK(kernel_pmap); if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa)) && (pve = pmap_remove_pv(m, kernel_pmap, va))) pmap_free_pv_entry(pve); PMAP_UNLOCK(kernel_pmap); rw_wunlock(&pvh_global_lock); va = va & ~PAGE_MASK; cpu_dcache_wbinv_range(va, PAGE_SIZE); cpu_l2cache_wbinv_range(va, PAGE_SIZE); cpu_tlb_flushD_SE(va); cpu_cpwait(); *pte = 0; } } /* * Used to map a range of physical addresses into kernel * virtual address space. * * The value passed in '*virt' is a suggested virtual address for * the mapping. Architectures which can support a direct-mapped * physical to virtual region can return the appropriate address * within that region, leaving '*virt' unchanged. Other * architectures should map the pages starting at '*virt' and * update '*virt' with the first usable address after the mapped * region. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) { vm_offset_t sva = *virt; vm_offset_t va = sva; PDEBUG(1, printf("pmap_map: virt = %08x, start = %08x, end = %08x, " "prot = %d\n", (uint32_t) *virt, (uint32_t) start, (uint32_t) end, prot)); while (start < end) { pmap_kenter(va, start); va += PAGE_SIZE; start += PAGE_SIZE; } *virt = va; return (sva); } static void pmap_wb_page(vm_page_t m) { struct pv_entry *pv; TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, FALSE, (pv->pv_flags & PVF_WRITE) == 0); } static void pmap_inv_page(vm_page_t m) { struct pv_entry *pv; TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, TRUE); } /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. */ void pmap_qenter(vm_offset_t va, vm_page_t *m, int count) { int i; for (i = 0; i < count; i++) { pmap_wb_page(m[i]); pmap_kenter_internal(va, VM_PAGE_TO_PHYS(m[i]), KENTER_CACHE); va += PAGE_SIZE; } } /* * this routine jerks page mappings from the * kernel -- it is meant only for temporary mappings. */ void pmap_qremove(vm_offset_t va, int count) { vm_paddr_t pa; int i; for (i = 0; i < count; i++) { pa = vtophys(va); if (pa) { pmap_inv_page(PHYS_TO_VM_PAGE(pa)); pmap_kremove(va); } va += PAGE_SIZE; } } /* * pmap_object_init_pt preloads the ptes for a given object * into the specified pmap. This eliminates the blast of soft * faults on process startup and immediately after an mmap. */ void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is elgible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { pd_entry_t *pde; pt_entry_t *pte; if (!pmap_get_pde_pte(pmap, addr, &pde, &pte)) return (FALSE); KASSERT(pte != NULL, ("Valid mapping but no pte ?")); if (*pte == 0) return (TRUE); return (FALSE); } /* * Fetch pointers to the PDE/PTE for the given pmap/VA pair. * Returns TRUE if the mapping exists, else FALSE. * * NOTE: This function is only used by a couple of arm-specific modules. * It is not safe to take any pmap locks here, since we could be right * in the middle of debugging the pmap anyway... * * It is possible for this routine to return FALSE even though a valid * mapping does exist. This is because we don't lock, so the metadata * state may be inconsistent. * * NOTE: We can return a NULL *ptp in the case where the L1 pde is * a "section" mapping. */ boolean_t pmap_get_pde_pte(pmap_t pm, vm_offset_t va, pd_entry_t **pdp, pt_entry_t **ptp) { struct l2_dtable *l2; pd_entry_t *pl1pd, l1pd; pt_entry_t *ptep; u_short l1idx; if (pm->pm_l1 == NULL) return (FALSE); l1idx = L1_IDX(va); *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; l1pd = *pl1pd; if (l1pte_section_p(l1pd)) { *ptp = NULL; return (TRUE); } if (pm->pm_l2 == NULL) return (FALSE); l2 = pm->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { return (FALSE); } *ptp = &ptep[l2pte_index(va)]; return (TRUE); } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ void pmap_remove_all(vm_page_t m) { pv_entry_t pv; pt_entry_t *ptep; struct l2_bucket *l2b; boolean_t flush = FALSE; pmap_t curpm; int flags = 0; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_all: page %p is not managed", m)); if (TAILQ_EMPTY(&m->md.pv_list)) return; rw_wlock(&pvh_global_lock); /* * XXX This call shouldn't exist. Iterating over the PV list twice, * once in pmap_clearbit() and again below, is both unnecessary and * inefficient. The below code should itself write back the cache * entry before it destroys the mapping. */ pmap_clearbit(m, PVF_WRITE); curpm = vmspace_pmap(curproc->p_vmspace); while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { if (flush == FALSE && (pv->pv_pmap == curpm || pv->pv_pmap == kernel_pmap)) flush = TRUE; PMAP_LOCK(pv->pv_pmap); /* * Cached contents were written-back in pmap_clearbit(), * but we still have to invalidate the cache entry to make * sure stale data are not retrieved when another page will be * mapped under this virtual address. */ if (pmap_is_current(pv->pv_pmap)) { cpu_dcache_inv_range(pv->pv_va, PAGE_SIZE); if (pmap_has_valid_mapping(pv->pv_pmap, pv->pv_va)) cpu_l2cache_inv_range(pv->pv_va, PAGE_SIZE); } if (pv->pv_flags & PVF_UNMAN) { /* remove the pv entry, but do not remove the mapping * and remember this is a kernel mapped page */ m->md.pv_kva = pv->pv_va; } else { /* remove the mapping and pv entry */ l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); KASSERT(l2b != NULL, ("No l2 bucket")); ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; *ptep = 0; PTE_SYNC_CURRENT(pv->pv_pmap, ptep); pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); pv->pv_pmap->pm_stats.resident_count--; flags |= pv->pv_flags; } pmap_nuke_pv(m, pv->pv_pmap, pv); PMAP_UNLOCK(pv->pv_pmap); pmap_free_pv_entry(pv); } if (flush) { if (PV_BEEN_EXECD(flags)) pmap_tlb_flushID(curpm); else pmap_tlb_flushD(curpm); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; vm_offset_t next_bucket; u_int flags; int flush; CTR4(KTR_PMAP, "pmap_protect: pmap %p sva 0x%08x eva 0x%08x prot %x", pm, sva, eva, prot); if ((prot & VM_PROT_READ) == 0) { pmap_remove(pm, sva, eva); return; } if (prot & VM_PROT_WRITE) { /* * If this is a read->write transition, just ignore it and let * vm_fault() take care of it later. */ return; } rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); /* * OK, at this point, we know we're doing write-protect operation. * If the pmap is active, write-back the range. */ pmap_dcache_wb_range(pm, sva, eva - sva, FALSE, FALSE); flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; flags = 0; while (sva < eva) { next_bucket = L2_NEXT_BUCKET(sva); if (next_bucket > eva) next_bucket = eva; l2b = pmap_get_l2_bucket(pm, sva); if (l2b == NULL) { sva = next_bucket; continue; } ptep = &l2b->l2b_kva[l2pte_index(sva)]; while (sva < next_bucket) { if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) { struct vm_page *pg; u_int f; pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); pte &= ~L2_S_PROT_W; *ptep = pte; PTE_SYNC(ptep); if (!(pg->oflags & VPO_UNMANAGED)) { f = pmap_modify_pv(pg, pm, sva, PVF_WRITE, 0); if (f & PVF_WRITE) vm_page_dirty(pg); } else f = 0; if (flush >= 0) { flush++; flags |= f; } else if (PV_BEEN_EXECD(f)) pmap_tlb_flushID_SE(pm, sva); else if (PV_BEEN_REFD(f)) pmap_tlb_flushD_SE(pm, sva); } sva += PAGE_SIZE; ptep++; } } if (flush) { if (PV_BEEN_EXECD(flags)) pmap_tlb_flushID(pm); else if (PV_BEEN_REFD(flags)) pmap_tlb_flushD(pm); } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind __unused) { int rv; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); rv = pmap_enter_locked(pmap, va, m, prot, flags); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (rv); } /* * The pvh global and pmap locks must be held. */ static int pmap_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags) { struct l2_bucket *l2b = NULL; struct vm_page *opg; struct pv_entry *pve = NULL; pt_entry_t *ptep, npte, opte; u_int nflags; u_int oflags; vm_paddr_t pa; PMAP_ASSERT_LOCKED(pmap); rw_assert(&pvh_global_lock, RA_WLOCKED); if (va == vector_page) { pa = systempage.pv_pa; m = NULL; } else { if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); pa = VM_PAGE_TO_PHYS(m); } nflags = 0; if (prot & VM_PROT_WRITE) nflags |= PVF_WRITE; if (prot & VM_PROT_EXECUTE) nflags |= PVF_EXEC; if ((flags & PMAP_ENTER_WIRED) != 0) nflags |= PVF_WIRED; PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, " "flags = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, flags)); if (pmap == kernel_pmap) { l2b = pmap_get_l2_bucket(pmap, va); if (l2b == NULL) l2b = pmap_grow_l2_bucket(pmap, va); } else { do_l2b_alloc: l2b = pmap_alloc_l2_bucket(pmap, va); if (l2b == NULL) { if ((flags & PMAP_ENTER_NOSLEEP) == 0) { PMAP_UNLOCK(pmap); rw_wunlock(&pvh_global_lock); VM_WAIT; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); goto do_l2b_alloc; } return (KERN_RESOURCE_SHORTAGE); } } ptep = &l2b->l2b_kva[l2pte_index(va)]; opte = *ptep; npte = pa; oflags = 0; if (opte) { /* * There is already a mapping at this address. * If the physical address is different, lookup the * vm_page. */ if (l2pte_pa(opte) != pa) opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); else opg = m; } else opg = NULL; if ((prot & (VM_PROT_ALL)) || (!m || m->md.pvh_attrs & PVF_REF)) { /* * - The access type indicates that we don't need * to do referenced emulation. * OR * - The physical page has already been referenced * so no need to re-do referenced emulation here. */ npte |= L2_S_PROTO; nflags |= PVF_REF; if (m && ((prot & VM_PROT_WRITE) != 0 || (m->md.pvh_attrs & PVF_MOD))) { /* * This is a writable mapping, and the * page's mod state indicates it has * already been modified. Make it * writable from the outset. */ nflags |= PVF_MOD; if (!(m->md.pvh_attrs & PVF_MOD)) vm_page_dirty(m); } if (m && opte) vm_page_aflag_set(m, PGA_REFERENCED); } else { /* * Need to do page referenced emulation. */ npte |= L2_TYPE_INV; } if (prot & VM_PROT_WRITE) { npte |= L2_S_PROT_W; if (m != NULL && (m->oflags & VPO_UNMANAGED) == 0) vm_page_aflag_set(m, PGA_WRITEABLE); } if (m->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) npte |= pte_l2_s_cache_mode; if (m && m == opg) { /* * We're changing the attrs of an existing mapping. */ oflags = pmap_modify_pv(m, pmap, va, PVF_WRITE | PVF_EXEC | PVF_WIRED | PVF_MOD | PVF_REF, nflags); /* * We may need to flush the cache if we're * doing rw-ro... */ if (pmap_is_current(pmap) && (oflags & PVF_NC) == 0 && (opte & L2_S_PROT_W) != 0 && (prot & VM_PROT_WRITE) == 0 && (opte & L2_TYPE_MASK) != L2_TYPE_INV) { cpu_dcache_wb_range(va, PAGE_SIZE); cpu_l2cache_wb_range(va, PAGE_SIZE); } } else { /* * New mapping, or changing the backing page * of an existing mapping. */ if (opg) { /* * Replacing an existing mapping with a new one. * It is part of our managed memory so we * must remove it from the PV list */ if ((pve = pmap_remove_pv(opg, pmap, va))) { /* note for patch: the oflags/invalidation was moved * because PG_FICTITIOUS pages could free the pve */ oflags = pve->pv_flags; /* * If the old mapping was valid (ref/mod * emulation creates 'invalid' mappings * initially) then make sure to frob * the cache. */ if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) { if (PV_BEEN_EXECD(oflags)) { pmap_idcache_wbinv_range(pmap, va, PAGE_SIZE); } else if (PV_BEEN_REFD(oflags)) { pmap_dcache_wb_range(pmap, va, PAGE_SIZE, TRUE, (oflags & PVF_WRITE) == 0); } } /* free/allocate a pv_entry for UNMANAGED pages if * this physical page is not/is already mapped. */ if (m && (m->oflags & VPO_UNMANAGED) && !m->md.pv_kva && TAILQ_EMPTY(&m->md.pv_list)) { pmap_free_pv_entry(pve); pve = NULL; } } else if (m && (!(m->oflags & VPO_UNMANAGED) || m->md.pv_kva || !TAILQ_EMPTY(&m->md.pv_list))) pve = pmap_get_pv_entry(); } else if (m && (!(m->oflags & VPO_UNMANAGED) || m->md.pv_kva || !TAILQ_EMPTY(&m->md.pv_list))) pve = pmap_get_pv_entry(); if (m) { if ((m->oflags & VPO_UNMANAGED)) { if (!TAILQ_EMPTY(&m->md.pv_list) || m->md.pv_kva) { KASSERT(pve != NULL, ("No pv")); nflags |= PVF_UNMAN; pmap_enter_pv(m, pve, pmap, va, nflags); } else m->md.pv_kva = va; } else { KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, ("pmap_enter: managed mapping within the clean submap")); KASSERT(pve != NULL, ("No pv")); pmap_enter_pv(m, pve, pmap, va, nflags); } } } /* * Make sure userland mappings get the right permissions */ if (pmap != kernel_pmap && va != vector_page) { npte |= L2_S_PROT_U; } /* * Keep the stats up to date */ if (opte == 0) { l2b->l2b_occupancy++; pmap->pm_stats.resident_count++; } /* * If this is just a wiring change, the two PTEs will be * identical, so there's no need to update the page table. */ if (npte != opte) { boolean_t is_cached = pmap_is_current(pmap); *ptep = npte; if (is_cached) { /* * We only need to frob the cache/tlb if this pmap * is current */ PTE_SYNC(ptep); if (L1_IDX(va) != L1_IDX(vector_page) && l2pte_valid(npte)) { /* * This mapping is likely to be accessed as * soon as we return to userland. Fix up the * L1 entry to avoid taking another * page/domain fault. */ pd_entry_t *pl1pd, l1pd; pl1pd = &pmap->pm_l1->l1_kva[L1_IDX(va)]; l1pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO; if (*pl1pd != l1pd) { *pl1pd = l1pd; PTE_SYNC(pl1pd); } } } if (PV_BEEN_EXECD(oflags)) pmap_tlb_flushID_SE(pmap, va); else if (PV_BEEN_REFD(oflags)) pmap_tlb_flushD_SE(pmap, va); if (m) pmap_fix_cache(m, pmap, va); } return (KERN_SUCCESS); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_page_t m; vm_pindex_t diff, psize; VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); m = m_start; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { pmap_enter_locked(pmap, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP); m = TAILQ_NEXT(m, listq); } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No page table pages. * but is *MUCH* faster than pmap_enter... */ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); pmap_enter_locked(pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range * must have the wired attribute set. In contrast, invalid mappings * cannot have the wired attribute set, so they are ignored. * * XXX Wired mappings of unmanaged pages cannot be counted by this pmap * implementation. */ void pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; pv_entry_t pv; vm_offset_t next_bucket; vm_page_t m; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); while (sva < eva) { next_bucket = L2_NEXT_BUCKET(sva); if (next_bucket > eva) next_bucket = eva; l2b = pmap_get_l2_bucket(pmap, sva); if (l2b == NULL) { sva = next_bucket; continue; } for (ptep = &l2b->l2b_kva[l2pte_index(sva)]; sva < next_bucket; sva += PAGE_SIZE, ptep++) { if ((pte = *ptep) == 0 || (m = PHYS_TO_VM_PAGE(l2pte_pa(pte))) == NULL || (m->oflags & VPO_UNMANAGED) != 0) continue; pv = pmap_find_pv(m, pmap, sva); if ((pv->pv_flags & PVF_WIRED) == 0) panic("pmap_unwire: pv %p isn't wired", pv); pv->pv_flags &= ~PVF_WIRED; pmap->pm_stats.wired_count--; } } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va) { vm_paddr_t pa; PMAP_LOCK(pmap); pa = pmap_extract_locked(pmap, va); PMAP_UNLOCK(pmap); return (pa); } static vm_paddr_t pmap_extract_locked(pmap_t pmap, vm_offset_t va) { struct l2_dtable *l2; pd_entry_t l1pd; pt_entry_t *ptep, pte; vm_paddr_t pa; u_int l1idx; if (pmap != kernel_pmap) PMAP_ASSERT_LOCKED(pmap); l1idx = L1_IDX(va); l1pd = pmap->pm_l1->l1_kva[l1idx]; if (l1pte_section_p(l1pd)) { /* * These should only happen for the kernel pmap. */ KASSERT(pmap == kernel_pmap, ("unexpected section")); /* XXX: what to do about the bits > 32 ? */ if (l1pd & L1_S_SUPERSEC) pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); else pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); } else { /* * Note that we can't rely on the validity of the L1 * descriptor as an indication that a mapping exists. * We have to look it up in the L2 dtable. */ l2 = pmap->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) return (0); pte = ptep[l2pte_index(va)]; if (pte == 0) return (0); if ((pte & L2_TYPE_MASK) == L2_TYPE_L) pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); else pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); } return (pa); } /* * Atomically extract and hold the physical page with the given * pmap and virtual address pair if that mapping permits the given * protection. * */ vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) { struct l2_dtable *l2; pd_entry_t l1pd; pt_entry_t *ptep, pte; vm_paddr_t pa, paddr; vm_page_t m = NULL; u_int l1idx; l1idx = L1_IDX(va); paddr = 0; PMAP_LOCK(pmap); retry: l1pd = pmap->pm_l1->l1_kva[l1idx]; if (l1pte_section_p(l1pd)) { /* * These should only happen for kernel_pmap */ KASSERT(pmap == kernel_pmap, ("huh")); /* XXX: what to do about the bits > 32 ? */ if (l1pd & L1_S_SUPERSEC) pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); else pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) goto retry; if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } else { /* * Note that we can't rely on the validity of the L1 * descriptor as an indication that a mapping exists. * We have to look it up in the L2 dtable. */ l2 = pmap->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { PMAP_UNLOCK(pmap); return (NULL); } ptep = &ptep[l2pte_index(va)]; pte = *ptep; if (pte == 0) { PMAP_UNLOCK(pmap); return (NULL); } if (pte & L2_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { if ((pte & L2_TYPE_MASK) == L2_TYPE_L) pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); else pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) goto retry; m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } PMAP_UNLOCK(pmap); PA_UNLOCK_COND(paddr); return (m); } vm_paddr_t pmap_dump_kextract(vm_offset_t va, pt2_entry_t *pte2p) { struct l2_dtable *l2; pd_entry_t l1pd; pt_entry_t *ptep, pte; vm_paddr_t pa; u_int l1idx; l1idx = L1_IDX(va); l1pd = kernel_pmap->pm_l1->l1_kva[l1idx]; if (l1pte_section_p(l1pd)) { if (l1pd & L1_S_SUPERSEC) pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); else pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); pte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); } else { l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { pte = 0; pa = 0; goto out; } pte = ptep[l2pte_index(va)]; if (pte == 0) { pa = 0; goto out; } if ((pte & L2_TYPE_MASK) == L2_TYPE_L) pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); else pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); } out: if (pte2p != NULL) *pte2p = pte; return (pa); } /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. */ int pmap_pinit(pmap_t pmap) { PDEBUG(1, printf("pmap_pinit: pmap = %08x\n", (uint32_t) pmap)); pmap_alloc_l1(pmap); bzero(pmap->pm_l2, sizeof(pmap->pm_l2)); CPU_ZERO(&pmap->pm_active); TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); pmap->pm_stats.resident_count = 1; if (vector_page < KERNBASE) { pmap_enter(pmap, vector_page, PHYS_TO_VM_PAGE(systempage.pv_pa), VM_PROT_READ, PMAP_ENTER_WIRED | VM_PROT_READ, 0); } return (1); } /*************************************************** * page management routines. ***************************************************/ static void pmap_free_pv_entry(pv_entry_t pv) { pv_entry_count--; uma_zfree(pvzone, pv); } /* * get a new pv_entry, allocating a block from the system * when needed. * the memory allocation is performed bypassing the malloc code * because of the possibility of allocations at interrupt time. */ static pv_entry_t pmap_get_pv_entry(void) { pv_entry_t ret_value; pv_entry_count++; if (pv_entry_count > pv_entry_high_water) pagedaemon_wakeup(); ret_value = uma_zalloc(pvzone, M_NOWAIT); return ret_value; } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 void pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) { struct l2_bucket *l2b; vm_offset_t next_bucket; pt_entry_t *ptep; u_int total; u_int mappings, is_exec, is_refd; int flushall = 0; /* * we lock in the pmap => pv_head direction */ rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); total = 0; while (sva < eva) { /* * Do one L2 bucket's worth at a time. */ next_bucket = L2_NEXT_BUCKET(sva); if (next_bucket > eva) next_bucket = eva; l2b = pmap_get_l2_bucket(pm, sva); if (l2b == NULL) { sva = next_bucket; continue; } ptep = &l2b->l2b_kva[l2pte_index(sva)]; mappings = 0; while (sva < next_bucket) { struct vm_page *pg; pt_entry_t pte; vm_paddr_t pa; pte = *ptep; if (pte == 0) { /* * Nothing here, move along */ sva += PAGE_SIZE; ptep++; continue; } pm->pm_stats.resident_count--; pa = l2pte_pa(pte); is_exec = 0; is_refd = 1; /* * Update flags. In a number of circumstances, * we could cluster a lot of these and do a * number of sequential pages in one go. */ if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { struct pv_entry *pve; pve = pmap_remove_pv(pg, pm, sva); if (pve) { is_exec = PV_BEEN_EXECD(pve->pv_flags); is_refd = PV_BEEN_REFD(pve->pv_flags); pmap_free_pv_entry(pve); } } if (l2pte_valid(pte) && pmap_is_current(pm)) { if (total < PMAP_REMOVE_CLEAN_LIST_SIZE) { total++; if (is_exec) { cpu_idcache_wbinv_range(sva, PAGE_SIZE); cpu_l2cache_wbinv_range(sva, PAGE_SIZE); cpu_tlb_flushID_SE(sva); } else if (is_refd) { cpu_dcache_wbinv_range(sva, PAGE_SIZE); cpu_l2cache_wbinv_range(sva, PAGE_SIZE); cpu_tlb_flushD_SE(sva); } } else if (total == PMAP_REMOVE_CLEAN_LIST_SIZE) { /* flushall will also only get set for * for a current pmap */ cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); flushall = 1; total++; } } *ptep = 0; PTE_SYNC(ptep); sva += PAGE_SIZE; ptep++; mappings++; } pmap_free_l2_bucket(pm, l2b, mappings); } rw_wunlock(&pvh_global_lock); if (flushall) cpu_tlb_flushID(); PMAP_UNLOCK(pm); } /* * pmap_zero_page() * * Zero a given physical page by mapping it at a page hook point. * In doing the zero page op, the page we zero is mapped cachable, as with * StrongARM accesses to non-cached pages are non-burst making writing * _any_ bulk data very slow. */ #if ARM_MMU_GENERIC != 0 || defined(CPU_XSCALE_CORE3) void pmap_zero_page_generic(vm_paddr_t phys, int off, int size) { if (_arm_bzero && size >= _min_bzero_size && _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) return; mtx_lock(&cmtx); /* * Hook in the page, zero it, invalidate the TLB as needed. * * Note the temporary zero-page mapping must be a non-cached page in * order to work without corruption when write-allocate is enabled. */ *cdst_pte = L2_S_PROTO | phys | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE); PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); if (off || size != PAGE_SIZE) bzero((void *)(cdstp + off), size); else bzero_page(cdstp); mtx_unlock(&cmtx); } #endif /* ARM_MMU_GENERIC != 0 */ #if ARM_MMU_XSCALE == 1 void pmap_zero_page_xscale(vm_paddr_t phys, int off, int size) { if (_arm_bzero && size >= _min_bzero_size && _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) return; mtx_lock(&cmtx); /* * Hook in the page, zero it, and purge the cache for that * zeroed page. Invalidate the TLB as needed. */ *cdst_pte = L2_S_PROTO | phys | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); if (off || size != PAGE_SIZE) bzero((void *)(cdstp + off), size); else bzero_page(cdstp); mtx_unlock(&cmtx); xscale_cache_clean_minidata(); } /* * Change the PTEs for the specified kernel mappings such that they * will use the mini data cache instead of the main data cache. */ void pmap_use_minicache(vm_offset_t va, vm_size_t size) { struct l2_bucket *l2b; pt_entry_t *ptep, *sptep, pte; vm_offset_t next_bucket, eva; #if (ARM_NMMUS > 1) || defined(CPU_XSCALE_CORE3) if (xscale_use_minidata == 0) return; #endif eva = va + size; while (va < eva) { next_bucket = L2_NEXT_BUCKET(va); if (next_bucket > eva) next_bucket = eva; l2b = pmap_get_l2_bucket(kernel_pmap, va); sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; while (va < next_bucket) { pte = *ptep; if (!l2pte_minidata(pte)) { cpu_dcache_wbinv_range(va, PAGE_SIZE); cpu_tlb_flushD_SE(va); *ptep = pte & ~L2_B; } ptep++; va += PAGE_SIZE; } PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); } cpu_cpwait(); } #endif /* ARM_MMU_XSCALE == 1 */ /* * pmap_zero_page zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. */ void pmap_zero_page(vm_page_t m) { pmap_zero_page_func(VM_PAGE_TO_PHYS(m), 0, PAGE_SIZE); } /* * pmap_zero_page_area zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. * * off and size may not cover an area beyond a single hardware page. */ void pmap_zero_page_area(vm_page_t m, int off, int size) { pmap_zero_page_func(VM_PAGE_TO_PHYS(m), off, size); } /* * pmap_zero_page_idle zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. This * is intended to be called from the vm_pagezero process only and * outside of Giant. */ void pmap_zero_page_idle(vm_page_t m) { pmap_zero_page(m); } #if 0 /* * pmap_clean_page() * * This is a local function used to work out the best strategy to clean * a single page referenced by its entry in the PV table. It should be used by * pmap_copy_page, pmap_zero page and maybe some others later on. * * Its policy is effectively: * o If there are no mappings, we don't bother doing anything with the cache. * o If there is one mapping, we clean just that page. * o If there are multiple mappings, we clean the entire cache. * * So that some functions can be further optimised, it returns 0 if it didn't * clean the entire cache, or 1 if it did. * * XXX One bug in this routine is that if the pv_entry has a single page * mapped at 0x00000000 a whole cache clean will be performed rather than * just the 1 page. Since this should not occur in everyday use and if it does * it will just result in not the most efficient clean for the page. * * We don't yet use this function but may want to. */ static int pmap_clean_page(struct pv_entry *pv, boolean_t is_src) { pmap_t pm, pm_to_clean = NULL; struct pv_entry *npv; u_int cache_needs_cleaning = 0; u_int flags = 0; vm_offset_t page_to_clean = 0; if (pv == NULL) { /* nothing mapped in so nothing to flush */ return (0); } /* * Since we flush the cache each time we change to a different * user vmspace, we only need to flush the page if it is in the * current pmap. */ if (curthread) pm = vmspace_pmap(curproc->p_vmspace); else pm = kernel_pmap; for (npv = pv; npv; npv = TAILQ_NEXT(npv, pv_list)) { if (npv->pv_pmap == kernel_pmap || npv->pv_pmap == pm) { flags |= npv->pv_flags; /* * The page is mapped non-cacheable in * this map. No need to flush the cache. */ if (npv->pv_flags & PVF_NC) { #ifdef DIAGNOSTIC if (cache_needs_cleaning) panic("pmap_clean_page: " "cache inconsistency"); #endif break; } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) continue; if (cache_needs_cleaning) { page_to_clean = 0; break; } else { page_to_clean = npv->pv_va; pm_to_clean = npv->pv_pmap; } cache_needs_cleaning = 1; } } if (page_to_clean) { if (PV_BEEN_EXECD(flags)) pmap_idcache_wbinv_range(pm_to_clean, page_to_clean, PAGE_SIZE); else pmap_dcache_wb_range(pm_to_clean, page_to_clean, PAGE_SIZE, !is_src, (flags & PVF_WRITE) == 0); } else if (cache_needs_cleaning) { if (PV_BEEN_EXECD(flags)) pmap_idcache_wbinv_all(pm); else pmap_dcache_wbinv_all(pm); return (1); } return (0); } #endif /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ /* * pmap_copy_page() * * Copy one physical page into another, by mapping the pages into * hook points. The same comment regarding cachability as in * pmap_zero_page also applies here. */ #if ARM_MMU_GENERIC != 0 || defined (CPU_XSCALE_CORE3) void pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) { #if 0 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); #endif /* * Clean the source page. Hold the source page's lock for * the duration of the copy so that no other mappings can * be created while we have a potentially aliased mapping. */ #if 0 /* * XXX: Not needed while we call cpu_dcache_wbinv_all() in * pmap_copy_page(). */ (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); #endif /* * Map the pages into the page hook points, copy them, and purge * the cache for the appropriate page. Invalidate the TLB * as required. */ mtx_lock(&cmtx); *csrc_pte = L2_S_PROTO | src | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; PTE_SYNC(csrc_pte); *cdst_pte = L2_S_PROTO | dst | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(csrcp); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); bcopy_page(csrcp, cdstp); mtx_unlock(&cmtx); cpu_dcache_inv_range(csrcp, PAGE_SIZE); cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); cpu_l2cache_inv_range(csrcp, PAGE_SIZE); cpu_l2cache_wbinv_range(cdstp, PAGE_SIZE); } void pmap_copy_page_offs_generic(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt) { mtx_lock(&cmtx); *csrc_pte = L2_S_PROTO | a_phys | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; PTE_SYNC(csrc_pte); *cdst_pte = L2_S_PROTO | b_phys | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(csrcp); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); bcopy((char *)csrcp + a_offs, (char *)cdstp + b_offs, cnt); mtx_unlock(&cmtx); cpu_dcache_inv_range(csrcp + a_offs, cnt); cpu_dcache_wbinv_range(cdstp + b_offs, cnt); cpu_l2cache_inv_range(csrcp + a_offs, cnt); cpu_l2cache_wbinv_range(cdstp + b_offs, cnt); } #endif /* ARM_MMU_GENERIC != 0 */ #if ARM_MMU_XSCALE == 1 void pmap_copy_page_xscale(vm_paddr_t src, vm_paddr_t dst) { #if 0 /* XXX: Only needed for pmap_clean_page(), which is commented out. */ struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); #endif /* * Clean the source page. Hold the source page's lock for * the duration of the copy so that no other mappings can * be created while we have a potentially aliased mapping. */ #if 0 /* * XXX: Not needed while we call cpu_dcache_wbinv_all() in * pmap_copy_page(). */ (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); #endif /* * Map the pages into the page hook points, copy them, and purge * the cache for the appropriate page. Invalidate the TLB * as required. */ mtx_lock(&cmtx); *csrc_pte = L2_S_PROTO | src | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ PTE_SYNC(csrc_pte); *cdst_pte = L2_S_PROTO | dst | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(csrcp); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); bcopy_page(csrcp, cdstp); mtx_unlock(&cmtx); xscale_cache_clean_minidata(); } void pmap_copy_page_offs_xscale(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt) { mtx_lock(&cmtx); *csrc_pte = L2_S_PROTO | a_phys | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); PTE_SYNC(csrc_pte); *cdst_pte = L2_S_PROTO | b_phys | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(csrcp); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); bcopy((char *)csrcp + a_offs, (char *)cdstp + b_offs, cnt); mtx_unlock(&cmtx); xscale_cache_clean_minidata(); } #endif /* ARM_MMU_XSCALE == 1 */ void pmap_copy_page(vm_page_t src, vm_page_t dst) { cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); if (_arm_memcpy && PAGE_SIZE >= _min_memcpy_size && _arm_memcpy((void *)VM_PAGE_TO_PHYS(dst), (void *)VM_PAGE_TO_PHYS(src), PAGE_SIZE, IS_PHYSICAL) == 0) return; pmap_copy_page_func(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst)); } /* * We have code to do unmapped I/O. However, it isn't quite right and * causes un-page-aligned I/O to devices to fail (most notably newfs * or fsck). We give up a little performance to not allow unmapped I/O * to gain stability. */ int unmapped_buf_allowed = 0; void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { vm_page_t a_pg, b_pg; vm_offset_t a_pg_offset, b_pg_offset; int cnt; cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); while (xfersize > 0) { a_pg = ma[a_offset >> PAGE_SHIFT]; a_pg_offset = a_offset & PAGE_MASK; cnt = min(xfersize, PAGE_SIZE - a_pg_offset); b_pg = mb[b_offset >> PAGE_SHIFT]; b_pg_offset = b_offset & PAGE_MASK; cnt = min(cnt, PAGE_SIZE - b_pg_offset); pmap_copy_page_offs_func(VM_PAGE_TO_PHYS(a_pg), a_pg_offset, VM_PAGE_TO_PHYS(b_pg), b_pg_offset, cnt); xfersize -= cnt; a_offset += cnt; b_offset += cnt; } } vm_offset_t pmap_quick_enter_page(vm_page_t m) { /* * Don't bother with a PCPU pageframe, since we don't support * SMP for anything pre-armv7. Use pmap_kenter() to ensure * caching is handled correctly for multiple mappings of the * same physical page. */ mtx_assert(&qmap_mtx, MA_NOTOWNED); mtx_lock(&qmap_mtx); pmap_kenter(qmap_addr, VM_PAGE_TO_PHYS(m)); return (qmap_addr); } void pmap_quick_remove_page(vm_offset_t addr) { KASSERT(addr == qmap_addr, ("pmap_quick_remove_page: invalid address")); mtx_assert(&qmap_mtx, MA_OWNED); pmap_kremove(addr); mtx_unlock(&qmap_mtx); } /* * this routine returns true if a physical page resides * in the given pmap. */ boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops = 0; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_page_exists_quick: page %p is not managed", m)); rv = FALSE; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } rw_wunlock(&pvh_global_lock); return (rv); } /* * pmap_page_wired_mappings: * * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { pv_entry_t pv; int count; count = 0; if ((m->oflags & VPO_UNMANAGED) != 0) return (count); rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) if ((pv->pv_flags & PVF_WIRED) != 0) count++; rw_wunlock(&pvh_global_lock); return (count); } /* * This function is advisory. */ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { } /* * pmap_ts_referenced: * * Return the count of reference bits for a page, clearing all of them. */ int pmap_ts_referenced(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); return (pmap_clearbit(m, PVF_REF)); } boolean_t pmap_is_modified(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_modified: page %p is not managed", m)); if (m->md.pvh_attrs & PVF_MOD) return (TRUE); return(FALSE); } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("pmap_clear_modify: page %p is exclusive busied", m)); /* * If the page is not PGA_WRITEABLE, then no mappings can be modified. * If the object containing the page is locked and the page is not * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. */ if ((m->aflags & PGA_WRITEABLE) == 0) return; if (m->md.pvh_attrs & PVF_MOD) pmap_clearbit(m, PVF_MOD); } /* * pmap_is_referenced: * * Return whether or not the specified physical page was referenced * in any physical maps. */ boolean_t pmap_is_referenced(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_referenced: page %p is not managed", m)); return ((m->md.pvh_attrs & PVF_REF) != 0); } /* * Clear the write and modified bits in each of the given page's mappings. */ void pmap_remove_write(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_write: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (vm_page_xbusied(m) || (m->aflags & PGA_WRITEABLE) != 0) pmap_clearbit(m, PVF_WRITE); } /* * perform the pmap work for mincore */ int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; vm_paddr_t pa; vm_page_t m; int val; boolean_t managed; PMAP_LOCK(pmap); retry: l2b = pmap_get_l2_bucket(pmap, addr); if (l2b == NULL) { val = 0; goto out; } ptep = &l2b->l2b_kva[l2pte_index(addr)]; pte = *ptep; if (!l2pte_valid(pte)) { val = 0; goto out; } val = MINCORE_INCORE; if (pte & L2_S_PROT_W) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; managed = false; pa = l2pte_pa(pte); m = PHYS_TO_VM_PAGE(pa); if (m != NULL && !(m->oflags & VPO_UNMANAGED)) managed = true; if (managed) { /* * The ARM pmap tries to maintain a per-mapping * reference bit. The trouble is that it's kept in * the PV entry, not the PTE, so it's costly to access * here. You would need to acquire the pvh global * lock, call pmap_find_pv(), and introduce a custom * version of vm_page_pa_tryrelock() that releases and * reacquires the pvh global lock. In the end, I * doubt it's worthwhile. This may falsely report * the given address as referenced. */ if ((m->md.pvh_attrs & PVF_REF) != 0) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; } if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) goto retry; } else out: PA_UNLOCK_COND(*locked_pa); PMAP_UNLOCK(pmap); return (val); } void pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) { } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more superpage mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { } #define BOOTSTRAP_DEBUG /* * pmap_map_section: * * Create a single section mapping. */ void pmap_map_section(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, int cache) { pd_entry_t *pde = (pd_entry_t *) l1pt; pd_entry_t fl; KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("ouin2")); switch (cache) { case PTE_NOCACHE: default: fl = 0; break; case PTE_CACHE: fl = pte_l1_s_cache_mode; break; case PTE_PAGETABLE: fl = pte_l1_s_cache_mode_pt; break; } pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); PTE_SYNC(&pde[va >> L1_S_SHIFT]); } /* * pmap_link_l2pt: * * Link the L2 page table specified by l2pv.pv_pa into the L1 * page table at the slot for "va". */ void pmap_link_l2pt(vm_offset_t l1pt, vm_offset_t va, struct pv_addr *l2pv) { pd_entry_t *pde = (pd_entry_t *) l1pt, proto; u_int slot = va >> L1_S_SHIFT; proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; #ifdef VERBOSE_INIT_ARM printf("pmap_link_l2pt: pa=0x%x va=0x%x\n", l2pv->pv_pa, l2pv->pv_va); #endif pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); PTE_SYNC(&pde[slot]); SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); } /* * pmap_map_entry * * Create a single page mapping. */ void pmap_map_entry(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, int cache) { pd_entry_t *pde = (pd_entry_t *) l1pt; pt_entry_t fl; pt_entry_t *pte; KASSERT(((va | pa) & PAGE_MASK) == 0, ("ouin")); switch (cache) { case PTE_NOCACHE: default: fl = 0; break; case PTE_CACHE: fl = pte_l2_s_cache_mode; break; case PTE_PAGETABLE: fl = pte_l2_s_cache_mode_pt; break; } if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) panic("pmap_map_entry: no L2 table for VA 0x%08x", va); pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); if (pte == NULL) panic("pmap_map_entry: can't find L2 table for VA 0x%08x", va); pte[l2pte_index(va)] = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl; PTE_SYNC(&pte[l2pte_index(va)]); } /* * pmap_map_chunk: * * Map a chunk of memory using the most efficient mappings * possible (section. large page, small page) into the * provided L1 and L2 tables at the specified virtual address. */ vm_size_t pmap_map_chunk(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, vm_size_t size, int prot, int cache) { pd_entry_t *pde = (pd_entry_t *) l1pt; pt_entry_t *pte, f1, f2s, f2l; vm_size_t resid; int i; resid = roundup2(size, PAGE_SIZE); if (l1pt == 0) panic("pmap_map_chunk: no L1 table provided"); #ifdef VERBOSE_INIT_ARM printf("pmap_map_chunk: pa=0x%x va=0x%x size=0x%x resid=0x%x " "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); #endif switch (cache) { case PTE_NOCACHE: default: f1 = 0; f2l = 0; f2s = 0; break; case PTE_CACHE: f1 = pte_l1_s_cache_mode; f2l = pte_l2_l_cache_mode; f2s = pte_l2_s_cache_mode; break; case PTE_PAGETABLE: f1 = pte_l1_s_cache_mode_pt; f2l = pte_l2_l_cache_mode_pt; f2s = pte_l2_s_cache_mode_pt; break; } size = resid; while (resid > 0) { /* See if we can use a section mapping. */ if (L1_S_MAPPABLE_P(va, pa, resid)) { #ifdef VERBOSE_INIT_ARM printf("S"); #endif pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, prot) | f1 | L1_S_DOM(PMAP_DOMAIN_KERNEL); PTE_SYNC(&pde[va >> L1_S_SHIFT]); va += L1_S_SIZE; pa += L1_S_SIZE; resid -= L1_S_SIZE; continue; } /* * Ok, we're going to use an L2 table. Make sure * one is actually in the corresponding L1 slot * for the current VA. */ if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) panic("pmap_map_chunk: no L2 table for VA 0x%08x", va); pte = (pt_entry_t *) kernel_pt_lookup( pde[L1_IDX(va)] & L1_C_ADDR_MASK); if (pte == NULL) panic("pmap_map_chunk: can't find L2 table for VA" "0x%08x", va); /* See if we can use a L2 large page mapping. */ if (L2_L_MAPPABLE_P(va, pa, resid)) { #ifdef VERBOSE_INIT_ARM printf("L"); #endif for (i = 0; i < 16; i++) { pte[l2pte_index(va) + i] = L2_L_PROTO | pa | L2_L_PROT(PTE_KERNEL, prot) | f2l; PTE_SYNC(&pte[l2pte_index(va) + i]); } va += L2_L_SIZE; pa += L2_L_SIZE; resid -= L2_L_SIZE; continue; } /* Use a small page mapping. */ #ifdef VERBOSE_INIT_ARM printf("P"); #endif pte[l2pte_index(va)] = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; PTE_SYNC(&pte[l2pte_index(va)]); va += PAGE_SIZE; pa += PAGE_SIZE; resid -= PAGE_SIZE; } #ifdef VERBOSE_INIT_ARM printf("\n"); #endif return (size); } void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { /* * Remember the memattr in a field that gets used to set the appropriate * bits in the PTEs as mappings are established. */ m->md.pv_memattr = ma; /* * It appears that this function can only be called before any mappings * for the page are established on ARM. If this ever changes, this code * will need to walk the pv_list and make each of the existing mappings * uncacheable, being careful to sync caches and PTEs (and maybe * invalidate TLB?) for any current mapping it modifies. */ if (m->md.pv_kva != 0 || TAILQ_FIRST(&m->md.pv_list) != NULL) panic("Can't change memattr on page with existing mappings"); } Index: head/sys/arm/arm/pmap-v6.c =================================================================== --- head/sys/arm/arm/pmap-v6.c (revision 299068) +++ head/sys/arm/arm/pmap-v6.c (revision 299069) @@ -1,6805 +1,6805 @@ /*- * Copyright (c) 1991 Regents of the University of California. * Copyright (c) 1994 John S. Dyson * Copyright (c) 1994 David Greenman * Copyright (c) 2005-2010 Alan L. Cox * Copyright (c) 2014-2016 Svatopluk Kraus * Copyright (c) 2014-2016 Michal Meloun * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 */ /*- * Copyright (c) 2003 Networks Associates Technology, Inc. * All rights reserved. * * This software was developed for the FreeBSD Project by Jake Burkholder, * Safeport Network Services, and Network Associates Laboratories, the * Security Research Division of Network Associates, Inc. under * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA * CHATS research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * Since the information managed by this module is * also stored by the logical address mapping module, * this module may throw away valid virtual-to-physical * mappings at almost any time. However, invalidations * of virtual-to-physical mappings must be done as * requested. * * In order to cope with hardware architectures which * make virtual-to-physical map invalidates expensive, * this module may delay invalidate or reduced protection * operations until such time as they are actually * necessary. This module is given full information as * to which processors are currently using which maps, * and to when physical maps must be made correct. */ #include "opt_vm.h" #include "opt_pmap.h" #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #else #include #endif #ifdef DDB #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #endif #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif #ifndef DIAGNOSTIC #define PMAP_INLINE __inline #else #define PMAP_INLINE #endif #ifdef PMAP_DEBUG static void pmap_zero_page_check(vm_page_t m); void pmap_debug(int level); int pmap_pid_dump(int pid); #define PDEBUG(_lev_,_stat_) \ if (pmap_debug_level >= (_lev_)) \ ((_stat_)) #define dprintf printf int pmap_debug_level = 1; #else /* PMAP_DEBUG */ #define PDEBUG(_lev_,_stat_) /* Nothing */ #define dprintf(x, arg...) #endif /* PMAP_DEBUG */ /* * Level 2 page tables map definion ('max' is excluded). */ #define PT2V_MIN_ADDRESS ((vm_offset_t)PT2MAP) #define PT2V_MAX_ADDRESS ((vm_offset_t)PT2MAP + PT2MAP_SIZE) #define UPT2V_MIN_ADDRESS ((vm_offset_t)PT2MAP) #define UPT2V_MAX_ADDRESS \ ((vm_offset_t)(PT2MAP + (KERNBASE >> PT2MAP_SHIFT))) /* * Promotion to a 1MB (PTE1) page mapping requires that the corresponding * 4KB (PTE2) page mappings have identical settings for the following fields: */ #define PTE2_PROMOTE (PTE2_V | PTE2_A | PTE2_NM | PTE2_S | PTE2_NG | \ PTE2_NX | PTE2_RO | PTE2_U | PTE2_W | \ PTE2_ATTR_MASK) #define PTE1_PROMOTE (PTE1_V | PTE1_A | PTE1_NM | PTE1_S | PTE1_NG | \ PTE1_NX | PTE1_RO | PTE1_U | PTE1_W | \ PTE1_ATTR_MASK) #define ATTR_TO_L1(l2_attr) ((((l2_attr) & L2_TEX0) ? L1_S_TEX0 : 0) | \ (((l2_attr) & L2_C) ? L1_S_C : 0) | \ (((l2_attr) & L2_B) ? L1_S_B : 0) | \ (((l2_attr) & PTE2_A) ? PTE1_A : 0) | \ (((l2_attr) & PTE2_NM) ? PTE1_NM : 0) | \ (((l2_attr) & PTE2_S) ? PTE1_S : 0) | \ (((l2_attr) & PTE2_NG) ? PTE1_NG : 0) | \ (((l2_attr) & PTE2_NX) ? PTE1_NX : 0) | \ (((l2_attr) & PTE2_RO) ? PTE1_RO : 0) | \ (((l2_attr) & PTE2_U) ? PTE1_U : 0) | \ (((l2_attr) & PTE2_W) ? PTE1_W : 0)) #define ATTR_TO_L2(l1_attr) ((((l1_attr) & L1_S_TEX0) ? L2_TEX0 : 0) | \ (((l1_attr) & L1_S_C) ? L2_C : 0) | \ (((l1_attr) & L1_S_B) ? L2_B : 0) | \ (((l1_attr) & PTE1_A) ? PTE2_A : 0) | \ (((l1_attr) & PTE1_NM) ? PTE2_NM : 0) | \ (((l1_attr) & PTE1_S) ? PTE2_S : 0) | \ (((l1_attr) & PTE1_NG) ? PTE2_NG : 0) | \ (((l1_attr) & PTE1_NX) ? PTE2_NX : 0) | \ (((l1_attr) & PTE1_RO) ? PTE2_RO : 0) | \ (((l1_attr) & PTE1_U) ? PTE2_U : 0) | \ (((l1_attr) & PTE1_W) ? PTE2_W : 0)) /* * PTE2 descriptors creation macros. */ #define PTE2_ATTR_DEFAULT vm_memattr_to_pte2(VM_MEMATTR_DEFAULT) #define PTE2_ATTR_PT vm_memattr_to_pte2(pt_memattr) #define PTE2_KPT(pa) PTE2_KERN(pa, PTE2_AP_KRW, PTE2_ATTR_PT) #define PTE2_KPT_NG(pa) PTE2_KERN_NG(pa, PTE2_AP_KRW, PTE2_ATTR_PT) #define PTE2_KRW(pa) PTE2_KERN(pa, PTE2_AP_KRW, PTE2_ATTR_DEFAULT) #define PTE2_KRO(pa) PTE2_KERN(pa, PTE2_AP_KR, PTE2_ATTR_DEFAULT) #define PV_STATS #ifdef PV_STATS #define PV_STAT(x) do { x ; } while (0) #else #define PV_STAT(x) do { } while (0) #endif /* * The boot_pt1 is used temporary in very early boot stage as L1 page table. * We can init many things with no memory allocation thanks to its static * allocation and this brings two main advantages: * (1) other cores can be started very simply, * (2) various boot loaders can be supported as its arguments can be processed * in virtual address space and can be moved to safe location before * first allocation happened. * Only disadvantage is that boot_pt1 is used only in very early boot stage. * However, the table is uninitialized and so lays in bss. Therefore kernel * image size is not influenced. * * QQQ: In the future, maybe, boot_pt1 can be used for soft reset and * CPU suspend/resume game. */ extern pt1_entry_t boot_pt1[]; vm_paddr_t base_pt1; pt1_entry_t *kern_pt1; pt2_entry_t *kern_pt2tab; pt2_entry_t *PT2MAP; static uint32_t ttb_flags; static vm_memattr_t pt_memattr; ttb_entry_t pmap_kern_ttb; struct pmap kernel_pmap_store; LIST_HEAD(pmaplist, pmap); static struct pmaplist allpmaps; static struct mtx allpmaps_lock; vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ static vm_offset_t kernel_vm_end_new; vm_offset_t kernel_vm_end = KERNBASE + NKPT2PG * NPT2_IN_PG * PTE1_SIZE; vm_offset_t vm_max_kernel_address; vm_paddr_t kernel_l1pa; static struct rwlock __aligned(CACHE_LINE_SIZE) pvh_global_lock; /* * Data for the pv entry allocation mechanism */ static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; static struct md_page *pv_table; /* XXX: Is it used only the list in md_page? */ static int shpgperproc = PMAP_SHPGPERPROC; struct pv_chunk *pv_chunkbase; /* KVA block for pv_chunks */ int pv_maxchunks; /* How many chunks we have KVA for */ vm_offset_t pv_vafree; /* freelist stored in the PTE */ vm_paddr_t first_managed_pa; #define pa_to_pvh(pa) (&pv_table[pte1_index(pa - first_managed_pa)]) /* * All those kernel PT submaps that BSD is so fond of */ struct sysmaps { struct mtx lock; pt2_entry_t *CMAP1; pt2_entry_t *CMAP2; pt2_entry_t *CMAP3; caddr_t CADDR1; caddr_t CADDR2; caddr_t CADDR3; }; static struct sysmaps sysmaps_pcpu[MAXCPU]; static pt2_entry_t *CMAP3; static caddr_t CADDR3; caddr_t _tmppt = 0; struct msgbuf *msgbufp = NULL; /* XXX move it to machdep.c */ /* * Crashdump maps. */ static caddr_t crashdumpmap; static pt2_entry_t *PMAP1 = NULL, *PMAP2; static pt2_entry_t *PADDR1 = NULL, *PADDR2; #ifdef DDB static pt2_entry_t *PMAP3; static pt2_entry_t *PADDR3; static int PMAP3cpu __unused; /* for SMP only */ #endif #ifdef SMP static int PMAP1cpu; static int PMAP1changedcpu; SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, &PMAP1changedcpu, 0, "Number of times pmap_pte2_quick changed CPU with same PMAP1"); #endif static int PMAP1changed; SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, &PMAP1changed, 0, "Number of times pmap_pte2_quick changed PMAP1"); static int PMAP1unchanged; SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, &PMAP1unchanged, 0, "Number of times pmap_pte2_quick didn't change PMAP1"); static struct mtx PMAP2mutex; static __inline void pt2_wirecount_init(vm_page_t m); static boolean_t pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va); void cache_icache_sync_fresh(vm_offset_t va, vm_paddr_t pa, vm_size_t size); /* * Function to set the debug level of the pmap code. */ #ifdef PMAP_DEBUG void pmap_debug(int level) { pmap_debug_level = level; dprintf("pmap_debug: level=%d\n", pmap_debug_level); } #endif /* PMAP_DEBUG */ /* * This table must corespond with memory attribute configuration in vm.h. * First entry is used for normal system mapping. * * Device memory is always marked as shared. * Normal memory is shared only in SMP . * Not outer shareable bits are not used yet. * Class 6 cannot be used on ARM11. */ #define TEXDEF_TYPE_SHIFT 0 #define TEXDEF_TYPE_MASK 0x3 #define TEXDEF_INNER_SHIFT 2 #define TEXDEF_INNER_MASK 0x3 #define TEXDEF_OUTER_SHIFT 4 #define TEXDEF_OUTER_MASK 0x3 #define TEXDEF_NOS_SHIFT 6 #define TEXDEF_NOS_MASK 0x1 #define TEX(t, i, o, s) \ ((t) << TEXDEF_TYPE_SHIFT) | \ ((i) << TEXDEF_INNER_SHIFT) | \ ((o) << TEXDEF_OUTER_SHIFT | \ ((s) << TEXDEF_NOS_SHIFT)) static uint32_t tex_class[8] = { /* type inner cache outer cache */ TEX(PRRR_MEM, NMRR_WB_WA, NMRR_WB_WA, 0), /* 0 - ATTR_WB_WA */ TEX(PRRR_MEM, NMRR_NC, NMRR_NC, 0), /* 1 - ATTR_NOCACHE */ TEX(PRRR_DEV, NMRR_NC, NMRR_NC, 0), /* 2 - ATTR_DEVICE */ TEX(PRRR_SO, NMRR_NC, NMRR_NC, 0), /* 3 - ATTR_SO */ TEX(PRRR_MEM, NMRR_WT, NMRR_WT, 0), /* 4 - ATTR_WT */ TEX(PRRR_MEM, NMRR_NC, NMRR_NC, 0), /* 5 - NOT USED YET */ TEX(PRRR_MEM, NMRR_NC, NMRR_NC, 0), /* 6 - NOT USED YET */ TEX(PRRR_MEM, NMRR_NC, NMRR_NC, 0), /* 7 - NOT USED YET */ }; #undef TEX static uint32_t pte2_attr_tab[8] = { PTE2_ATTR_WB_WA, /* 0 - VM_MEMATTR_WB_WA */ PTE2_ATTR_NOCACHE, /* 1 - VM_MEMATTR_NOCACHE */ PTE2_ATTR_DEVICE, /* 2 - VM_MEMATTR_DEVICE */ PTE2_ATTR_SO, /* 3 - VM_MEMATTR_SO */ PTE2_ATTR_WT, /* 4 - VM_MEMATTR_WRITE_THROUGH */ 0, /* 5 - NOT USED YET */ 0, /* 6 - NOT USED YET */ 0 /* 7 - NOT USED YET */ }; CTASSERT(VM_MEMATTR_WB_WA == 0); CTASSERT(VM_MEMATTR_NOCACHE == 1); CTASSERT(VM_MEMATTR_DEVICE == 2); CTASSERT(VM_MEMATTR_SO == 3); CTASSERT(VM_MEMATTR_WRITE_THROUGH == 4); static inline uint32_t vm_memattr_to_pte2(vm_memattr_t ma) { KASSERT((u_int)ma < 5, ("%s: bad vm_memattr_t %d", __func__, ma)); return (pte2_attr_tab[(u_int)ma]); } static inline uint32_t vm_page_pte2_attr(vm_page_t m) { return (vm_memattr_to_pte2(m->md.pat_mode)); } /* * Convert TEX definition entry to TTB flags. */ static uint32_t encode_ttb_flags(int idx) { uint32_t inner, outer, nos, reg; inner = (tex_class[idx] >> TEXDEF_INNER_SHIFT) & TEXDEF_INNER_MASK; outer = (tex_class[idx] >> TEXDEF_OUTER_SHIFT) & TEXDEF_OUTER_MASK; nos = (tex_class[idx] >> TEXDEF_NOS_SHIFT) & TEXDEF_NOS_MASK; reg = nos << 5; reg |= outer << 3; if (cpuinfo.coherent_walk) reg |= (inner & 0x1) << 6; reg |= (inner & 0x2) >> 1; #ifdef SMP reg |= 1 << 1; #endif return reg; } /* * Set TEX remapping registers in current CPU. */ void pmap_set_tex(void) { uint32_t prrr, nmrr; uint32_t type, inner, outer, nos; int i; #ifdef PMAP_PTE_NOCACHE /* XXX fixme */ if (cpuinfo.coherent_walk) { pt_memattr = VM_MEMATTR_WB_WA; ttb_flags = encode_ttb_flags(0); } else { pt_memattr = VM_MEMATTR_NOCACHE; ttb_flags = encode_ttb_flags(1); } #else pt_memattr = VM_MEMATTR_WB_WA; ttb_flags = encode_ttb_flags(0); #endif prrr = 0; nmrr = 0; /* Build remapping register from TEX classes. */ for (i = 0; i < 8; i++) { type = (tex_class[i] >> TEXDEF_TYPE_SHIFT) & TEXDEF_TYPE_MASK; inner = (tex_class[i] >> TEXDEF_INNER_SHIFT) & TEXDEF_INNER_MASK; outer = (tex_class[i] >> TEXDEF_OUTER_SHIFT) & TEXDEF_OUTER_MASK; nos = (tex_class[i] >> TEXDEF_NOS_SHIFT) & TEXDEF_NOS_MASK; prrr |= type << (i * 2); prrr |= nos << (i + 24); nmrr |= inner << (i * 2); nmrr |= outer << (i * 2 + 16); } /* Add shareable bits for device memory. */ prrr |= PRRR_DS0 | PRRR_DS1; /* Add shareable bits for normal memory in SMP case. */ #ifdef SMP prrr |= PRRR_NS1; #endif cp15_prrr_set(prrr); cp15_nmrr_set(nmrr); /* Caches are disabled, so full TLB flush should be enough. */ tlb_flush_all_local(); } /* * KERNBASE must be multiple of NPT2_IN_PG * PTE1_SIZE. In other words, * KERNBASE is mapped by first L2 page table in L2 page table page. It * meets same constrain due to PT2MAP being placed just under KERNBASE. */ CTASSERT((KERNBASE & (NPT2_IN_PG * PTE1_SIZE - 1)) == 0); CTASSERT((KERNBASE - VM_MAXUSER_ADDRESS) >= PT2MAP_SIZE); /* * In crazy dreams, PAGE_SIZE could be a multiple of PTE2_SIZE in general. * For now, anyhow, the following check must be fulfilled. */ CTASSERT(PAGE_SIZE == PTE2_SIZE); /* * We don't want to mess up MI code with all MMU and PMAP definitions, * so some things, which depend on other ones, are defined independently. * Now, it is time to check that we don't screw up something. */ CTASSERT(PDRSHIFT == PTE1_SHIFT); /* * Check L1 and L2 page table entries definitions consistency. */ CTASSERT(NB_IN_PT1 == (sizeof(pt1_entry_t) * NPTE1_IN_PT1)); CTASSERT(NB_IN_PT2 == (sizeof(pt2_entry_t) * NPTE2_IN_PT2)); /* * Check L2 page tables page consistency. */ CTASSERT(PAGE_SIZE == (NPT2_IN_PG * NB_IN_PT2)); CTASSERT((1 << PT2PG_SHIFT) == NPT2_IN_PG); /* * Check PT2TAB consistency. * PT2TAB_ENTRIES is defined as a division of NPTE1_IN_PT1 by NPT2_IN_PG. * This should be done without remainder. */ CTASSERT(NPTE1_IN_PT1 == (PT2TAB_ENTRIES * NPT2_IN_PG)); /* * A PT2MAP magic. * * All level 2 page tables (PT2s) are mapped continuously and accordingly * into PT2MAP address space. As PT2 size is less than PAGE_SIZE, this can * be done only if PAGE_SIZE is a multiple of PT2 size. All PT2s in one page * must be used together, but not necessary at once. The first PT2 in a page * must map things on correctly aligned address and the others must follow * in right order. */ #define NB_IN_PT2TAB (PT2TAB_ENTRIES * sizeof(pt2_entry_t)) #define NPT2_IN_PT2TAB (NB_IN_PT2TAB / NB_IN_PT2) #define NPG_IN_PT2TAB (NB_IN_PT2TAB / PAGE_SIZE) /* * Check PT2TAB consistency. * NPT2_IN_PT2TAB is defined as a division of NB_IN_PT2TAB by NB_IN_PT2. * NPG_IN_PT2TAB is defined as a division of NB_IN_PT2TAB by PAGE_SIZE. * The both should be done without remainder. */ CTASSERT(NB_IN_PT2TAB == (NPT2_IN_PT2TAB * NB_IN_PT2)); CTASSERT(NB_IN_PT2TAB == (NPG_IN_PT2TAB * PAGE_SIZE)); /* * The implementation was made general, however, with the assumption * bellow in mind. In case of another value of NPG_IN_PT2TAB, * the code should be once more rechecked. */ CTASSERT(NPG_IN_PT2TAB == 1); /* * Get offset of PT2 in a page * associated with given PT1 index. */ static __inline u_int page_pt2off(u_int pt1_idx) { return ((pt1_idx & PT2PG_MASK) * NB_IN_PT2); } /* * Get physical address of PT2 * associated with given PT2s page and PT1 index. */ static __inline vm_paddr_t page_pt2pa(vm_paddr_t pgpa, u_int pt1_idx) { return (pgpa + page_pt2off(pt1_idx)); } /* * Get first entry of PT2 * associated with given PT2s page and PT1 index. */ static __inline pt2_entry_t * page_pt2(vm_offset_t pgva, u_int pt1_idx) { return ((pt2_entry_t *)(pgva + page_pt2off(pt1_idx))); } /* * Get virtual address of PT2s page (mapped in PT2MAP) * which holds PT2 which holds entry which maps given virtual address. */ static __inline vm_offset_t pt2map_pt2pg(vm_offset_t va) { va &= ~(NPT2_IN_PG * PTE1_SIZE - 1); return ((vm_offset_t)pt2map_entry(va)); } /***************************************************************************** * * THREE pmap initialization milestones exist: * * locore.S * -> fundamental init (including MMU) in ASM * * initarm() * -> fundamental init continues in C * -> first available physical address is known * * pmap_bootstrap_prepare() -> FIRST PMAP MILESTONE (first epoch begins) * -> basic (safe) interface for physical address allocation is made * -> basic (safe) interface for virtual mapping is made * -> limited not SMP coherent work is possible * * -> more fundamental init continues in C * -> locks and some more things are available * -> all fundamental allocations and mappings are done * * pmap_bootstrap() -> SECOND PMAP MILESTONE (second epoch begins) * -> phys_avail[] and virtual_avail is set * -> control is passed to vm subsystem * -> physical and virtual address allocation are off limit * -> low level mapping functions, some SMP coherent, * are available, which cannot be used before vm subsystem * is being inited * * mi_startup() * -> vm subsystem is being inited * * pmap_init() -> THIRD PMAP MILESTONE (third epoch begins) * -> pmap is fully inited * *****************************************************************************/ /***************************************************************************** * * PMAP first stage initialization and utility functions * for pre-bootstrap epoch. * * After pmap_bootstrap_prepare() is called, the following functions * can be used: * * (1) strictly only for this stage functions for physical page allocations, * virtual space allocations, and mappings: * * vm_paddr_t pmap_preboot_get_pages(u_int num); * void pmap_preboot_map_pages(vm_paddr_t pa, vm_offset_t va, u_int num); * vm_offset_t pmap_preboot_reserve_pages(u_int num); * vm_offset_t pmap_preboot_get_vpages(u_int num); * void pmap_preboot_map_attr(vm_paddr_t pa, vm_offset_t va, vm_size_t size, * vm_prot_t prot, vm_memattr_t attr); * * (2) for all stages: * * vm_paddr_t pmap_kextract(vm_offset_t va); * * NOTE: This is not SMP coherent stage. * *****************************************************************************/ #define KERNEL_P2V(pa) \ ((vm_offset_t)((pa) - arm_physmem_kernaddr + KERNVIRTADDR)) #define KERNEL_V2P(va) \ ((vm_paddr_t)((va) - KERNVIRTADDR + arm_physmem_kernaddr)) static vm_paddr_t last_paddr; /* * Pre-bootstrap epoch page allocator. */ vm_paddr_t pmap_preboot_get_pages(u_int num) { vm_paddr_t ret; ret = last_paddr; last_paddr += num * PAGE_SIZE; return (ret); } /* - * The fundamental initalization of PMAP stuff. + * The fundamental initialization of PMAP stuff. * * Some things already happened in locore.S and some things could happen * before pmap_bootstrap_prepare() is called, so let's recall what is done: * 1. Caches are disabled. * 2. We are running on virtual addresses already with 'boot_pt1' * as L1 page table. * 3. So far, all virtual addresses can be converted to physical ones and * vice versa by the following macros: * KERNEL_P2V(pa) .... physical to virtual ones, * KERNEL_V2P(va) .... virtual to physical ones. * * What is done herein: * 1. The 'boot_pt1' is replaced by real kernel L1 page table 'kern_pt1'. * 2. PT2MAP magic is brought to live. * 3. Basic preboot functions for page allocations and mappings can be used. * 4. Everything is prepared for L1 cache enabling. * * Variations: * 1. To use second TTB register, so kernel and users page tables will be * separated. This way process forking - pmap_pinit() - could be faster, * it saves physical pages and KVA per a process, and it's simple change. * However, it will lead, due to hardware matter, to the following: * (a) 2G space for kernel and 2G space for users. * (b) 1G space for kernel in low addresses and 3G for users above it. * A question is: Is the case (b) really an option? Note that case (b) * does save neither physical memory and KVA. */ void pmap_bootstrap_prepare(vm_paddr_t last) { vm_paddr_t pt2pg_pa, pt2tab_pa, pa, size; vm_offset_t pt2pg_va; pt1_entry_t *pte1p; pt2_entry_t *pte2p; u_int i; uint32_t actlr_mask, actlr_set, l1_attr; /* * Now, we are going to make real kernel mapping. Note that we are * already running on some mapping made in locore.S and we expect * that it's large enough to ensure nofault access to physical memory * allocated herein before switch. * * As kernel image and everything needed before are and will be mapped * by section mappings, we align last physical address to PTE1_SIZE. */ last_paddr = pte1_roundup(last); /* * Allocate and zero page(s) for kernel L1 page table. * * Note that it's first allocation on space which was PTE1_SIZE * aligned and as such base_pt1 is aligned to NB_IN_PT1 too. */ base_pt1 = pmap_preboot_get_pages(NPG_IN_PT1); kern_pt1 = (pt1_entry_t *)KERNEL_P2V(base_pt1); bzero((void*)kern_pt1, NB_IN_PT1); pte1_sync_range(kern_pt1, NB_IN_PT1); /* Allocate and zero page(s) for kernel PT2TAB. */ pt2tab_pa = pmap_preboot_get_pages(NPG_IN_PT2TAB); kern_pt2tab = (pt2_entry_t *)KERNEL_P2V(pt2tab_pa); bzero(kern_pt2tab, NB_IN_PT2TAB); pte2_sync_range(kern_pt2tab, NB_IN_PT2TAB); /* Allocate and zero page(s) for kernel L2 page tables. */ pt2pg_pa = pmap_preboot_get_pages(NKPT2PG); pt2pg_va = KERNEL_P2V(pt2pg_pa); size = NKPT2PG * PAGE_SIZE; bzero((void*)pt2pg_va, size); pte2_sync_range((pt2_entry_t *)pt2pg_va, size); /* * Add a physical memory segment (vm_phys_seg) corresponding to the * preallocated pages for kernel L2 page tables so that vm_page * structures representing these pages will be created. The vm_page * structures are required for promotion of the corresponding kernel * virtual addresses to section mappings. */ vm_phys_add_seg(pt2tab_pa, pmap_preboot_get_pages(0)); /* * Insert allocated L2 page table pages to PT2TAB and make * link to all PT2s in L1 page table. See how kernel_vm_end * is initialized. * * We play simple and safe. So every KVA will have underlaying * L2 page table, even kernel image mapped by sections. */ pte2p = kern_pt2tab_entry(KERNBASE); for (pa = pt2pg_pa; pa < pt2pg_pa + size; pa += PTE2_SIZE) pt2tab_store(pte2p++, PTE2_KPT(pa)); pte1p = kern_pte1(KERNBASE); for (pa = pt2pg_pa; pa < pt2pg_pa + size; pa += NB_IN_PT2) pte1_store(pte1p++, PTE1_LINK(pa)); /* Make section mappings for kernel. */ l1_attr = ATTR_TO_L1(PTE2_ATTR_DEFAULT); pte1p = kern_pte1(KERNBASE); for (pa = KERNEL_V2P(KERNBASE); pa < last; pa += PTE1_SIZE) pte1_store(pte1p++, PTE1_KERN(pa, PTE1_AP_KRW, l1_attr)); /* * Get free and aligned space for PT2MAP and make L1 page table links * to L2 page tables held in PT2TAB. * * Note that pages holding PT2s are stored in PT2TAB as pt2_entry_t * descriptors and PT2TAB page(s) itself is(are) used as PT2s. Thus * each entry in PT2TAB maps all PT2s in a page. This implies that * virtual address of PT2MAP must be aligned to NPT2_IN_PG * PTE1_SIZE. */ PT2MAP = (pt2_entry_t *)(KERNBASE - PT2MAP_SIZE); pte1p = kern_pte1((vm_offset_t)PT2MAP); for (pa = pt2tab_pa, i = 0; i < NPT2_IN_PT2TAB; i++, pa += NB_IN_PT2) { pte1_store(pte1p++, PTE1_LINK(pa)); } /* * Store PT2TAB in PT2TAB itself, i.e. self reference mapping. * Each pmap will hold own PT2TAB, so the mapping should be not global. */ pte2p = kern_pt2tab_entry((vm_offset_t)PT2MAP); for (pa = pt2tab_pa, i = 0; i < NPG_IN_PT2TAB; i++, pa += PTE2_SIZE) { pt2tab_store(pte2p++, PTE2_KPT_NG(pa)); } /* * Choose correct L2 page table and make mappings for allocations * made herein which replaces temporary locore.S mappings after a while. * Note that PT2MAP cannot be used until we switch to kern_pt1. * * Note, that these allocations started aligned on 1M section and * kernel PT1 was allocated first. Making of mappings must follow * order of physical allocations as we've used KERNEL_P2V() macro * for virtual addresses resolution. */ pte2p = kern_pt2tab_entry((vm_offset_t)kern_pt1); pt2pg_va = KERNEL_P2V(pte2_pa(pte2_load(pte2p))); pte2p = page_pt2(pt2pg_va, pte1_index((vm_offset_t)kern_pt1)); /* Make mapping for kernel L1 page table. */ for (pa = base_pt1, i = 0; i < NPG_IN_PT1; i++, pa += PTE2_SIZE) pte2_store(pte2p++, PTE2_KPT(pa)); /* Make mapping for kernel PT2TAB. */ for (pa = pt2tab_pa, i = 0; i < NPG_IN_PT2TAB; i++, pa += PTE2_SIZE) pte2_store(pte2p++, PTE2_KPT(pa)); /* Finally, switch from 'boot_pt1' to 'kern_pt1'. */ pmap_kern_ttb = base_pt1 | ttb_flags; cpuinfo_get_actlr_modifier(&actlr_mask, &actlr_set); reinit_mmu(pmap_kern_ttb, actlr_mask, actlr_set); /* * Initialize the first available KVA. As kernel image is mapped by * sections, we are leaving some gap behind. */ virtual_avail = (vm_offset_t)kern_pt2tab + NPG_IN_PT2TAB * PAGE_SIZE; } /* * Setup L2 page table page for given KVA. * Used in pre-bootstrap epoch. * * Note that we have allocated NKPT2PG pages for L2 page tables in advance * and used them for mapping KVA starting from KERNBASE. However, this is not * enough. Vectors and devices need L2 page tables too. Note that they are * even above VM_MAX_KERNEL_ADDRESS. */ static __inline vm_paddr_t pmap_preboot_pt2pg_setup(vm_offset_t va) { pt2_entry_t *pte2p, pte2; vm_paddr_t pt2pg_pa; /* Get associated entry in PT2TAB. */ pte2p = kern_pt2tab_entry(va); /* Just return, if PT2s page exists already. */ pte2 = pt2tab_load(pte2p); if (pte2_is_valid(pte2)) return (pte2_pa(pte2)); KASSERT(va >= VM_MAX_KERNEL_ADDRESS, ("%s: NKPT2PG too small", __func__)); /* * Allocate page for PT2s and insert it to PT2TAB. * In other words, map it into PT2MAP space. */ pt2pg_pa = pmap_preboot_get_pages(1); pt2tab_store(pte2p, PTE2_KPT(pt2pg_pa)); /* Zero all PT2s in allocated page. */ bzero((void*)pt2map_pt2pg(va), PAGE_SIZE); pte2_sync_range((pt2_entry_t *)pt2map_pt2pg(va), PAGE_SIZE); return (pt2pg_pa); } /* * Setup L2 page table for given KVA. * Used in pre-bootstrap epoch. */ static void pmap_preboot_pt2_setup(vm_offset_t va) { pt1_entry_t *pte1p; vm_paddr_t pt2pg_pa, pt2_pa; /* Setup PT2's page. */ pt2pg_pa = pmap_preboot_pt2pg_setup(va); pt2_pa = page_pt2pa(pt2pg_pa, pte1_index(va)); /* Insert PT2 to PT1. */ pte1p = kern_pte1(va); pte1_store(pte1p, PTE1_LINK(pt2_pa)); } /* * Get L2 page entry associated with given KVA. * Used in pre-bootstrap epoch. */ static __inline pt2_entry_t* pmap_preboot_vtopte2(vm_offset_t va) { pt1_entry_t *pte1p; /* Setup PT2 if needed. */ pte1p = kern_pte1(va); if (!pte1_is_valid(pte1_load(pte1p))) /* XXX - sections ?! */ pmap_preboot_pt2_setup(va); return (pt2map_entry(va)); } /* * Pre-bootstrap epoch page(s) mapping(s). */ void pmap_preboot_map_pages(vm_paddr_t pa, vm_offset_t va, u_int num) { u_int i; pt2_entry_t *pte2p; /* Map all the pages. */ for (i = 0; i < num; i++) { pte2p = pmap_preboot_vtopte2(va); pte2_store(pte2p, PTE2_KRW(pa)); va += PAGE_SIZE; pa += PAGE_SIZE; } } /* * Pre-bootstrap epoch virtual space alocator. */ vm_offset_t pmap_preboot_reserve_pages(u_int num) { u_int i; vm_offset_t start, va; pt2_entry_t *pte2p; /* Allocate virtual space. */ start = va = virtual_avail; virtual_avail += num * PAGE_SIZE; /* Zero the mapping. */ for (i = 0; i < num; i++) { pte2p = pmap_preboot_vtopte2(va); pte2_store(pte2p, 0); va += PAGE_SIZE; } return (start); } /* * Pre-bootstrap epoch page(s) allocation and mapping(s). */ vm_offset_t pmap_preboot_get_vpages(u_int num) { vm_paddr_t pa; vm_offset_t va; /* Allocate physical page(s). */ pa = pmap_preboot_get_pages(num); /* Allocate virtual space. */ va = virtual_avail; virtual_avail += num * PAGE_SIZE; /* Map and zero all. */ pmap_preboot_map_pages(pa, va, num); bzero((void *)va, num * PAGE_SIZE); return (va); } /* * Pre-bootstrap epoch page mapping(s) with attributes. */ void pmap_preboot_map_attr(vm_paddr_t pa, vm_offset_t va, vm_size_t size, vm_prot_t prot, vm_memattr_t attr) { u_int num; u_int l1_attr, l1_prot, l2_prot, l2_attr; pt1_entry_t *pte1p; pt2_entry_t *pte2p; l2_prot = prot & VM_PROT_WRITE ? PTE2_AP_KRW : PTE2_AP_KR; l2_prot |= (prot & VM_PROT_EXECUTE) ? PTE2_X : PTE2_NX; l2_attr = vm_memattr_to_pte2(attr); l1_prot = ATTR_TO_L1(l2_prot); l1_attr = ATTR_TO_L1(l2_attr); /* Map all the pages. */ num = round_page(size); while (num > 0) { if ((((va | pa) & PTE1_OFFSET) == 0) && (num >= PTE1_SIZE)) { pte1p = kern_pte1(va); pte1_store(pte1p, PTE1_KERN(pa, l1_prot, l1_attr)); va += PTE1_SIZE; pa += PTE1_SIZE; num -= PTE1_SIZE; } else { pte2p = pmap_preboot_vtopte2(va); pte2_store(pte2p, PTE2_KERN(pa, l2_prot, l2_attr)); va += PAGE_SIZE; pa += PAGE_SIZE; num -= PAGE_SIZE; } } } /* * Extract from the kernel page table the physical address * that is mapped by the given virtual address "va". */ vm_paddr_t pmap_kextract(vm_offset_t va) { vm_paddr_t pa; pt1_entry_t pte1; pt2_entry_t pte2; pte1 = pte1_load(kern_pte1(va)); if (pte1_is_section(pte1)) { pa = pte1_pa(pte1) | (va & PTE1_OFFSET); } else if (pte1_is_link(pte1)) { /* * We should beware of concurrent promotion that changes * pte1 at this point. However, it's not a problem as PT2 * page is preserved by promotion in PT2TAB. So even if * it happens, using of PT2MAP is still safe. * * QQQ: However, concurrent removing is a problem which * ends in abort on PT2MAP space. Locking must be used * to deal with this. */ pte2 = pte2_load(pt2map_entry(va)); pa = pte2_pa(pte2) | (va & PTE2_OFFSET); } else { panic("%s: va %#x pte1 %#x", __func__, va, pte1); } return (pa); } /* * Extract from the kernel page table the physical address * that is mapped by the given virtual address "va". Also * return L2 page table entry which maps the address. * * This is only intended to be used for panic dumps. */ vm_paddr_t pmap_dump_kextract(vm_offset_t va, pt2_entry_t *pte2p) { vm_paddr_t pa; pt1_entry_t pte1; pt2_entry_t pte2; pte1 = pte1_load(kern_pte1(va)); if (pte1_is_section(pte1)) { pa = pte1_pa(pte1) | (va & PTE1_OFFSET); pte2 = pa | ATTR_TO_L2(pte1) | PTE2_V; } else if (pte1_is_link(pte1)) { pte2 = pte2_load(pt2map_entry(va)); pa = pte2_pa(pte2); } else { pte2 = 0; pa = 0; } if (pte2p != NULL) *pte2p = pte2; return (pa); } /***************************************************************************** * * PMAP second stage initialization and utility functions * for bootstrap epoch. * * After pmap_bootstrap() is called, the following functions for * mappings can be used: * * void pmap_kenter(vm_offset_t va, vm_paddr_t pa); * void pmap_kremove(vm_offset_t va); * vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, * int prot); * * NOTE: This is not SMP coherent stage. And physical page allocation is not * allowed during this stage. * *****************************************************************************/ /* * Initialize kernel PMAP locks and lists, kernel_pmap itself, and * reserve various virtual spaces for temporary mappings. */ void pmap_bootstrap(vm_offset_t firstaddr) { pt2_entry_t *unused __unused; struct sysmaps *sysmaps; u_int i; /* * Initialize the kernel pmap (which is statically allocated). */ PMAP_LOCK_INIT(kernel_pmap); kernel_l1pa = (vm_paddr_t)kern_pt1; /* for libkvm */ kernel_pmap->pm_pt1 = kern_pt1; kernel_pmap->pm_pt2tab = kern_pt2tab; CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); /* * Initialize the global pv list lock. */ rw_init(&pvh_global_lock, "pmap pv global"); LIST_INIT(&allpmaps); /* * Request a spin mutex so that changes to allpmaps cannot be * preempted by smp_rendezvous_cpus(). */ mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN); mtx_lock_spin(&allpmaps_lock); LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); /* * Reserve some special page table entries/VA space for temporary * mapping of pages. */ #define SYSMAP(c, p, v, n) do { \ v = (c)pmap_preboot_reserve_pages(n); \ p = pt2map_entry((vm_offset_t)v); \ } while (0) /* * Local CMAP1/CMAP2 are used for zeroing and copying pages. * Local CMAP3 is used for data cache cleaning. * Global CMAP3 is used for the idle process page zeroing. */ for (i = 0; i < MAXCPU; i++) { sysmaps = &sysmaps_pcpu[i]; mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF); SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1); SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1); SYSMAP(caddr_t, sysmaps->CMAP3, sysmaps->CADDR3, 1); } SYSMAP(caddr_t, CMAP3, CADDR3, 1); /* * Crashdump maps. */ SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS); /* * _tmppt is used for reading arbitrary physical pages via /dev/mem. */ SYSMAP(caddr_t, unused, _tmppt, 1); /* * PADDR1 and PADDR2 are used by pmap_pte2_quick() and pmap_pte2(), * respectively. PADDR3 is used by pmap_pte2_ddb(). */ SYSMAP(pt2_entry_t *, PMAP1, PADDR1, 1); SYSMAP(pt2_entry_t *, PMAP2, PADDR2, 1); #ifdef DDB SYSMAP(pt2_entry_t *, PMAP3, PADDR3, 1); #endif mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF); /* * Note that in very short time in initarm(), we are going to - * initialize phys_avail[] array and no futher page allocation + * initialize phys_avail[] array and no further page allocation * can happen after that until vm subsystem will be initialized. */ kernel_vm_end_new = kernel_vm_end; virtual_end = vm_max_kernel_address; } static void pmap_init_qpages(void) { struct pcpu *pc; int i; CPU_FOREACH(i) { pc = pcpu_find(i); pc->pc_qmap_addr = kva_alloc(PAGE_SIZE); if (pc->pc_qmap_addr == 0) panic("%s: unable to allocate KVA", __func__); } } SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_qpages, NULL); /* * The function can already be use in second initialization stage. * As such, the function DOES NOT call pmap_growkernel() where PT2 * allocation can happen. So if used, be sure that PT2 for given * virtual address is allocated already! * * Add a wired page to the kva. * Note: not SMP coherent. */ static __inline void pmap_kenter_prot_attr(vm_offset_t va, vm_paddr_t pa, uint32_t prot, uint32_t attr) { pt1_entry_t *pte1p; pt2_entry_t *pte2p; pte1p = kern_pte1(va); if (!pte1_is_valid(pte1_load(pte1p))) { /* XXX - sections ?! */ /* * This is a very low level function, so PT2 and particularly * PT2PG associated with given virtual address must be already * allocated. It's a pain mainly during pmap initialization * stage. However, called after pmap initialization with * virtual address not under kernel_vm_end will lead to * the same misery. */ if (!pte2_is_valid(pte2_load(kern_pt2tab_entry(va)))) panic("%s: kernel PT2 not allocated!", __func__); } pte2p = pt2map_entry(va); pte2_store(pte2p, PTE2_KERN(pa, prot, attr)); } PMAP_INLINE void pmap_kenter(vm_offset_t va, vm_paddr_t pa) { pmap_kenter_prot_attr(va, pa, PTE2_AP_KRW, PTE2_ATTR_DEFAULT); } /* * Remove a page from the kernel pagetables. * Note: not SMP coherent. */ PMAP_INLINE void pmap_kremove(vm_offset_t va) { pt2_entry_t *pte2p; pte2p = pt2map_entry(va); pte2_clear(pte2p); } /* * Share new kernel PT2PG with all pmaps. * The caller is responsible for maintaining TLB consistency. */ static void pmap_kenter_pt2tab(vm_offset_t va, pt2_entry_t npte2) { pmap_t pmap; pt2_entry_t *pte2p; mtx_lock_spin(&allpmaps_lock); LIST_FOREACH(pmap, &allpmaps, pm_list) { pte2p = pmap_pt2tab_entry(pmap, va); pt2tab_store(pte2p, npte2); } mtx_unlock_spin(&allpmaps_lock); } /* * Share new kernel PTE1 with all pmaps. * The caller is responsible for maintaining TLB consistency. */ static void pmap_kenter_pte1(vm_offset_t va, pt1_entry_t npte1) { pmap_t pmap; pt1_entry_t *pte1p; mtx_lock_spin(&allpmaps_lock); LIST_FOREACH(pmap, &allpmaps, pm_list) { pte1p = pmap_pte1(pmap, va); pte1_store(pte1p, npte1); } mtx_unlock_spin(&allpmaps_lock); } /* * Used to map a range of physical addresses into kernel * virtual address space. * * The value passed in '*virt' is a suggested virtual address for * the mapping. Architectures which can support a direct-mapped * physical to virtual region can return the appropriate address * within that region, leaving '*virt' unchanged. Other * architectures should map the pages starting at '*virt' and * update '*virt' with the first usable address after the mapped * region. * * NOTE: Read the comments above pmap_kenter_prot_attr() as * the function is used herein! */ vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) { vm_offset_t va, sva; vm_paddr_t pte1_offset; pt1_entry_t npte1; uint32_t l1prot, l2prot; uint32_t l1attr, l2attr; PDEBUG(1, printf("%s: virt = %#x, start = %#x, end = %#x (size = %#x)," " prot = %d\n", __func__, *virt, start, end, end - start, prot)); l2prot = (prot & VM_PROT_WRITE) ? PTE2_AP_KRW : PTE2_AP_KR; l2prot |= (prot & VM_PROT_EXECUTE) ? PTE2_X : PTE2_NX; l1prot = ATTR_TO_L1(l2prot); l2attr = PTE2_ATTR_DEFAULT; l1attr = ATTR_TO_L1(l2attr); va = *virt; /* * Does the physical address range's size and alignment permit at * least one section mapping to be created? */ pte1_offset = start & PTE1_OFFSET; if ((end - start) - ((PTE1_SIZE - pte1_offset) & PTE1_OFFSET) >= PTE1_SIZE) { /* * Increase the starting virtual address so that its alignment * does not preclude the use of section mappings. */ if ((va & PTE1_OFFSET) < pte1_offset) va = pte1_trunc(va) + pte1_offset; else if ((va & PTE1_OFFSET) > pte1_offset) va = pte1_roundup(va) + pte1_offset; } sva = va; while (start < end) { if ((start & PTE1_OFFSET) == 0 && end - start >= PTE1_SIZE) { KASSERT((va & PTE1_OFFSET) == 0, ("%s: misaligned va %#x", __func__, va)); npte1 = PTE1_KERN(start, l1prot, l1attr); pmap_kenter_pte1(va, npte1); va += PTE1_SIZE; start += PTE1_SIZE; } else { pmap_kenter_prot_attr(va, start, l2prot, l2attr); va += PAGE_SIZE; start += PAGE_SIZE; } } tlb_flush_range(sva, va - sva); *virt = va; return (sva); } /* * Make a temporary mapping for a physical address. * This is only intended to be used for panic dumps. */ void * pmap_kenter_temporary(vm_paddr_t pa, int i) { vm_offset_t va; /* QQQ: 'i' should be less or equal to MAXDUMPPGS. */ va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); pmap_kenter(va, pa); tlb_flush_local(va); return ((void *)crashdumpmap); } /************************************* * * TLB & cache maintenance routines. * *************************************/ /* * We inline these within pmap.c for speed. */ PMAP_INLINE void pmap_tlb_flush(pmap_t pmap, vm_offset_t va) { if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) tlb_flush(va); } PMAP_INLINE void pmap_tlb_flush_range(pmap_t pmap, vm_offset_t sva, vm_size_t size) { if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) tlb_flush_range(sva, size); } /* * Abuse the pte2 nodes for unmapped kva to thread a kva freelist through. * Requirements: * - Must deal with pages in order to ensure that none of the PTE2_* bits * are ever set, PTE2_V in particular. * - Assumes we can write to pte2s without pte2_store() atomic ops. * - Assumes nothing will ever test these addresses for 0 to indicate * no mapping instead of correctly checking PTE2_V. * - Assumes a vm_offset_t will fit in a pte2 (true for arm). * Because PTE2_V is never set, there can be no mappings to invalidate. */ static vm_offset_t pmap_pte2list_alloc(vm_offset_t *head) { pt2_entry_t *pte2p; vm_offset_t va; va = *head; if (va == 0) panic("pmap_ptelist_alloc: exhausted ptelist KVA"); pte2p = pt2map_entry(va); *head = *pte2p; if (*head & PTE2_V) panic("%s: va with PTE2_V set!", __func__); *pte2p = 0; return (va); } static void pmap_pte2list_free(vm_offset_t *head, vm_offset_t va) { pt2_entry_t *pte2p; if (va & PTE2_V) panic("%s: freeing va with PTE2_V set!", __func__); pte2p = pt2map_entry(va); *pte2p = *head; /* virtual! PTE2_V is 0 though */ *head = va; } static void pmap_pte2list_init(vm_offset_t *head, void *base, int npages) { int i; vm_offset_t va; *head = 0; for (i = npages - 1; i >= 0; i--) { va = (vm_offset_t)base + i * PAGE_SIZE; pmap_pte2list_free(head, va); } } /***************************************************************************** * * PMAP third and final stage initialization. * * After pmap_init() is called, PMAP subsystem is fully initialized. * *****************************************************************************/ SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, &pv_entry_max, 0, "Max number of PV entries"); SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, &shpgperproc, 0, "Page share factor per proc"); static u_long nkpt2pg = NKPT2PG; SYSCTL_ULONG(_vm_pmap, OID_AUTO, nkpt2pg, CTLFLAG_RD, &nkpt2pg, 0, "Pre-allocated pages for kernel PT2s"); static int sp_enabled = 1; SYSCTL_INT(_vm_pmap, OID_AUTO, sp_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &sp_enabled, 0, "Are large page mappings enabled?"); static SYSCTL_NODE(_vm_pmap, OID_AUTO, pte1, CTLFLAG_RD, 0, "1MB page mapping counters"); static u_long pmap_pte1_demotions; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, demotions, CTLFLAG_RD, &pmap_pte1_demotions, 0, "1MB page demotions"); static u_long pmap_pte1_mappings; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, mappings, CTLFLAG_RD, &pmap_pte1_mappings, 0, "1MB page mappings"); static u_long pmap_pte1_p_failures; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, p_failures, CTLFLAG_RD, &pmap_pte1_p_failures, 0, "1MB page promotion failures"); static u_long pmap_pte1_promotions; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, promotions, CTLFLAG_RD, &pmap_pte1_promotions, 0, "1MB page promotions"); static u_long pmap_pte1_kern_demotions; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, kern_demotions, CTLFLAG_RD, &pmap_pte1_kern_demotions, 0, "1MB page kernel demotions"); static u_long pmap_pte1_kern_promotions; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, kern_promotions, CTLFLAG_RD, &pmap_pte1_kern_promotions, 0, "1MB page kernel promotions"); static __inline ttb_entry_t pmap_ttb_get(pmap_t pmap) { return (vtophys(pmap->pm_pt1) | ttb_flags); } /* * Initialize a vm_page's machine-dependent fields. * * Variations: * 1. Pages for L2 page tables are always not managed. So, pv_list and * pt2_wirecount can share same physical space. However, proper * initialization on a page alloc for page tables and reinitialization * on the page free must be ensured. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.pv_list); pt2_wirecount_init(m); m->md.pat_mode = VM_MEMATTR_DEFAULT; } /* * Virtualization for faster way how to zero whole page. */ static __inline void pagezero(void *page) { bzero(page, PAGE_SIZE); } /* * Zero L2 page table page. * Use same KVA as in pmap_zero_page(). */ static __inline vm_paddr_t pmap_pt2pg_zero(vm_page_t m) { vm_paddr_t pa; struct sysmaps *sysmaps; pa = VM_PAGE_TO_PHYS(m); /* * XXX: For now, we map whole page even if it's already zero, * to sync it even if the sync is only DSB. */ sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(pa, PTE2_AP_KRW, vm_page_pte2_attr(m))); /* Even VM_ALLOC_ZERO request is only advisory. */ if ((m->flags & PG_ZERO) == 0) pagezero(sysmaps->CADDR2); pte2_sync_range((pt2_entry_t *)sysmaps->CADDR2, PAGE_SIZE); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); return (pa); } /* * Init just allocated page as L2 page table(s) holder * and return its physical address. */ static __inline vm_paddr_t pmap_pt2pg_init(pmap_t pmap, vm_offset_t va, vm_page_t m) { vm_paddr_t pa; pt2_entry_t *pte2p; /* Check page attributes. */ if (m->md.pat_mode != pt_memattr) pmap_page_set_memattr(m, pt_memattr); /* Zero page and init wire counts. */ pa = pmap_pt2pg_zero(m); pt2_wirecount_init(m); /* * Map page to PT2MAP address space for given pmap. * Note that PT2MAP space is shared with all pmaps. */ if (pmap == kernel_pmap) pmap_kenter_pt2tab(va, PTE2_KPT(pa)); else { pte2p = pmap_pt2tab_entry(pmap, va); pt2tab_store(pte2p, PTE2_KPT_NG(pa)); } return (pa); } /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ void pmap_init(void) { vm_size_t s; pt2_entry_t *pte2p, pte2; u_int i, pte1_idx, pv_npg; PDEBUG(1, printf("%s: phys_start = %#x\n", __func__, PHYSADDR)); /* * Initialize the vm page array entries for kernel pmap's * L2 page table pages allocated in advance. */ pte1_idx = pte1_index(KERNBASE - PT2MAP_SIZE); pte2p = kern_pt2tab_entry(KERNBASE - PT2MAP_SIZE); for (i = 0; i < nkpt2pg + NPG_IN_PT2TAB; i++, pte2p++) { vm_paddr_t pa; vm_page_t m; pte2 = pte2_load(pte2p); KASSERT(pte2_is_valid(pte2), ("%s: no valid entry", __func__)); pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pa); KASSERT(m >= vm_page_array && m < &vm_page_array[vm_page_array_size], ("%s: L2 page table page is out of range", __func__)); m->pindex = pte1_idx; m->phys_addr = pa; pte1_idx += NPT2_IN_PG; } /* * Initialize the address space (zone) for the pv entries. Set a * high water mark so that the system can recover from excessive * numbers of pv entries. */ TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count; TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max); pv_entry_max = roundup(pv_entry_max, _NPCPV); pv_entry_high_water = 9 * (pv_entry_max / 10); /* * Are large page mappings enabled? */ TUNABLE_INT_FETCH("vm.pmap.sp_enabled", &sp_enabled); if (sp_enabled) { KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, ("%s: can't assign to pagesizes[1]", __func__)); pagesizes[1] = PTE1_SIZE; } /* * Calculate the size of the pv head table for sections. * Handle the possibility that "vm_phys_segs[...].end" is zero. * Note that the table is only for sections which could be promoted. */ first_managed_pa = pte1_trunc(vm_phys_segs[0].start); pv_npg = (pte1_trunc(vm_phys_segs[vm_phys_nsegs - 1].end - PAGE_SIZE) - first_managed_pa) / PTE1_SIZE + 1; /* * Allocate memory for the pv head table for sections. */ s = (vm_size_t)(pv_npg * sizeof(struct md_page)); s = round_page(s); pv_table = (struct md_page *)kmem_malloc(kernel_arena, s, M_WAITOK | M_ZERO); for (i = 0; i < pv_npg; i++) TAILQ_INIT(&pv_table[i].pv_list); pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc); pv_chunkbase = (struct pv_chunk *)kva_alloc(PAGE_SIZE * pv_maxchunks); if (pv_chunkbase == NULL) panic("%s: not enough kvm for pv chunks", __func__); pmap_pte2list_init(&pv_vafree, pv_chunkbase, pv_maxchunks); } /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) { u_int anychanged; pt2_entry_t *epte2p, *pte2p, pte2; vm_page_t m; vm_paddr_t pa; anychanged = 0; pte2p = pt2map_entry(sva); epte2p = pte2p + count; while (pte2p < epte2p) { m = *ma++; pa = VM_PAGE_TO_PHYS(m); pte2 = pte2_load(pte2p); if ((pte2_pa(pte2) != pa) || (pte2_attr(pte2) != vm_page_pte2_attr(m))) { anychanged++; pte2_store(pte2p, PTE2_KERN(pa, PTE2_AP_KRW, vm_page_pte2_attr(m))); } pte2p++; } if (__predict_false(anychanged)) tlb_flush_range(sva, count * PAGE_SIZE); } /* * This routine tears out page mappings from the * kernel -- it is meant only for temporary mappings. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qremove(vm_offset_t sva, int count) { vm_offset_t va; va = sva; while (count-- > 0) { pmap_kremove(va); va += PAGE_SIZE; } tlb_flush_range(sva, va - sva); } /* * Are we current address space or kernel? */ static __inline int pmap_is_current(pmap_t pmap) { return (pmap == kernel_pmap || (pmap == vmspace_pmap(curthread->td_proc->p_vmspace))); } /* * If the given pmap is not the current or kernel pmap, the returned * pte2 must be released by passing it to pmap_pte2_release(). */ static pt2_entry_t * pmap_pte2(pmap_t pmap, vm_offset_t va) { pt1_entry_t pte1; vm_paddr_t pt2pg_pa; pte1 = pte1_load(pmap_pte1(pmap, va)); if (pte1_is_section(pte1)) panic("%s: attempt to map PTE1", __func__); if (pte1_is_link(pte1)) { /* Are we current address space or kernel? */ if (pmap_is_current(pmap)) return (pt2map_entry(va)); /* Note that L2 page table size is not equal to PAGE_SIZE. */ pt2pg_pa = trunc_page(pte1_link_pa(pte1)); mtx_lock(&PMAP2mutex); if (pte2_pa(pte2_load(PMAP2)) != pt2pg_pa) { pte2_store(PMAP2, PTE2_KPT(pt2pg_pa)); tlb_flush((vm_offset_t)PADDR2); } return (PADDR2 + (arm32_btop(va) & (NPTE2_IN_PG - 1))); } return (NULL); } /* * Releases a pte2 that was obtained from pmap_pte2(). * Be prepared for the pte2p being NULL. */ static __inline void pmap_pte2_release(pt2_entry_t *pte2p) { if ((pt2_entry_t *)(trunc_page((vm_offset_t)pte2p)) == PADDR2) { mtx_unlock(&PMAP2mutex); } } /* * Super fast pmap_pte2 routine best used when scanning * the pv lists. This eliminates many coarse-grained * invltlb calls. Note that many of the pv list * scans are across different pmaps. It is very wasteful * to do an entire tlb flush for checking a single mapping. * * If the given pmap is not the current pmap, pvh_global_lock * must be held and curthread pinned to a CPU. */ static pt2_entry_t * pmap_pte2_quick(pmap_t pmap, vm_offset_t va) { pt1_entry_t pte1; vm_paddr_t pt2pg_pa; pte1 = pte1_load(pmap_pte1(pmap, va)); if (pte1_is_section(pte1)) panic("%s: attempt to map PTE1", __func__); if (pte1_is_link(pte1)) { /* Are we current address space or kernel? */ if (pmap_is_current(pmap)) return (pt2map_entry(va)); rw_assert(&pvh_global_lock, RA_WLOCKED); KASSERT(curthread->td_pinned > 0, ("%s: curthread not pinned", __func__)); /* Note that L2 page table size is not equal to PAGE_SIZE. */ pt2pg_pa = trunc_page(pte1_link_pa(pte1)); if (pte2_pa(pte2_load(PMAP1)) != pt2pg_pa) { pte2_store(PMAP1, PTE2_KPT(pt2pg_pa)); #ifdef SMP PMAP1cpu = PCPU_GET(cpuid); #endif tlb_flush_local((vm_offset_t)PADDR1); PMAP1changed++; } else #ifdef SMP if (PMAP1cpu != PCPU_GET(cpuid)) { PMAP1cpu = PCPU_GET(cpuid); tlb_flush_local((vm_offset_t)PADDR1); PMAP1changedcpu++; } else #endif PMAP1unchanged++; return (PADDR1 + (arm32_btop(va) & (NPTE2_IN_PG - 1))); } return (NULL); } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va) { vm_paddr_t pa; pt1_entry_t pte1; pt2_entry_t *pte2p; PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, va)); if (pte1_is_section(pte1)) pa = pte1_pa(pte1) | (va & PTE1_OFFSET); else if (pte1_is_link(pte1)) { pte2p = pmap_pte2(pmap, va); pa = pte2_pa(pte2_load(pte2p)) | (va & PTE2_OFFSET); pmap_pte2_release(pte2p); } else pa = 0; PMAP_UNLOCK(pmap); return (pa); } /* * Routine: pmap_extract_and_hold * Function: * Atomically extract and hold the physical page * with the given pmap and virtual address pair * if that mapping permits the given protection. */ vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) { vm_paddr_t pa, lockpa; pt1_entry_t pte1; pt2_entry_t pte2, *pte2p; vm_page_t m; lockpa = 0; m = NULL; PMAP_LOCK(pmap); retry: pte1 = pte1_load(pmap_pte1(pmap, va)); if (pte1_is_section(pte1)) { if (!(pte1 & PTE1_RO) || !(prot & VM_PROT_WRITE)) { pa = pte1_pa(pte1) | (va & PTE1_OFFSET); if (vm_page_pa_tryrelock(pmap, pa, &lockpa)) goto retry; m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } else if (pte1_is_link(pte1)) { pte2p = pmap_pte2(pmap, va); pte2 = pte2_load(pte2p); pmap_pte2_release(pte2p); if (pte2_is_valid(pte2) && (!(pte2 & PTE2_RO) || !(prot & VM_PROT_WRITE))) { pa = pte2_pa(pte2); if (vm_page_pa_tryrelock(pmap, pa, &lockpa)) goto retry; m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } PA_UNLOCK_COND(lockpa); PMAP_UNLOCK(pmap); return (m); } /* * Grow the number of kernel L2 page table entries, if needed. */ void pmap_growkernel(vm_offset_t addr) { vm_page_t m; vm_paddr_t pt2pg_pa, pt2_pa; pt1_entry_t pte1; pt2_entry_t pte2; PDEBUG(1, printf("%s: addr = %#x\n", __func__, addr)); /* * All the time kernel_vm_end is first KVA for which underlying * L2 page table is either not allocated or linked from L1 page table * (not considering sections). Except for two possible cases: * * (1) in the very beginning as long as pmap_growkernel() was * not called, it could be first unused KVA (which is not * rounded up to PTE1_SIZE), * * (2) when all KVA space is mapped and kernel_map->max_offset * address is not rounded up to PTE1_SIZE. (For example, * it could be 0xFFFFFFFF.) */ kernel_vm_end = pte1_roundup(kernel_vm_end); mtx_assert(&kernel_map->system_mtx, MA_OWNED); addr = roundup2(addr, PTE1_SIZE); if (addr - 1 >= kernel_map->max_offset) addr = kernel_map->max_offset; while (kernel_vm_end < addr) { pte1 = pte1_load(kern_pte1(kernel_vm_end)); if (pte1_is_valid(pte1)) { kernel_vm_end += PTE1_SIZE; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } continue; } /* * kernel_vm_end_new is used in pmap_pinit() when kernel * mappings are entered to new pmap all at once to avoid race * between pmap_kenter_pte1() and kernel_vm_end increase. * The same aplies to pmap_kenter_pt2tab(). */ kernel_vm_end_new = kernel_vm_end + PTE1_SIZE; pte2 = pt2tab_load(kern_pt2tab_entry(kernel_vm_end)); if (!pte2_is_valid(pte2)) { /* * Install new PT2s page into kernel PT2TAB. */ m = vm_page_alloc(NULL, pte1_index(kernel_vm_end) & ~PT2PG_MASK, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (m == NULL) panic("%s: no memory to grow kernel", __func__); /* * QQQ: To link all new L2 page tables from L1 page * table now and so pmap_kenter_pte1() them * at once together with pmap_kenter_pt2tab() * could be nice speed up. However, * pmap_growkernel() does not happen so often... * QQQ: The other TTBR is another option. */ pt2pg_pa = pmap_pt2pg_init(kernel_pmap, kernel_vm_end, m); } else pt2pg_pa = pte2_pa(pte2); pt2_pa = page_pt2pa(pt2pg_pa, pte1_index(kernel_vm_end)); pmap_kenter_pte1(kernel_vm_end, PTE1_LINK(pt2_pa)); kernel_vm_end = kernel_vm_end_new; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } } } static int kvm_size(SYSCTL_HANDLER_ARGS) { unsigned long ksize = vm_max_kernel_address - KERNBASE; return (sysctl_handle_long(oidp, &ksize, 0, req)); } SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_size, "IU", "Size of KVM"); static int kvm_free(SYSCTL_HANDLER_ARGS) { unsigned long kfree = vm_max_kernel_address - kernel_vm_end; return (sysctl_handle_long(oidp, &kfree, 0, req)); } SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_free, "IU", "Amount of KVM free"); /*********************************************** * * Pmap allocation/deallocation routines. * ***********************************************/ /* * Initialize the pmap for the swapper process. */ void pmap_pinit0(pmap_t pmap) { PDEBUG(1, printf("%s: pmap = %p\n", __func__, pmap)); PMAP_LOCK_INIT(pmap); /* * Kernel page table directory and pmap stuff around is already * initialized, we are using it right now and here. So, finish * only PMAP structures initialization for process0 ... * * Since the L1 page table and PT2TAB is shared with the kernel pmap, * which is already included in the list "allpmaps", this pmap does * not need to be inserted into that list. */ pmap->pm_pt1 = kern_pt1; pmap->pm_pt2tab = kern_pt2tab; CPU_ZERO(&pmap->pm_active); PCPU_SET(curpmap, pmap); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); CPU_SET(0, &pmap->pm_active); } static __inline void pte1_copy_nosync(pt1_entry_t *spte1p, pt1_entry_t *dpte1p, vm_offset_t sva, vm_offset_t eva) { u_int idx, count; idx = pte1_index(sva); count = (pte1_index(eva) - idx + 1) * sizeof(pt1_entry_t); bcopy(spte1p + idx, dpte1p + idx, count); } static __inline void pt2tab_copy_nosync(pt2_entry_t *spte2p, pt2_entry_t *dpte2p, vm_offset_t sva, vm_offset_t eva) { u_int idx, count; idx = pt2tab_index(sva); count = (pt2tab_index(eva) - idx + 1) * sizeof(pt2_entry_t); bcopy(spte2p + idx, dpte2p + idx, count); } /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. */ int pmap_pinit(pmap_t pmap) { pt1_entry_t *pte1p; pt2_entry_t *pte2p; vm_paddr_t pa, pt2tab_pa; u_int i; PDEBUG(6, printf("%s: pmap = %p, pm_pt1 = %p\n", __func__, pmap, pmap->pm_pt1)); /* * No need to allocate L2 page table space yet but we do need * a valid L1 page table and PT2TAB table. * * Install shared kernel mappings to these tables. It's a little * tricky as some parts of KVA are reserved for vectors, devices, * and whatever else. These parts are supposed to be above * vm_max_kernel_address. Thus two regions should be installed: * * (1) . * * QQQ: The second region should be stable enough to be installed * only once in time when the tables are allocated. * QQQ: Maybe copy of both regions at once could be faster ... * QQQ: Maybe the other TTBR is an option. * * Finally, install own PT2TAB table to these tables. */ if (pmap->pm_pt1 == NULL) { pmap->pm_pt1 = (pt1_entry_t *)kmem_alloc_contig(kernel_arena, NB_IN_PT1, M_NOWAIT | M_ZERO, 0, -1UL, NB_IN_PT1, 0, pt_memattr); if (pmap->pm_pt1 == NULL) return (0); } if (pmap->pm_pt2tab == NULL) { /* * QQQ: (1) PT2TAB must be contiguous. If PT2TAB is one page * only, what should be the only size for 32 bit systems, * then we could allocate it with vm_page_alloc() and all * the stuff needed as other L2 page table pages. * (2) Note that a process PT2TAB is special L2 page table * page. Its mapping in kernel_arena is permanent and can * be used no matter which process is current. Its mapping * in PT2MAP can be used only for current process. */ pmap->pm_pt2tab = (pt2_entry_t *)kmem_alloc_attr(kernel_arena, NB_IN_PT2TAB, M_NOWAIT | M_ZERO, 0, -1UL, pt_memattr); if (pmap->pm_pt2tab == NULL) { /* * QQQ: As struct pmap is allocated from UMA with * UMA_ZONE_NOFREE flag, it's important to leave * no allocation in pmap if initialization failed. */ kmem_free(kernel_arena, (vm_offset_t)pmap->pm_pt1, NB_IN_PT1); pmap->pm_pt1 = NULL; return (0); } /* * QQQ: Each L2 page table page vm_page_t has pindex set to * pte1 index of virtual address mapped by this page. * It's not valid for non kernel PT2TABs themselves. * The pindex of these pages can not be altered because * of the way how they are allocated now. However, it * should not be a problem. */ } mtx_lock_spin(&allpmaps_lock); /* * To avoid race with pmap_kenter_pte1() and pmap_kenter_pt2tab(), * kernel_vm_end_new is used here instead of kernel_vm_end. */ pte1_copy_nosync(kern_pt1, pmap->pm_pt1, KERNBASE, kernel_vm_end_new - 1); pte1_copy_nosync(kern_pt1, pmap->pm_pt1, vm_max_kernel_address, 0xFFFFFFFF); pt2tab_copy_nosync(kern_pt2tab, pmap->pm_pt2tab, KERNBASE, kernel_vm_end_new - 1); pt2tab_copy_nosync(kern_pt2tab, pmap->pm_pt2tab, vm_max_kernel_address, 0xFFFFFFFF); LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); /* * Store PT2MAP PT2 pages (a.k.a. PT2TAB) in PT2TAB itself. * I.e. self reference mapping. The PT2TAB is private, however mapped * into shared PT2MAP space, so the mapping should be not global. */ pt2tab_pa = vtophys(pmap->pm_pt2tab); pte2p = pmap_pt2tab_entry(pmap, (vm_offset_t)PT2MAP); for (pa = pt2tab_pa, i = 0; i < NPG_IN_PT2TAB; i++, pa += PTE2_SIZE) { pt2tab_store(pte2p++, PTE2_KPT_NG(pa)); } /* Insert PT2MAP PT2s into pmap PT1. */ pte1p = pmap_pte1(pmap, (vm_offset_t)PT2MAP); for (pa = pt2tab_pa, i = 0; i < NPT2_IN_PT2TAB; i++, pa += NB_IN_PT2) { pte1_store(pte1p++, PTE1_LINK(pa)); } /* * Now synchronize new mapping which was made above. */ pte1_sync_range(pmap->pm_pt1, NB_IN_PT1); pte2_sync_range(pmap->pm_pt2tab, NB_IN_PT2TAB); CPU_ZERO(&pmap->pm_active); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); return (1); } #ifdef INVARIANTS static boolean_t pt2tab_user_is_empty(pt2_entry_t *tab) { u_int i, end; end = pt2tab_index(VM_MAXUSER_ADDRESS); for (i = 0; i < end; i++) if (tab[i] != 0) return (FALSE); return (TRUE); } #endif /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pmap) { #ifdef INVARIANTS vm_offset_t start, end; #endif KASSERT(pmap->pm_stats.resident_count == 0, ("%s: pmap resident count %ld != 0", __func__, pmap->pm_stats.resident_count)); KASSERT(pt2tab_user_is_empty(pmap->pm_pt2tab), ("%s: has allocated user PT2(s)", __func__)); KASSERT(CPU_EMPTY(&pmap->pm_active), ("%s: pmap %p is active on some CPU(s)", __func__, pmap)); mtx_lock_spin(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); #ifdef INVARIANTS start = pte1_index(KERNBASE) * sizeof(pt1_entry_t); end = (pte1_index(0xFFFFFFFF) + 1) * sizeof(pt1_entry_t); bzero((char *)pmap->pm_pt1 + start, end - start); start = pt2tab_index(KERNBASE) * sizeof(pt2_entry_t); end = (pt2tab_index(0xFFFFFFFF) + 1) * sizeof(pt2_entry_t); bzero((char *)pmap->pm_pt2tab + start, end - start); #endif /* * We are leaving PT1 and PT2TAB allocated on released pmap, * so hopefully UMA vmspace_zone will always be inited with * UMA_ZONE_NOFREE flag. */ } /********************************************************* * * L2 table pages and their pages management routines. * *********************************************************/ /* * Virtual interface for L2 page table wire counting. * * Each L2 page table in a page has own counter which counts a number of * valid mappings in a table. Global page counter counts mappings in all * tables in a page plus a single itself mapping in PT2TAB. * * During a promotion we leave the associated L2 page table counter * untouched, so the table (strictly speaking a page which holds it) * is never freed if promoted. * * If a page m->wire_count == 1 then no valid mappings exist in any L2 page * table in the page and the page itself is only mapped in PT2TAB. */ static __inline void pt2_wirecount_init(vm_page_t m) { u_int i; /* * Note: A page m is allocated with VM_ALLOC_WIRED flag and * m->wire_count should be already set correctly. * So, there is no need to set it again herein. */ for (i = 0; i < NPT2_IN_PG; i++) m->md.pt2_wirecount[i] = 0; } static __inline void pt2_wirecount_inc(vm_page_t m, uint32_t pte1_idx) { /* * Note: A just modificated pte2 (i.e. already allocated) * is acquiring one extra reference which must be * explicitly cleared. It influences the KASSERTs herein. * All L2 page tables in a page always belong to the same * pmap, so we allow only one extra reference for the page. */ KASSERT(m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] < (NPTE2_IN_PT2 + 1), ("%s: PT2 is overflowing ...", __func__)); KASSERT(m->wire_count <= (NPTE2_IN_PG + 1), ("%s: PT2PG is overflowing ...", __func__)); m->wire_count++; m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]++; } static __inline void pt2_wirecount_dec(vm_page_t m, uint32_t pte1_idx) { KASSERT(m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] != 0, ("%s: PT2 is underflowing ...", __func__)); KASSERT(m->wire_count > 1, ("%s: PT2PG is underflowing ...", __func__)); m->wire_count--; m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]--; } static __inline void pt2_wirecount_set(vm_page_t m, uint32_t pte1_idx, uint16_t count) { KASSERT(count <= NPTE2_IN_PT2, ("%s: invalid count %u", __func__, count)); KASSERT(m->wire_count > m->md.pt2_wirecount[pte1_idx & PT2PG_MASK], ("%s: PT2PG corrupting (%u, %u) ...", __func__, m->wire_count, m->md.pt2_wirecount[pte1_idx & PT2PG_MASK])); m->wire_count -= m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]; m->wire_count += count; m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] = count; KASSERT(m->wire_count <= (NPTE2_IN_PG + 1), ("%s: PT2PG is overflowed (%u) ...", __func__, m->wire_count)); } static __inline uint32_t pt2_wirecount_get(vm_page_t m, uint32_t pte1_idx) { return (m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]); } static __inline boolean_t pt2_is_empty(vm_page_t m, vm_offset_t va) { return (m->md.pt2_wirecount[pte1_index(va) & PT2PG_MASK] == 0); } static __inline boolean_t pt2_is_full(vm_page_t m, vm_offset_t va) { return (m->md.pt2_wirecount[pte1_index(va) & PT2PG_MASK] == NPTE2_IN_PT2); } static __inline boolean_t pt2pg_is_empty(vm_page_t m) { return (m->wire_count == 1); } /* * This routine is called if the L2 page table * is not mapped correctly. */ static vm_page_t _pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags) { uint32_t pte1_idx; pt1_entry_t *pte1p; pt2_entry_t pte2; vm_page_t m; vm_paddr_t pt2pg_pa, pt2_pa; pte1_idx = pte1_index(va); pte1p = pmap->pm_pt1 + pte1_idx; KASSERT(pte1_load(pte1p) == 0, ("%s: pm_pt1[%#x] is not zero: %#x", __func__, pte1_idx, pte1_load(pte1p))); pte2 = pt2tab_load(pmap_pt2tab_entry(pmap, va)); if (!pte2_is_valid(pte2)) { /* * Install new PT2s page into pmap PT2TAB. */ m = vm_page_alloc(NULL, pte1_idx & ~PT2PG_MASK, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (m == NULL) { if ((flags & PMAP_ENTER_NOSLEEP) == 0) { PMAP_UNLOCK(pmap); rw_wunlock(&pvh_global_lock); VM_WAIT; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); } /* * Indicate the need to retry. While waiting, * the L2 page table page may have been allocated. */ return (NULL); } pmap->pm_stats.resident_count++; pt2pg_pa = pmap_pt2pg_init(pmap, va, m); } else { pt2pg_pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pt2pg_pa); } pt2_wirecount_inc(m, pte1_idx); pt2_pa = page_pt2pa(pt2pg_pa, pte1_idx); pte1_store(pte1p, PTE1_LINK(pt2_pa)); return (m); } static vm_page_t pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags) { u_int pte1_idx; pt1_entry_t *pte1p, pte1; vm_page_t m; pte1_idx = pte1_index(va); retry: pte1p = pmap->pm_pt1 + pte1_idx; pte1 = pte1_load(pte1p); /* * This supports switching from a 1MB page to a * normal 4K page. */ if (pte1_is_section(pte1)) { (void)pmap_demote_pte1(pmap, pte1p, va); /* * Reload pte1 after demotion. * * Note: Demotion can even fail as either PT2 is not find for * the virtual address or PT2PG can not be allocated. */ pte1 = pte1_load(pte1p); } /* * If the L2 page table page is mapped, we just increment the * hold count, and activate it. */ if (pte1_is_link(pte1)) { m = PHYS_TO_VM_PAGE(pte1_link_pa(pte1)); pt2_wirecount_inc(m, pte1_idx); } else { /* * Here if the PT2 isn't mapped, or if it has * been deallocated. */ m = _pmap_allocpte2(pmap, va, flags); if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0) goto retry; } return (m); } static __inline void pmap_free_zero_pages(struct spglist *free) { vm_page_t m; while ((m = SLIST_FIRST(free)) != NULL) { SLIST_REMOVE_HEAD(free, plinks.s.ss); /* Preserve the page's PG_ZERO setting. */ vm_page_free_toq(m); } } /* * Schedule the specified unused L2 page table page to be freed. Specifically, * add the page to the specified list of pages that will be released to the * physical memory manager after the TLB has been updated. */ static __inline void pmap_add_delayed_free_list(vm_page_t m, struct spglist *free) { /* * Put page on a list so that it is released after * *ALL* TLB shootdown is done */ #ifdef PMAP_DEBUG pmap_zero_page_check(m); #endif m->flags |= PG_ZERO; SLIST_INSERT_HEAD(free, m, plinks.s.ss); } /* * Unwire L2 page tables page. */ static void pmap_unwire_pt2pg(pmap_t pmap, vm_offset_t va, vm_page_t m) { pt1_entry_t *pte1p, opte1 __unused; pt2_entry_t *pte2p; uint32_t i; KASSERT(pt2pg_is_empty(m), ("%s: pmap %p PT2PG %p wired", __func__, pmap, m)); /* * Unmap all L2 page tables in the page from L1 page table. * * QQQ: Individual L2 page tables (except the last one) can be unmapped * earlier. However, we are doing that this way. */ KASSERT(m->pindex == (pte1_index(va) & ~PT2PG_MASK), ("%s: pmap %p va %#x PT2PG %p bad index", __func__, pmap, va, m)); pte1p = pmap->pm_pt1 + m->pindex; for (i = 0; i < NPT2_IN_PG; i++, pte1p++) { KASSERT(m->md.pt2_wirecount[i] == 0, ("%s: pmap %p PT2 %u (PG %p) wired", __func__, pmap, i, m)); opte1 = pte1_load(pte1p); if (pte1_is_link(opte1)) { pte1_clear(pte1p); /* * Flush intermediate TLB cache. */ pmap_tlb_flush(pmap, (m->pindex + i) << PTE1_SHIFT); } #ifdef INVARIANTS else KASSERT((opte1 == 0) || pte1_is_section(opte1), ("%s: pmap %p va %#x bad pte1 %x at %u", __func__, pmap, va, opte1, i)); #endif } /* * Unmap the page from PT2TAB. */ pte2p = pmap_pt2tab_entry(pmap, va); (void)pt2tab_load_clear(pte2p); pmap_tlb_flush(pmap, pt2map_pt2pg(va)); m->wire_count = 0; pmap->pm_stats.resident_count--; /* * This is a release store so that the ordinary store unmapping * the L2 page table page is globally performed before TLB shoot- * down is begun. */ atomic_subtract_rel_int(&vm_cnt.v_wire_count, 1); } /* * Decrements a L2 page table page's wire count, which is used to record the * number of valid page table entries within the page. If the wire count * drops to zero, then the page table page is unmapped. Returns TRUE if the * page table page was unmapped and FALSE otherwise. */ static __inline boolean_t pmap_unwire_pt2(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { pt2_wirecount_dec(m, pte1_index(va)); if (pt2pg_is_empty(m)) { /* * QQQ: Wire count is zero, so whole page should be zero and * we can set PG_ZERO flag to it. * Note that when promotion is enabled, it takes some * more efforts. See pmap_unwire_pt2_all() below. */ pmap_unwire_pt2pg(pmap, va, m); pmap_add_delayed_free_list(m, free); return (TRUE); } else return (FALSE); } /* * Drop a L2 page table page's wire count at once, which is used to record * the number of valid L2 page table entries within the page. If the wire * count drops to zero, then the L2 page table page is unmapped. */ static __inline void pmap_unwire_pt2_all(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { u_int pte1_idx = pte1_index(va); KASSERT(m->pindex == (pte1_idx & ~PT2PG_MASK), ("%s: PT2 page's pindex is wrong", __func__)); KASSERT(m->wire_count > pt2_wirecount_get(m, pte1_idx), ("%s: bad pt2 wire count %u > %u", __func__, m->wire_count, pt2_wirecount_get(m, pte1_idx))); /* * It's possible that the L2 page table was never used. * It happened in case that a section was created without promotion. */ if (pt2_is_full(m, va)) { pt2_wirecount_set(m, pte1_idx, 0); /* * QQQ: We clear L2 page table now, so when L2 page table page * is going to be freed, we can set it PG_ZERO flag ... * This function is called only on section mappings, so * hopefully it's not to big overload. * * XXX: If pmap is current, existing PT2MAP mapping could be * used for zeroing. */ pmap_zero_page_area(m, page_pt2off(pte1_idx), NB_IN_PT2); } #ifdef INVARIANTS else KASSERT(pt2_is_empty(m, va), ("%s: PT2 is not empty (%u)", __func__, pt2_wirecount_get(m, pte1_idx))); #endif if (pt2pg_is_empty(m)) { pmap_unwire_pt2pg(pmap, va, m); pmap_add_delayed_free_list(m, free); } } /* * After removing a L2 page table entry, this routine is used to * conditionally free the page, and manage the hold/wire counts. */ static boolean_t pmap_unuse_pt2(pmap_t pmap, vm_offset_t va, struct spglist *free) { pt1_entry_t pte1; vm_page_t mpte; if (va >= VM_MAXUSER_ADDRESS) return (FALSE); pte1 = pte1_load(pmap_pte1(pmap, va)); mpte = PHYS_TO_VM_PAGE(pte1_link_pa(pte1)); return (pmap_unwire_pt2(pmap, va, mpte, free)); } /************************************* * * Page management routines. * *************************************/ CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); CTASSERT(_NPCM == 11); CTASSERT(_NPCPV == 336); static __inline struct pv_chunk * pv_to_chunk(pv_entry_t pv) { return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); } #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) #define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */ #define PC_FREE10 0x0000fffful /* Free values for index 10 */ static const uint32_t pc_freemask[_NPCM] = { PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE10 }; SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, "Current number of pv entries"); #ifdef PV_STATS static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, "Current number of pv entry chunks"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, "Current number of pv entry chunks allocated"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, "Current number of pv entry chunks frees"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, "Number of times tried to get a chunk page but failed."); static long pv_entry_frees, pv_entry_allocs; static int pv_entry_spare; SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, "Current number of pv entry frees"); SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, "Current number of pv entry allocs"); SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, "Current number of spare pv entries"); #endif /* * Is given page managed? */ static __inline boolean_t is_managed(vm_paddr_t pa) { vm_offset_t pgnum; vm_page_t m; pgnum = atop(pa); if (pgnum >= first_page) { m = PHYS_TO_VM_PAGE(pa); if (m == NULL) return (FALSE); if ((m->oflags & VPO_UNMANAGED) == 0) return (TRUE); } return (FALSE); } static __inline boolean_t pte1_is_managed(pt1_entry_t pte1) { return (is_managed(pte1_pa(pte1))); } static __inline boolean_t pte2_is_managed(pt2_entry_t pte2) { return (is_managed(pte2_pa(pte2))); } /* * We are in a serious low memory condition. Resort to * drastic measures to free some pages so we can allocate * another pv entry chunk. */ static vm_page_t pmap_pv_reclaim(pmap_t locked_pmap) { struct pch newtail; struct pv_chunk *pc; struct md_page *pvh; pt1_entry_t *pte1p; pmap_t pmap; pt2_entry_t *pte2p, tpte2; pv_entry_t pv; vm_offset_t va; vm_page_t m, m_pc; struct spglist free; uint32_t inuse; int bit, field, freed; PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); pmap = NULL; m_pc = NULL; SLIST_INIT(&free); TAILQ_INIT(&newtail); while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && (pv_vafree == 0 || SLIST_EMPTY(&free))) { TAILQ_REMOVE(&pv_chunks, pc, pc_lru); if (pmap != pc->pc_pmap) { if (pmap != NULL) { if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } pmap = pc->pc_pmap; /* Avoid deadlock and lock recursion. */ if (pmap > locked_pmap) PMAP_LOCK(pmap); else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) { pmap = NULL; TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } } /* * Destroy every non-wired, 4 KB page mapping in the chunk. */ freed = 0; for (field = 0; field < _NPCM; field++) { for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = ffs(inuse) - 1; pv = &pc->pc_pventry[field * 32 + bit]; va = pv->pv_va; pte1p = pmap_pte1(pmap, va); if (pte1_is_section(pte1_load(pte1p))) continue; pte2p = pmap_pte2(pmap, va); tpte2 = pte2_load(pte2p); if ((tpte2 & PTE2_W) == 0) tpte2 = pte2_load_clear(pte2p); pmap_pte2_release(pte2p); if ((tpte2 & PTE2_W) != 0) continue; KASSERT(tpte2 != 0, ("pmap_pv_reclaim: pmap %p va %#x zero pte", pmap, va)); pmap_tlb_flush(pmap, va); m = PHYS_TO_VM_PAGE(pte2_pa(tpte2)); if (pte2_is_dirty(tpte2)) vm_page_dirty(m); if ((tpte2 & PTE2_A) != 0) vm_page_aflag_set(m, PGA_REFERENCED); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) { vm_page_aflag_clear(m, PGA_WRITEABLE); } } pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt2(pmap, va, &free); freed++; } } if (freed == 0) { TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } /* Every freed mapping is for a 4 KB page. */ pmap->pm_stats.resident_count -= freed; PV_STAT(pv_entry_frees += freed); PV_STAT(pv_entry_spare += freed); pv_entry_count -= freed; TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); for (field = 0; field < _NPCM; field++) if (pc->pc_map[field] != pc_freemask[field]) { TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); /* * One freed pv entry in locked_pmap is * sufficient. */ if (pmap == locked_pmap) goto out; break; } if (field == _NPCM) { PV_STAT(pv_entry_spare -= _NPCPV); PV_STAT(pc_chunk_count--); PV_STAT(pc_chunk_frees++); /* Entire chunk is free; return it. */ m_pc = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); pmap_qremove((vm_offset_t)pc, 1); pmap_pte2list_free(&pv_vafree, (vm_offset_t)pc); break; } } out: TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru); if (pmap != NULL) { if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) { m_pc = SLIST_FIRST(&free); SLIST_REMOVE_HEAD(&free, plinks.s.ss); /* Recycle a freed page table page. */ m_pc->wire_count = 1; atomic_add_int(&vm_cnt.v_wire_count, 1); } pmap_free_zero_pages(&free); return (m_pc); } static void free_pv_chunk(struct pv_chunk *pc) { vm_page_t m; TAILQ_REMOVE(&pv_chunks, pc, pc_lru); PV_STAT(pv_entry_spare -= _NPCPV); PV_STAT(pc_chunk_count--); PV_STAT(pc_chunk_frees++); /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); pmap_qremove((vm_offset_t)pc, 1); vm_page_unwire(m, PQ_NONE); vm_page_free(m); pmap_pte2list_free(&pv_vafree, (vm_offset_t)pc); } /* * Free the pv_entry back to the free list. */ static void free_pv_entry(pmap_t pmap, pv_entry_t pv) { struct pv_chunk *pc; int idx, field, bit; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); pv_entry_count--; pc = pv_to_chunk(pv); idx = pv - &pc->pc_pventry[0]; field = idx / 32; bit = idx % 32; pc->pc_map[field] |= 1ul << bit; for (idx = 0; idx < _NPCM; idx++) if (pc->pc_map[idx] != pc_freemask[idx]) { /* * 98% of the time, pc is already at the head of the * list. If it isn't already, move it to the head. */ if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != pc)) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); } return; } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } /* * Get a new pv_entry, allocating a block from the system * when needed. */ static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try) { static const struct timeval printinterval = { 60, 0 }; static struct timeval lastprint; int bit, field; pv_entry_t pv; struct pv_chunk *pc; vm_page_t m; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(pv_entry_allocs++); pv_entry_count++; if (pv_entry_count > pv_entry_high_water) if (ratecheck(&lastprint, &printinterval)) printf("Approaching the limit on PV entries, consider " "increasing either the vm.pmap.shpgperproc or the " "vm.pmap.pv_entry_max tunable.\n"); retry: pc = TAILQ_FIRST(&pmap->pm_pvchunk); if (pc != NULL) { for (field = 0; field < _NPCM; field++) { if (pc->pc_map[field]) { bit = ffs(pc->pc_map[field]) - 1; break; } } if (field < _NPCM) { pv = &pc->pc_pventry[field * 32 + bit]; pc->pc_map[field] &= ~(1ul << bit); /* If this was the last item, move it to tail */ for (field = 0; field < _NPCM; field++) if (pc->pc_map[field] != 0) { PV_STAT(pv_entry_spare--); return (pv); /* not full, return */ } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); PV_STAT(pv_entry_spare--); return (pv); } } /* * Access to the pte2list "pv_vafree" is synchronized by the pvh * global lock. If "pv_vafree" is currently non-empty, it will * remain non-empty until pmap_pte2list_alloc() completes. */ if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { if (try) { pv_entry_count--; PV_STAT(pc_chunk_tryfail++); return (NULL); } m = pmap_pv_reclaim(pmap); if (m == NULL) goto retry; } PV_STAT(pc_chunk_count++); PV_STAT(pc_chunk_allocs++); pc = (struct pv_chunk *)pmap_pte2list_alloc(&pv_vafree); pmap_qenter((vm_offset_t)pc, &m, 1); pc->pc_pmap = pmap; pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */ for (field = 1; field < _NPCM; field++) pc->pc_map[field] = pc_freemask[field]; TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); pv = &pc->pc_pventry[0]; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); PV_STAT(pv_entry_spare += _NPCPV - 1); return (pv); } /* * Create a pv entry for page at pa for * (pmap, va). */ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); pv = get_pv_entry(pmap, FALSE); pv->pv_va = va; TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); } static __inline pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); break; } } return (pv); } static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); free_pv_entry(pmap, pv); } static void pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va) { struct md_page *pvh; rw_assert(&pvh_global_lock, RA_WLOCKED); pmap_pvh_free(&m->md, pmap, va); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } } static void pmap_pv_demote_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa) { struct md_page *pvh; pv_entry_t pv; vm_offset_t va_last; vm_page_t m; rw_assert(&pvh_global_lock, RA_WLOCKED); KASSERT((pa & PTE1_OFFSET) == 0, ("pmap_pv_demote_pte1: pa is not 1mpage aligned")); /* * Transfer the 1mpage's pv entry for this mapping to the first * page's pv list. */ pvh = pa_to_pvh(pa); va = pte1_trunc(va); pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pv_demote_pte1: pv not found")); m = PHYS_TO_VM_PAGE(pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); /* Instantiate the remaining NPTE2_IN_PT2 - 1 pv entries. */ va_last = va + PTE1_SIZE - PAGE_SIZE; do { m++; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_pv_demote_pte1: page %p is not managed", m)); va += PAGE_SIZE; pmap_insert_entry(pmap, va, m); } while (va < va_last); } static void pmap_pv_promote_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa) { struct md_page *pvh; pv_entry_t pv; vm_offset_t va_last; vm_page_t m; rw_assert(&pvh_global_lock, RA_WLOCKED); KASSERT((pa & PTE1_OFFSET) == 0, ("pmap_pv_promote_pte1: pa is not 1mpage aligned")); /* * Transfer the first page's pv entry for this mapping to the * 1mpage's pv list. Aside from avoiding the cost of a call * to get_pv_entry(), a transfer avoids the possibility that * get_pv_entry() calls pmap_pv_reclaim() and that pmap_pv_reclaim() * removes one of the mappings that is being promoted. */ m = PHYS_TO_VM_PAGE(pa); va = pte1_trunc(va); pv = pmap_pvh_remove(&m->md, pmap, va); KASSERT(pv != NULL, ("pmap_pv_promote_pte1: pv not found")); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); /* Free the remaining NPTE2_IN_PT2 - 1 pv entries. */ va_last = va + PTE1_SIZE - PAGE_SIZE; do { m++; va += PAGE_SIZE; pmap_pvh_free(&m->md, pmap, va); } while (va < va_last); } /* * Conditionally create a pv entry. */ static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); if (pv_entry_count < pv_entry_high_water && (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); return (TRUE); } else return (FALSE); } /* * Create the pv entries for each of the pages within a section. */ static boolean_t pmap_pv_insert_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa) { struct md_page *pvh; pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); if (pv_entry_count < pv_entry_high_water && (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); return (TRUE); } else return (FALSE); } static inline void pmap_tlb_flush_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t npte1) { /* Kill all the small mappings or the big one only. */ if (pte1_is_section(npte1)) pmap_tlb_flush_range(pmap, pte1_trunc(va), PTE1_SIZE); else pmap_tlb_flush(pmap, pte1_trunc(va)); } /* * Update kernel pte1 on all pmaps. * * The following function is called only on one cpu with disabled interrupts. * In SMP case, smp_rendezvous_cpus() is used to stop other cpus. This way * nobody can invoke explicit hardware table walk during the update of pte1. * Unsolicited hardware table walk can still happen, invoked by speculative * data or instruction prefetch or even by speculative hardware table walk. * * The break-before-make approach should be implemented here. However, it's * not so easy to do that for kernel mappings as it would be unhappy to unmap * itself unexpectedly but voluntarily. */ static void pmap_update_pte1_kernel(vm_offset_t va, pt1_entry_t npte1) { pmap_t pmap; pt1_entry_t *pte1p; /* * Get current pmap. Interrupts should be disabled here * so PCPU_GET() is done atomically. */ pmap = PCPU_GET(curpmap); if (pmap == NULL) pmap = kernel_pmap; /* * (1) Change pte1 on current pmap. * (2) Flush all obsolete TLB entries on current CPU. * (3) Change pte1 on all pmaps. * (4) Flush all obsolete TLB entries on all CPUs in SMP case. */ pte1p = pmap_pte1(pmap, va); pte1_store(pte1p, npte1); /* Kill all the small mappings or the big one only. */ if (pte1_is_section(npte1)) { pmap_pte1_kern_promotions++; tlb_flush_range_local(pte1_trunc(va), PTE1_SIZE); } else { pmap_pte1_kern_demotions++; tlb_flush_local(pte1_trunc(va)); } /* * In SMP case, this function is called when all cpus are at smp * rendezvous, so there is no need to use 'allpmaps_lock' lock here. * In UP case, the function is called with this lock locked. */ LIST_FOREACH(pmap, &allpmaps, pm_list) { pte1p = pmap_pte1(pmap, va); pte1_store(pte1p, npte1); } #ifdef SMP /* Kill all the small mappings or the big one only. */ if (pte1_is_section(npte1)) tlb_flush_range(pte1_trunc(va), PTE1_SIZE); else tlb_flush(pte1_trunc(va)); #endif } #ifdef SMP struct pte1_action { vm_offset_t va; pt1_entry_t npte1; u_int update; /* CPU that updates the PTE1 */ }; static void pmap_update_pte1_action(void *arg) { struct pte1_action *act = arg; if (act->update == PCPU_GET(cpuid)) pmap_update_pte1_kernel(act->va, act->npte1); } /* * Change pte1 on current pmap. * Note that kernel pte1 must be changed on all pmaps. * * By ARM ARM manual, the behaviour is UNPREDICABLE when two or more TLB * entries map same VA. It's a problem when either promotion or demotion * is being done. The pte1 update and appropriate TLB flush must be done * atomically in general. */ static void pmap_change_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va, pt1_entry_t npte1) { if (pmap == kernel_pmap) { struct pte1_action act; sched_pin(); act.va = va; act.npte1 = npte1; act.update = PCPU_GET(cpuid); smp_rendezvous_cpus(all_cpus, smp_no_rendevous_barrier, pmap_update_pte1_action, NULL, &act); sched_unpin(); } else { register_t cspr; /* * Use break-before-make approach for changing userland * mappings. It can cause L1 translation aborts on other * cores in SMP case. So, special treatment is implemented * in pmap_fault(). Interrups are disabled here to make it * without any interruption as quick as possible. */ cspr = disable_interrupts(PSR_I | PSR_F); pte1_clear(pte1p); pmap_tlb_flush_pte1(pmap, va, npte1); pte1_store(pte1p, npte1); restore_interrupts(cspr); } } #else static void pmap_change_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va, pt1_entry_t npte1) { if (pmap == kernel_pmap) { mtx_lock_spin(&allpmaps_lock); pmap_update_pte1_kernel(va, npte1); mtx_unlock_spin(&allpmaps_lock); } else { register_t cspr; /* * Use break-before-make approach for changing userland * mappings. It's absolutely safe in UP case when interrupts * are disabled. */ cspr = disable_interrupts(PSR_I | PSR_F); pte1_clear(pte1p); pmap_tlb_flush_pte1(pmap, va, npte1); pte1_store(pte1p, npte1); restore_interrupts(cspr); } } #endif /* * Tries to promote the NPTE2_IN_PT2, contiguous 4KB page mappings that are * within a single page table page (PT2) to a single 1MB page mapping. * For promotion to occur, two conditions must be met: (1) the 4KB page * mappings must map aligned, contiguous physical memory and (2) the 4KB page * mappings must have identical characteristics. * * Managed (PG_MANAGED) mappings within the kernel address space are not * promoted. The reason is that kernel PTE1s are replicated in each pmap but * pmap_remove_write(), pmap_clear_modify(), and pmap_clear_reference() only * read the PTE1 from the kernel pmap. */ static void pmap_promote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va) { pt1_entry_t npte1; pt2_entry_t *fpte2p, fpte2, fpte2_fav; pt2_entry_t *pte2p, pte2; vm_offset_t pteva __unused; vm_page_t m __unused; PDEBUG(6, printf("%s(%p): try for va %#x pte1 %#x at %p\n", __func__, pmap, va, pte1_load(pte1p), pte1p)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Examine the first PTE2 in the specified PT2. Abort if this PTE2 is * either invalid, unused, or does not map the first 4KB physical page * within a 1MB page. */ fpte2p = pmap_pte2_quick(pmap, pte1_trunc(va)); fpte2 = pte2_load(fpte2p); if ((fpte2 & ((PTE2_FRAME & PTE1_OFFSET) | PTE2_A | PTE2_V)) != (PTE2_A | PTE2_V)) { pmap_pte1_p_failures++; CTR3(KTR_PMAP, "%s: failure(1) for va %#x in pmap %p", __func__, va, pmap); return; } if (pte2_is_managed(fpte2) && pmap == kernel_pmap) { pmap_pte1_p_failures++; CTR3(KTR_PMAP, "%s: failure(2) for va %#x in pmap %p", __func__, va, pmap); return; } if ((fpte2 & (PTE2_NM | PTE2_RO)) == PTE2_NM) { /* * When page is not modified, PTE2_RO can be set without * a TLB invalidation. */ fpte2 |= PTE2_RO; pte2_store(fpte2p, fpte2); } /* * Examine each of the other PTE2s in the specified PT2. Abort if this * PTE2 maps an unexpected 4KB physical page or does not have identical * characteristics to the first PTE2. */ fpte2_fav = (fpte2 & (PTE2_FRAME | PTE2_A | PTE2_V)); fpte2_fav += PTE1_SIZE - PTE2_SIZE; /* examine from the end */ for (pte2p = fpte2p + NPTE2_IN_PT2 - 1; pte2p > fpte2p; pte2p--) { pte2 = pte2_load(pte2p); if ((pte2 & (PTE2_FRAME | PTE2_A | PTE2_V)) != fpte2_fav) { pmap_pte1_p_failures++; CTR3(KTR_PMAP, "%s: failure(3) for va %#x in pmap %p", __func__, va, pmap); return; } if ((pte2 & (PTE2_NM | PTE2_RO)) == PTE2_NM) { /* * When page is not modified, PTE2_RO can be set * without a TLB invalidation. See note above. */ pte2 |= PTE2_RO; pte2_store(pte2p, pte2); pteva = pte1_trunc(va) | (pte2 & PTE1_OFFSET & PTE2_FRAME); CTR3(KTR_PMAP, "%s: protect for va %#x in pmap %p", __func__, pteva, pmap); } if ((pte2 & PTE2_PROMOTE) != (fpte2 & PTE2_PROMOTE)) { pmap_pte1_p_failures++; CTR3(KTR_PMAP, "%s: failure(4) for va %#x in pmap %p", __func__, va, pmap); return; } fpte2_fav -= PTE2_SIZE; } /* * The page table page in its current state will stay in PT2TAB * until the PTE1 mapping the section is demoted by pmap_demote_pte1() * or destroyed by pmap_remove_pte1(). * * Note that L2 page table size is not equal to PAGE_SIZE. */ m = PHYS_TO_VM_PAGE(trunc_page(pte1_link_pa(pte1_load(pte1p)))); KASSERT(m >= vm_page_array && m < &vm_page_array[vm_page_array_size], ("%s: PT2 page is out of range", __func__)); KASSERT(m->pindex == (pte1_index(va) & ~PT2PG_MASK), ("%s: PT2 page's pindex is wrong", __func__)); /* * Get pte1 from pte2 format. */ npte1 = (fpte2 & PTE1_FRAME) | ATTR_TO_L1(fpte2) | PTE1_V; /* * Promote the pv entries. */ if (pte2_is_managed(fpte2)) pmap_pv_promote_pte1(pmap, va, pte1_pa(npte1)); /* * Promote the mappings. */ pmap_change_pte1(pmap, pte1p, va, npte1); pmap_pte1_promotions++; CTR3(KTR_PMAP, "%s: success for va %#x in pmap %p", __func__, va, pmap); PDEBUG(6, printf("%s(%p): success for va %#x pte1 %#x(%#x) at %p\n", __func__, pmap, va, npte1, pte1_load(pte1p), pte1p)); } /* * Zero L2 page table page. */ static __inline void pmap_clear_pt2(pt2_entry_t *fpte2p) { pt2_entry_t *pte2p; for (pte2p = fpte2p; pte2p < fpte2p + NPTE2_IN_PT2; pte2p++) pte2_clear(pte2p); } /* * Removes a 1MB page mapping from the kernel pmap. */ static void pmap_remove_kernel_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va) { vm_page_t m; uint32_t pte1_idx; pt2_entry_t *fpte2p; vm_paddr_t pt2_pa; PMAP_LOCK_ASSERT(pmap, MA_OWNED); m = pmap_pt2_page(pmap, va); if (m == NULL) /* * QQQ: Is this function called only on promoted pte1? * We certainly do section mappings directly * (without promotion) in kernel !!! */ panic("%s: missing pt2 page", __func__); pte1_idx = pte1_index(va); /* * Initialize the L2 page table. */ fpte2p = page_pt2(pt2map_pt2pg(va), pte1_idx); pmap_clear_pt2(fpte2p); /* * Remove the mapping. */ pt2_pa = page_pt2pa(VM_PAGE_TO_PHYS(m), pte1_idx); pmap_kenter_pte1(va, PTE1_LINK(pt2_pa)); /* * QQQ: We do not need to invalidate PT2MAP mapping * as we did not change it. I.e. the L2 page table page * was and still is mapped the same way. */ } /* * Do the things to unmap a section in a process */ static void pmap_remove_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva, struct spglist *free) { pt1_entry_t opte1; struct md_page *pvh; vm_offset_t eva, va; vm_page_t m; PDEBUG(6, printf("%s(%p): va %#x pte1 %#x at %p\n", __func__, pmap, sva, pte1_load(pte1p), pte1p)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((sva & PTE1_OFFSET) == 0, ("%s: sva is not 1mpage aligned", __func__)); /* * Clear and invalidate the mapping. It should occupy one and only TLB * entry. So, pmap_tlb_flush() called with aligned address should be * sufficient. */ opte1 = pte1_load_clear(pte1p); pmap_tlb_flush(pmap, sva); if (pte1_is_wired(opte1)) pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE; pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE; if (pte1_is_managed(opte1)) { pvh = pa_to_pvh(pte1_pa(opte1)); pmap_pvh_free(pvh, pmap, sva); eva = sva + PTE1_SIZE; for (va = sva, m = PHYS_TO_VM_PAGE(pte1_pa(opte1)); va < eva; va += PAGE_SIZE, m++) { if (pte1_is_dirty(opte1)) vm_page_dirty(m); if (opte1 & PTE1_A) vm_page_aflag_set(m, PGA_REFERENCED); if (TAILQ_EMPTY(&m->md.pv_list) && TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } } if (pmap == kernel_pmap) { /* * L2 page table(s) can't be removed from kernel map as * kernel counts on it (stuff around pmap_growkernel()). */ pmap_remove_kernel_pte1(pmap, pte1p, sva); } else { /* * Get associated L2 page table page. * It's possible that the page was never allocated. */ m = pmap_pt2_page(pmap, sva); if (m != NULL) pmap_unwire_pt2_all(pmap, sva, m, free); } } /* * Fills L2 page table page with mappings to consecutive physical pages. */ static __inline void pmap_fill_pt2(pt2_entry_t *fpte2p, pt2_entry_t npte2) { pt2_entry_t *pte2p; for (pte2p = fpte2p; pte2p < fpte2p + NPTE2_IN_PT2; pte2p++) { pte2_store(pte2p, npte2); npte2 += PTE2_SIZE; } } /* * Tries to demote a 1MB page mapping. If demotion fails, the * 1MB page mapping is invalidated. */ static boolean_t pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va) { pt1_entry_t opte1, npte1; pt2_entry_t *fpte2p, npte2; vm_paddr_t pt2pg_pa, pt2_pa; vm_page_t m; struct spglist free; uint32_t pte1_idx, isnew = 0; PDEBUG(6, printf("%s(%p): try for va %#x pte1 %#x at %p\n", __func__, pmap, va, pte1_load(pte1p), pte1p)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); opte1 = pte1_load(pte1p); KASSERT(pte1_is_section(opte1), ("%s: opte1 not a section", __func__)); if ((opte1 & PTE1_A) == 0 || (m = pmap_pt2_page(pmap, va)) == NULL) { KASSERT(!pte1_is_wired(opte1), ("%s: PT2 page for a wired mapping is missing", __func__)); /* * Invalidate the 1MB page mapping and return * "failure" if the mapping was never accessed or the * allocation of the new page table page fails. */ if ((opte1 & PTE1_A) == 0 || (m = vm_page_alloc(NULL, pte1_index(va) & ~PT2PG_MASK, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) { SLIST_INIT(&free); pmap_remove_pte1(pmap, pte1p, pte1_trunc(va), &free); pmap_free_zero_pages(&free); CTR3(KTR_PMAP, "%s: failure for va %#x in pmap %p", __func__, va, pmap); return (FALSE); } if (va < VM_MAXUSER_ADDRESS) pmap->pm_stats.resident_count++; isnew = 1; /* * We init all L2 page tables in the page even if * we are going to change everything for one L2 page * table in a while. */ pt2pg_pa = pmap_pt2pg_init(pmap, va, m); } else { if (va < VM_MAXUSER_ADDRESS) { if (pt2_is_empty(m, va)) isnew = 1; /* Demoting section w/o promotion. */ #ifdef INVARIANTS else KASSERT(pt2_is_full(m, va), ("%s: bad PT2 wire" " count %u", __func__, pt2_wirecount_get(m, pte1_index(va)))); #endif } } pt2pg_pa = VM_PAGE_TO_PHYS(m); pte1_idx = pte1_index(va); /* * If the pmap is current, then the PT2MAP can provide access to * the page table page (promoted L2 page tables are not unmapped). * Otherwise, temporarily map the L2 page table page (m) into * the kernel's address space at either PADDR1 or PADDR2. * * Note that L2 page table size is not equal to PAGE_SIZE. */ if (pmap_is_current(pmap)) fpte2p = page_pt2(pt2map_pt2pg(va), pte1_idx); else if (curthread->td_pinned > 0 && rw_wowned(&pvh_global_lock)) { if (pte2_pa(pte2_load(PMAP1)) != pt2pg_pa) { pte2_store(PMAP1, PTE2_KPT(pt2pg_pa)); #ifdef SMP PMAP1cpu = PCPU_GET(cpuid); #endif tlb_flush_local((vm_offset_t)PADDR1); PMAP1changed++; } else #ifdef SMP if (PMAP1cpu != PCPU_GET(cpuid)) { PMAP1cpu = PCPU_GET(cpuid); tlb_flush_local((vm_offset_t)PADDR1); PMAP1changedcpu++; } else #endif PMAP1unchanged++; fpte2p = page_pt2((vm_offset_t)PADDR1, pte1_idx); } else { mtx_lock(&PMAP2mutex); if (pte2_pa(pte2_load(PMAP2)) != pt2pg_pa) { pte2_store(PMAP2, PTE2_KPT(pt2pg_pa)); tlb_flush((vm_offset_t)PADDR2); } fpte2p = page_pt2((vm_offset_t)PADDR2, pte1_idx); } pt2_pa = page_pt2pa(pt2pg_pa, pte1_idx); npte1 = PTE1_LINK(pt2_pa); KASSERT((opte1 & PTE1_A) != 0, ("%s: opte1 is missing PTE1_A", __func__)); KASSERT((opte1 & (PTE1_NM | PTE1_RO)) != PTE1_NM, ("%s: opte1 has PTE1_NM", __func__)); /* * Get pte2 from pte1 format. */ npte2 = pte1_pa(opte1) | ATTR_TO_L2(opte1) | PTE2_V; /* * If the L2 page table page is new, initialize it. If the mapping * has changed attributes, update the page table entries. */ if (isnew != 0) { pt2_wirecount_set(m, pte1_idx, NPTE2_IN_PT2); pmap_fill_pt2(fpte2p, npte2); } else if ((pte2_load(fpte2p) & PTE2_PROMOTE) != (npte2 & PTE2_PROMOTE)) pmap_fill_pt2(fpte2p, npte2); KASSERT(pte2_pa(pte2_load(fpte2p)) == pte2_pa(npte2), ("%s: fpte2p and npte2 map different physical addresses", __func__)); if (fpte2p == PADDR2) mtx_unlock(&PMAP2mutex); /* * Demote the mapping. This pmap is locked. The old PTE1 has * PTE1_A set. If the old PTE1 has not PTE1_RO set, it also * has not PTE1_NM set. Thus, there is no danger of a race with * another processor changing the setting of PTE1_A and/or PTE1_NM * between the read above and the store below. */ pmap_change_pte1(pmap, pte1p, va, npte1); /* * Demote the pv entry. This depends on the earlier demotion * of the mapping. Specifically, the (re)creation of a per- * page pv entry might trigger the execution of pmap_pv_reclaim(), * which might reclaim a newly (re)created per-page pv entry * and destroy the associated mapping. In order to destroy * the mapping, the PTE1 must have already changed from mapping * the 1mpage to referencing the page table page. */ if (pte1_is_managed(opte1)) pmap_pv_demote_pte1(pmap, va, pte1_pa(opte1)); pmap_pte1_demotions++; CTR3(KTR_PMAP, "%s: success for va %#x in pmap %p", __func__, va, pmap); PDEBUG(6, printf("%s(%p): success for va %#x pte1 %#x(%#x) at %p\n", __func__, pmap, va, npte1, pte1_load(pte1p), pte1p)); return (TRUE); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind) { pt1_entry_t *pte1p; pt2_entry_t *pte2p; pt2_entry_t npte2, opte2; pv_entry_t pv; vm_paddr_t opa, pa; vm_page_t mpte2, om; boolean_t wired; va = trunc_page(va); mpte2 = NULL; wired = (flags & PMAP_ENTER_WIRED) != 0; KASSERT(va <= vm_max_kernel_address, ("%s: toobig", __func__)); KASSERT(va < UPT2V_MIN_ADDRESS || va >= UPT2V_MAX_ADDRESS, ("%s: invalid to pmap_enter page table pages (va: 0x%x)", __func__, va)); if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); sched_pin(); /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < VM_MAXUSER_ADDRESS) { mpte2 = pmap_allocpte2(pmap, va, flags); if (mpte2 == NULL) { KASSERT((flags & PMAP_ENTER_NOSLEEP) != 0, ("pmap_allocpte2 failed with sleep allowed")); sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (KERN_RESOURCE_SHORTAGE); } } pte1p = pmap_pte1(pmap, va); if (pte1_is_section(pte1_load(pte1p))) panic("%s: attempted on 1MB page", __func__); pte2p = pmap_pte2_quick(pmap, va); if (pte2p == NULL) panic("%s: invalid L1 page table entry va=%#x", __func__, va); om = NULL; pa = VM_PAGE_TO_PHYS(m); opte2 = pte2_load(pte2p); opa = pte2_pa(opte2); /* * Mapping has not changed, must be protection or wiring change. */ if (pte2_is_valid(opte2) && (opa == pa)) { /* * Wiring change, just update stats. We don't worry about * wiring PT2 pages as they remain resident as long as there * are valid mappings in them. Hence, if a user page is wired, * the PT2 page will be also. */ if (wired && !pte2_is_wired(opte2)) pmap->pm_stats.wired_count++; else if (!wired && pte2_is_wired(opte2)) pmap->pm_stats.wired_count--; /* * Remove extra pte2 reference */ if (mpte2) pt2_wirecount_dec(mpte2, pte1_index(va)); if (pte2_is_managed(opte2)) om = m; goto validate; } /* * QQQ: We think that changing physical address on writeable mapping * is not safe. Well, maybe on kernel address space with correct * locking, it can make a sense. However, we have no idea why * anyone should do that on user address space. Are we wrong? */ KASSERT((opa == 0) || (opa == pa) || !pte2_is_valid(opte2) || ((opte2 & PTE2_RO) != 0), ("%s: pmap %p va %#x(%#x) opa %#x pa %#x - gotcha %#x %#x!", __func__, pmap, va, opte2, opa, pa, flags, prot)); pv = NULL; /* * Mapping has changed, invalidate old range and fall through to * handle validating new mapping. */ if (opa) { if (pte2_is_wired(opte2)) pmap->pm_stats.wired_count--; if (pte2_is_managed(opte2)) { om = PHYS_TO_VM_PAGE(opa); pv = pmap_pvh_remove(&om->md, pmap, va); } /* * Remove extra pte2 reference */ if (mpte2 != NULL) pt2_wirecount_dec(mpte2, va >> PTE1_SHIFT); } else pmap->pm_stats.resident_count++; /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0) { KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, ("%s: managed mapping within the clean submap", __func__)); if (pv == NULL) pv = get_pv_entry(pmap, FALSE); pv->pv_va = va; TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); } else if (pv != NULL) free_pv_entry(pmap, pv); /* * Increment counters */ if (wired) pmap->pm_stats.wired_count++; validate: /* * Now validate mapping with desired protection/wiring. */ npte2 = PTE2(pa, PTE2_NM, vm_page_pte2_attr(m)); if (prot & VM_PROT_WRITE) { if (pte2_is_managed(npte2)) vm_page_aflag_set(m, PGA_WRITEABLE); } else npte2 |= PTE2_RO; if ((prot & VM_PROT_EXECUTE) == 0) npte2 |= PTE2_NX; if (wired) npte2 |= PTE2_W; if (va < VM_MAXUSER_ADDRESS) npte2 |= PTE2_U; if (pmap != kernel_pmap) npte2 |= PTE2_NG; /* * If the mapping or permission bits are different, we need * to update the pte2. * * QQQ: Think again and again what to do * if the mapping is going to be changed! */ if ((opte2 & ~(PTE2_NM | PTE2_A)) != (npte2 & ~(PTE2_NM | PTE2_A))) { /* * Sync icache if exec permission and attribute VM_MEMATTR_WB_WA * is set. Do it now, before the mapping is stored and made * valid for hardware table walk. If done later, there is a race * for other threads of current process in lazy loading case. * Don't do it for kernel memory which is mapped with exec * permission even if the memory isn't going to hold executable * code. The only time when icache sync is needed is after * kernel module is loaded and the relocation info is processed. * And it's done in elf_cpu_load_file(). * * QQQ: (1) Does it exist any better way where * or how to sync icache? * (2) Now, we do it on a page basis. */ if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap && m->md.pat_mode == VM_MEMATTR_WB_WA && (opa != pa || (opte2 & PTE2_NX))) cache_icache_sync_fresh(va, pa, PAGE_SIZE); npte2 |= PTE2_A; if (flags & VM_PROT_WRITE) npte2 &= ~PTE2_NM; if (opte2 & PTE2_V) { /* Change mapping with break-before-make approach. */ opte2 = pte2_load_clear(pte2p); pmap_tlb_flush(pmap, va); pte2_store(pte2p, npte2); if (opte2 & PTE2_A) { if (pte2_is_managed(opte2)) vm_page_aflag_set(om, PGA_REFERENCED); } if (pte2_is_dirty(opte2)) { if (pte2_is_managed(opte2)) vm_page_dirty(om); } if (pte2_is_managed(opte2) && TAILQ_EMPTY(&om->md.pv_list) && ((om->flags & PG_FICTITIOUS) != 0 || TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) vm_page_aflag_clear(om, PGA_WRITEABLE); } else pte2_store(pte2p, npte2); } #if 0 else { /* * QQQ: In time when both access and not mofified bits are * emulated by software, this should not happen. Some * analysis is need, if this really happen. Missing * tlb flush somewhere could be the reason. */ panic("%s: pmap %p va %#x opte2 %x npte2 %x !!", __func__, pmap, va, opte2, npte2); } #endif /* * If both the L2 page table page and the reservation are fully * populated, then attempt promotion. */ if ((mpte2 == NULL || pt2_is_full(mpte2, va)) && sp_enabled && (m->flags & PG_FICTITIOUS) == 0 && vm_reserv_level_iffullpop(m) == 0) pmap_promote_pte1(pmap, pte1p, va); sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } /* * Do the things to unmap a page in a process. */ static int pmap_remove_pte2(pmap_t pmap, pt2_entry_t *pte2p, vm_offset_t va, struct spglist *free) { pt2_entry_t opte2; vm_page_t m; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* Clear and invalidate the mapping. */ opte2 = pte2_load_clear(pte2p); pmap_tlb_flush(pmap, va); KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %#x not link pte2 %#x", __func__, pmap, va, opte2)); if (opte2 & PTE2_W) pmap->pm_stats.wired_count -= 1; pmap->pm_stats.resident_count -= 1; if (pte2_is_managed(opte2)) { m = PHYS_TO_VM_PAGE(pte2_pa(opte2)); if (pte2_is_dirty(opte2)) vm_page_dirty(m); if (opte2 & PTE2_A) vm_page_aflag_set(m, PGA_REFERENCED); pmap_remove_entry(pmap, m, va); } return (pmap_unuse_pt2(pmap, va, free)); } /* * Remove a single page from a process address space. */ static void pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free) { pt2_entry_t *pte2p; rw_assert(&pvh_global_lock, RA_WLOCKED); KASSERT(curthread->td_pinned > 0, ("%s: curthread not pinned", __func__)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); if ((pte2p = pmap_pte2_quick(pmap, va)) == NULL || !pte2_is_valid(pte2_load(pte2p))) return; pmap_remove_pte2(pmap, pte2p, va, free); } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ void pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t nextva; pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; struct spglist free; /* * Perform an unsynchronized read. This is, however, safe. */ if (pmap->pm_stats.resident_count == 0) return; SLIST_INIT(&free); rw_wlock(&pvh_global_lock); sched_pin(); PMAP_LOCK(pmap); /* * Special handling of removing one page. A very common * operation and easy to short circuit some code. */ if (sva + PAGE_SIZE == eva) { pte1 = pte1_load(pmap_pte1(pmap, sva)); if (pte1_is_link(pte1)) { pmap_remove_page(pmap, sva, &free); goto out; } } for (; sva < eva; sva = nextva) { /* * Calculate address for next L2 page table. */ nextva = pte1_trunc(sva + PTE1_SIZE); if (nextva < sva) nextva = eva; if (pmap->pm_stats.resident_count == 0) break; pte1p = pmap_pte1(pmap, sva); pte1 = pte1_load(pte1p); /* * Weed out invalid mappings. Note: we assume that the L1 page * table is always allocated, and in kernel virtual. */ if (pte1 == 0) continue; if (pte1_is_section(pte1)) { /* * Are we removing the entire large page? If not, * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { pmap_remove_pte1(pmap, pte1p, sva, &free); continue; } else if (!pmap_demote_pte1(pmap, pte1p, sva)) { /* The large page mapping was destroyed. */ continue; } #ifdef INVARIANTS else { /* Update pte1 after demotion. */ pte1 = pte1_load(pte1p); } #endif } KASSERT(pte1_is_link(pte1), ("%s: pmap %p va %#x pte1 %#x at %p" " is not link", __func__, pmap, sva, pte1, pte1p)); /* * Limit our scan to either the end of the va represented * by the current L2 page table page, or to the end of the * range being removed. */ if (nextva > eva) nextva = eva; for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++, sva += PAGE_SIZE) { pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2)) continue; if (pmap_remove_pte2(pmap, pte2p, sva, &free)) break; } } out: sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ void pmap_remove_all(vm_page_t m) { struct md_page *pvh; pv_entry_t pv; pmap_t pmap; pt2_entry_t *pte2p, opte2; pt1_entry_t *pte1p; vm_offset_t va; struct spglist free; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); SLIST_INIT(&free); rw_wlock(&pvh_global_lock); sched_pin(); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, va); (void)pmap_demote_pte1(pmap, pte1p, va); PMAP_UNLOCK(pmap); } small_mappings: while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pmap->pm_stats.resident_count--; pte1p = pmap_pte1(pmap, pv->pv_va); KASSERT(!pte1_is_section(pte1_load(pte1p)), ("%s: found " "a 1mpage in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); opte2 = pte2_load_clear(pte2p); pmap_tlb_flush(pmap, pv->pv_va); KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %x zero pte2", __func__, pmap, pv->pv_va)); if (pte2_is_wired(opte2)) pmap->pm_stats.wired_count--; if (opte2 & PTE2_A) vm_page_aflag_set(m, PGA_REFERENCED); /* * Update the vm_page_t clean and reference bits. */ if (pte2_is_dirty(opte2)) vm_page_dirty(m); pmap_unuse_pt2(pmap, pv->pv_va, &free); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); sched_unpin(); rw_wunlock(&pvh_global_lock); pmap_free_zero_pages(&free); } /* * Just subroutine for pmap_remove_pages() to reasonably satisfy * good coding style, a.k.a. 80 character line width limit hell. */ static __inline void pmap_remove_pte1_quick(pmap_t pmap, pt1_entry_t pte1, pv_entry_t pv, struct spglist *free) { vm_paddr_t pa; vm_page_t m, mt, mpt2pg; struct md_page *pvh; pa = pte1_pa(pte1); m = PHYS_TO_VM_PAGE(pa); KASSERT(m->phys_addr == pa, ("%s: vm_page_t %p addr mismatch %#x %#x", __func__, m, m->phys_addr, pa)); KASSERT((m->flags & PG_FICTITIOUS) != 0 || m < &vm_page_array[vm_page_array_size], ("%s: bad pte1 %#x", __func__, pte1)); if (pte1_is_dirty(pte1)) { for (mt = m; mt < &m[PTE1_SIZE / PAGE_SIZE]; mt++) vm_page_dirty(mt); } pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE; pvh = pa_to_pvh(pa); TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); if (TAILQ_EMPTY(&pvh->pv_list)) { for (mt = m; mt < &m[PTE1_SIZE / PAGE_SIZE]; mt++) if (TAILQ_EMPTY(&mt->md.pv_list)) vm_page_aflag_clear(mt, PGA_WRITEABLE); } mpt2pg = pmap_pt2_page(pmap, pv->pv_va); if (mpt2pg != NULL) pmap_unwire_pt2_all(pmap, pv->pv_va, mpt2pg, free); } /* * Just subroutine for pmap_remove_pages() to reasonably satisfy * good coding style, a.k.a. 80 character line width limit hell. */ static __inline void pmap_remove_pte2_quick(pmap_t pmap, pt2_entry_t pte2, pv_entry_t pv, struct spglist *free) { vm_paddr_t pa; vm_page_t m; struct md_page *pvh; pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pa); KASSERT(m->phys_addr == pa, ("%s: vm_page_t %p addr mismatch %#x %#x", __func__, m, m->phys_addr, pa)); KASSERT((m->flags & PG_FICTITIOUS) != 0 || m < &vm_page_array[vm_page_array_size], ("%s: bad pte2 %#x", __func__, pte2)); if (pte2_is_dirty(pte2)) vm_page_dirty(m); pmap->pm_stats.resident_count--; TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(pa); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } pmap_unuse_pt2(pmap, pv->pv_va, free); } /* * Remove all pages from specified address space this aids process * exit speeds. Also, this code is special cased for current process * only, but can have the more generic (and slightly slower) mode enabled. * This is much faster than pmap_remove in the case of running down * an entire address space. */ void pmap_remove_pages(pmap_t pmap) { pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; pv_entry_t pv; struct pv_chunk *pc, *npc; struct spglist free; int field, idx; int32_t bit; uint32_t inuse, bitmask; boolean_t allfree; /* * Assert that the given pmap is only active on the current * CPU. Unfortunately, we cannot block another CPU from * activating the pmap while this function is executing. */ KASSERT(pmap == vmspace_pmap(curthread->td_proc->p_vmspace), ("%s: non-current pmap %p", __func__, pmap)); #if defined(SMP) && defined(INVARIANTS) { cpuset_t other_cpus; sched_pin(); other_cpus = pmap->pm_active; CPU_CLR(PCPU_GET(cpuid), &other_cpus); sched_unpin(); KASSERT(CPU_EMPTY(&other_cpus), ("%s: pmap %p active on other cpus", __func__, pmap)); } #endif SLIST_INIT(&free); rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); sched_pin(); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { KASSERT(pc->pc_pmap == pmap, ("%s: wrong pmap %p %p", __func__, pmap, pc->pc_pmap)); allfree = TRUE; for (field = 0; field < _NPCM; field++) { inuse = (~(pc->pc_map[field])) & pc_freemask[field]; while (inuse != 0) { bit = ffs(inuse) - 1; bitmask = 1UL << bit; idx = field * 32 + bit; pv = &pc->pc_pventry[idx]; inuse &= ~bitmask; /* * Note that we cannot remove wired pages * from a process' mapping at this time */ pte1p = pmap_pte1(pmap, pv->pv_va); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1)) { if (pte1_is_wired(pte1)) { allfree = FALSE; continue; } pte1_clear(pte1p); pmap_remove_pte1_quick(pmap, pte1, pv, &free); } else if (pte1_is_link(pte1)) { pte2p = pt2map_entry(pv->pv_va); pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2)) { printf("%s: pmap %p va %#x " "pte2 %#x\n", __func__, pmap, pv->pv_va, pte2); panic("bad pte2"); } if (pte2_is_wired(pte2)) { allfree = FALSE; continue; } pte2_clear(pte2p); pmap_remove_pte2_quick(pmap, pte2, pv, &free); } else { printf("%s: pmap %p va %#x pte1 %#x\n", __func__, pmap, pv->pv_va, pte1); panic("bad pte1"); } /* Mark free */ PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); pv_entry_count--; pc->pc_map[field] |= bitmask; } } if (allfree) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } } tlb_flush_all_ng_local(); sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * This code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No L2 page table pages. * but is *MUCH* faster than pmap_enter... */ static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpt2pg) { pt2_entry_t *pte2p, pte2; vm_paddr_t pa; struct spglist free; uint32_t l2prot; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->oflags & VPO_UNMANAGED) != 0, ("%s: managed mapping within the clean submap", __func__)); rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * In the case that a L2 page table page is not * resident, we are creating it here. */ if (va < VM_MAXUSER_ADDRESS) { u_int pte1_idx; pt1_entry_t pte1, *pte1p; vm_paddr_t pt2_pa; /* * Get L1 page table things. */ pte1_idx = pte1_index(va); pte1p = pmap_pte1(pmap, va); pte1 = pte1_load(pte1p); if (mpt2pg && (mpt2pg->pindex == (pte1_idx & ~PT2PG_MASK))) { /* * Each of NPT2_IN_PG L2 page tables on the page can * come here. Make sure that associated L1 page table * link is established. * * QQQ: It comes that we don't establish all links to * L2 page tables for newly allocated L2 page * tables page. */ KASSERT(!pte1_is_section(pte1), ("%s: pte1 %#x is section", __func__, pte1)); if (!pte1_is_link(pte1)) { pt2_pa = page_pt2pa(VM_PAGE_TO_PHYS(mpt2pg), pte1_idx); pte1_store(pte1p, PTE1_LINK(pt2_pa)); } pt2_wirecount_inc(mpt2pg, pte1_idx); } else { /* * If the L2 page table page is mapped, we just * increment the hold count, and activate it. */ if (pte1_is_section(pte1)) { return (NULL); } else if (pte1_is_link(pte1)) { mpt2pg = PHYS_TO_VM_PAGE(pte1_link_pa(pte1)); pt2_wirecount_inc(mpt2pg, pte1_idx); } else { mpt2pg = _pmap_allocpte2(pmap, va, PMAP_ENTER_NOSLEEP); if (mpt2pg == NULL) return (NULL); } } } else { mpt2pg = NULL; } /* * This call to pt2map_entry() makes the assumption that we are * entering the page into the current pmap. In order to support * quick entry into any pmap, one would likely use pmap_pte2_quick(). * But that isn't as quick as pt2map_entry(). */ pte2p = pt2map_entry(va); pte2 = pte2_load(pte2p); if (pte2_is_valid(pte2)) { if (mpt2pg != NULL) { /* * Remove extra pte2 reference */ pt2_wirecount_dec(mpt2pg, pte1_index(va)); mpt2pg = NULL; } return (NULL); } /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0 && !pmap_try_insert_pv_entry(pmap, va, m)) { if (mpt2pg != NULL) { SLIST_INIT(&free); if (pmap_unwire_pt2(pmap, va, mpt2pg, &free)) { pmap_tlb_flush(pmap, va); pmap_free_zero_pages(&free); } mpt2pg = NULL; } return (NULL); } /* * Increment counters */ pmap->pm_stats.resident_count++; /* * Now validate mapping with RO protection */ pa = VM_PAGE_TO_PHYS(m); l2prot = PTE2_RO | PTE2_NM; if (va < VM_MAXUSER_ADDRESS) l2prot |= PTE2_U | PTE2_NG; if ((prot & VM_PROT_EXECUTE) == 0) l2prot |= PTE2_NX; else if (m->md.pat_mode == VM_MEMATTR_WB_WA && pmap != kernel_pmap) { /* * Sync icache if exec permission and attribute VM_MEMATTR_WB_WA * is set. QQQ: For more info, see comments in pmap_enter(). */ cache_icache_sync_fresh(va, pa, PAGE_SIZE); } pte2_store(pte2p, PTE2(pa, l2prot, vm_page_pte2_attr(m))); return (mpt2pg); } void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Tries to create 1MB page mapping. Returns TRUE if successful and * FALSE otherwise. Fails if (1) a page table page cannot be allocated without * blocking, (2) a mapping already exists at the specified virtual address, or * (3) a pv entry cannot be allocated without reclaiming another pv entry. */ static boolean_t pmap_enter_pte1(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { pt1_entry_t *pte1p; vm_paddr_t pa; uint32_t l1prot; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); pte1p = pmap_pte1(pmap, va); if (pte1_is_valid(pte1_load(pte1p))) { CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p", __func__, va, pmap); return (FALSE); } if ((m->oflags & VPO_UNMANAGED) == 0) { /* * Abort this mapping if its PV entry could not be created. */ if (!pmap_pv_insert_pte1(pmap, va, VM_PAGE_TO_PHYS(m))) { CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p", __func__, va, pmap); return (FALSE); } } /* * Increment counters. */ pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE; /* * Map the section. * * QQQ: Why VM_PROT_WRITE is not evaluated and the mapping is * made readonly? */ pa = VM_PAGE_TO_PHYS(m); l1prot = PTE1_RO | PTE1_NM; if (va < VM_MAXUSER_ADDRESS) l1prot |= PTE1_U | PTE1_NG; if ((prot & VM_PROT_EXECUTE) == 0) l1prot |= PTE1_NX; else if (m->md.pat_mode == VM_MEMATTR_WB_WA && pmap != kernel_pmap) { /* * Sync icache if exec permission and attribute VM_MEMATTR_WB_WA * is set. QQQ: For more info, see comments in pmap_enter(). */ cache_icache_sync_fresh(va, pa, PTE1_SIZE); } pte1_store(pte1p, PTE1(pa, l1prot, ATTR_TO_L1(vm_page_pte2_attr(m)))); pmap_pte1_mappings++; CTR3(KTR_PMAP, "%s: success for va %#lx in pmap %p", __func__, va, pmap); return (TRUE); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_offset_t va; vm_page_t m, mpt2pg; vm_pindex_t diff, psize; PDEBUG(6, printf("%s: pmap %p start %#x end %#x m %p prot %#x\n", __func__, pmap, start, end, m_start, prot)); VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); mpt2pg = NULL; m = m_start; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { va = start + ptoa(diff); if ((va & PTE1_OFFSET) == 0 && va + PTE1_SIZE <= end && m->psind == 1 && sp_enabled && pmap_enter_pte1(pmap, va, m, prot)) m = &m[PTE1_SIZE / PAGE_SIZE - 1]; else mpt2pg = pmap_enter_quick_locked(pmap, va, m, prot, mpt2pg); m = TAILQ_NEXT(m, listq); } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * This code maps large physical mmap regions into the * processor address space. Note that some shortcuts * are taken, but the code works. */ void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { pt1_entry_t *pte1p; vm_paddr_t pa, pte2_pa; vm_page_t p; vm_memattr_t pat_mode; u_int l1attr, l1prot; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("%s: non-device object", __func__)); if ((addr & PTE1_OFFSET) == 0 && (size & PTE1_OFFSET) == 0) { if (!vm_object_populate(object, pindex, pindex + atop(size))) return; p = vm_page_lookup(object, pindex); KASSERT(p->valid == VM_PAGE_BITS_ALL, ("%s: invalid page %p", __func__, p)); pat_mode = p->md.pat_mode; /* * Abort the mapping if the first page is not physically * aligned to a 1MB page boundary. */ pte2_pa = VM_PAGE_TO_PHYS(p); if (pte2_pa & PTE1_OFFSET) return; /* * Skip the first page. Abort the mapping if the rest of * the pages are not physically contiguous or have differing * memory attributes. */ p = TAILQ_NEXT(p, listq); for (pa = pte2_pa + PAGE_SIZE; pa < pte2_pa + size; pa += PAGE_SIZE) { KASSERT(p->valid == VM_PAGE_BITS_ALL, ("%s: invalid page %p", __func__, p)); if (pa != VM_PAGE_TO_PHYS(p) || pat_mode != p->md.pat_mode) return; p = TAILQ_NEXT(p, listq); } /* * Map using 1MB pages. * * QQQ: Well, we are mapping a section, so same condition must * be hold like during promotion. It looks that only RW mapping * is done here, so readonly mapping must be done elsewhere. */ l1prot = PTE1_U | PTE1_NG | PTE1_RW | PTE1_M | PTE1_A; l1attr = ATTR_TO_L1(vm_memattr_to_pte2(pat_mode)); PMAP_LOCK(pmap); for (pa = pte2_pa; pa < pte2_pa + size; pa += PTE1_SIZE) { pte1p = pmap_pte1(pmap, addr); if (!pte1_is_valid(pte1_load(pte1p))) { pte1_store(pte1p, PTE1(pa, l1prot, l1attr)); pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE; pmap_pte1_mappings++; } /* Else continue on if the PTE1 is already valid. */ addr += PTE1_SIZE; } PMAP_UNLOCK(pmap); } } /* * Do the things to protect a 1mpage in a process. */ static void pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva, vm_prot_t prot) { pt1_entry_t npte1, opte1; vm_offset_t eva, va; vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((sva & PTE1_OFFSET) == 0, ("%s: sva is not 1mpage aligned", __func__)); opte1 = npte1 = pte1_load(pte1p); if (pte1_is_managed(opte1)) { eva = sva + PTE1_SIZE; for (va = sva, m = PHYS_TO_VM_PAGE(pte1_pa(opte1)); va < eva; va += PAGE_SIZE, m++) if (pte1_is_dirty(opte1)) vm_page_dirty(m); } if ((prot & VM_PROT_WRITE) == 0) npte1 |= PTE1_RO | PTE1_NM; if ((prot & VM_PROT_EXECUTE) == 0) npte1 |= PTE1_NX; /* * QQQ: Herein, execute permission is never set. * It only can be cleared. So, no icache * syncing is needed. */ if (npte1 != opte1) { pte1_store(pte1p, npte1); pmap_tlb_flush(pmap, sva); } } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { boolean_t pv_lists_locked; vm_offset_t nextva; pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, opte2, npte2; KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); if (prot == VM_PROT_NONE) { pmap_remove(pmap, sva, eva); return; } if ((prot & (VM_PROT_WRITE | VM_PROT_EXECUTE)) == (VM_PROT_WRITE | VM_PROT_EXECUTE)) return; if (pmap_is_current(pmap)) pv_lists_locked = FALSE; else { pv_lists_locked = TRUE; resume: rw_wlock(&pvh_global_lock); sched_pin(); } PMAP_LOCK(pmap); for (; sva < eva; sva = nextva) { /* * Calculate address for next L2 page table. */ nextva = pte1_trunc(sva + PTE1_SIZE); if (nextva < sva) nextva = eva; pte1p = pmap_pte1(pmap, sva); pte1 = pte1_load(pte1p); /* * Weed out invalid mappings. Note: we assume that L1 page * page table is always allocated, and in kernel virtual. */ if (pte1 == 0) continue; if (pte1_is_section(pte1)) { /* * Are we protecting the entire large page? If not, * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { pmap_protect_pte1(pmap, pte1p, sva, prot); continue; } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { PMAP_UNLOCK(pmap); goto resume; } sched_pin(); } if (!pmap_demote_pte1(pmap, pte1p, sva)) { /* * The large page mapping * was destroyed. */ continue; } #ifdef INVARIANTS else { /* Update pte1 after demotion */ pte1 = pte1_load(pte1p); } #endif } } KASSERT(pte1_is_link(pte1), ("%s: pmap %p va %#x pte1 %#x at %p" " is not link", __func__, pmap, sva, pte1, pte1p)); /* * Limit our scan to either the end of the va represented * by the current L2 page table page, or to the end of the * range being protected. */ if (nextva > eva) nextva = eva; for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++, sva += PAGE_SIZE) { vm_page_t m; opte2 = npte2 = pte2_load(pte2p); if (!pte2_is_valid(opte2)) continue; if ((prot & VM_PROT_WRITE) == 0) { if (pte2_is_managed(opte2) && pte2_is_dirty(opte2)) { m = PHYS_TO_VM_PAGE(pte2_pa(opte2)); vm_page_dirty(m); } npte2 |= PTE2_RO | PTE2_NM; } if ((prot & VM_PROT_EXECUTE) == 0) npte2 |= PTE2_NX; /* * QQQ: Herein, execute permission is never set. * It only can be cleared. So, no icache * syncing is needed. */ if (npte2 != opte2) { pte2_store(pte2p, npte2); pmap_tlb_flush(pmap, sva); } } } if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock); } PMAP_UNLOCK(pmap); } /* * pmap_pvh_wired_mappings: * * Return the updated number "count" of managed mappings that are wired. */ static int pmap_pvh_wired_mappings(struct md_page *pvh, int count) { pmap_t pmap; pt1_entry_t pte1; pt2_entry_t pte2; pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); sched_pin(); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va)); if (pte1_is_section(pte1)) { if (pte1_is_wired(pte1)) count++; } else { KASSERT(pte1_is_link(pte1), ("%s: pte1 %#x is not link", __func__, pte1)); pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va)); if (pte2_is_wired(pte2)) count++; } PMAP_UNLOCK(pmap); } sched_unpin(); return (count); } /* * pmap_page_wired_mappings: * * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { int count; count = 0; if ((m->oflags & VPO_UNMANAGED) != 0) return (count); rw_wlock(&pvh_global_lock); count = pmap_pvh_wired_mappings(&m->md, count); if ((m->flags & PG_FICTITIOUS) == 0) { count = pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)), count); } rw_wunlock(&pvh_global_lock); return (count); } /* * Returns TRUE if any of the given mappings were used to modify * physical memory. Otherwise, returns FALSE. Both page and 1mpage * mappings are supported. */ static boolean_t pmap_is_modified_pvh(struct md_page *pvh) { pv_entry_t pv; pt1_entry_t pte1; pt2_entry_t pte2; pmap_t pmap; boolean_t rv; rw_assert(&pvh_global_lock, RA_WLOCKED); rv = FALSE; sched_pin(); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va)); if (pte1_is_section(pte1)) { rv = pte1_is_dirty(pte1); } else { KASSERT(pte1_is_link(pte1), ("%s: pte1 %#x is not link", __func__, pte1)); pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va)); rv = pte2_is_dirty(pte2); } PMAP_UNLOCK(pmap); if (rv) break; } sched_unpin(); return (rv); } /* * pmap_is_modified: * * Return whether or not the specified physical page was modified * in any physical maps. */ boolean_t pmap_is_modified(vm_page_t m) { boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTE2s can have PG_M set. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); rw_wlock(&pvh_global_lock); rv = pmap_is_modified_pvh(&m->md) || ((m->flags & PG_FICTITIOUS) == 0 && pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m)))); rw_wunlock(&pvh_global_lock); return (rv); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is eligible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { pt1_entry_t pte1; pt2_entry_t pte2; boolean_t rv; rv = FALSE; PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, addr)); if (pte1_is_link(pte1)) { pte2 = pte2_load(pt2map_entry(addr)); rv = !pte2_is_valid(pte2) ; } PMAP_UNLOCK(pmap); return (rv); } /* * Returns TRUE if any of the given mappings were referenced and FALSE * otherwise. Both page and 1mpage mappings are supported. */ static boolean_t pmap_is_referenced_pvh(struct md_page *pvh) { pv_entry_t pv; pt1_entry_t pte1; pt2_entry_t pte2; pmap_t pmap; boolean_t rv; rw_assert(&pvh_global_lock, RA_WLOCKED); rv = FALSE; sched_pin(); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va)); if (pte1_is_section(pte1)) { rv = (pte1 & (PTE1_A | PTE1_V)) == (PTE1_A | PTE1_V); } else { pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va)); rv = (pte2 & (PTE2_A | PTE2_V)) == (PTE2_A | PTE2_V); } PMAP_UNLOCK(pmap); if (rv) break; } sched_unpin(); return (rv); } /* * pmap_is_referenced: * * Return whether or not the specified physical page was referenced * in any physical maps. */ boolean_t pmap_is_referenced(vm_page_t m) { boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); rw_wlock(&pvh_global_lock); rv = pmap_is_referenced_pvh(&m->md) || ((m->flags & PG_FICTITIOUS) == 0 && pmap_is_referenced_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m)))); rw_wunlock(&pvh_global_lock); return (rv); } #define PMAP_TS_REFERENCED_MAX 5 /* * pmap_ts_referenced: * * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ int pmap_ts_referenced(vm_page_t m) { struct md_page *pvh; pv_entry_t pv, pvf; pmap_t pmap; pt1_entry_t *pte1p, opte1; pt2_entry_t *pte2p; vm_paddr_t pa; int rtval = 0; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); pa = VM_PAGE_TO_PHYS(m); pvh = pa_to_pvh(pa); rw_wlock(&pvh_global_lock); sched_pin(); if ((m->flags & PG_FICTITIOUS) != 0 || (pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) goto small_mappings; pv = pvf; do { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, pv->pv_va); opte1 = pte1_load(pte1p); if ((opte1 & PTE1_A) != 0) { /* * Since this reference bit is shared by 256 4KB pages, * it should not be cleared every time it is tested. * Apply a simple "hash" function on the physical page * number, the virtual section number, and the pmap * address to select one 4KB page out of the 256 * on which testing the reference bit will result * in clearing that bit. This function is designed * to avoid the selection of the same 4KB page * for every 1MB page mapping. * * On demotion, a mapping that hasn't been referenced * is simply destroyed. To avoid the possibility of a * subsequent page fault on a demoted wired mapping, * always leave its reference bit set. Moreover, * since the section is wired, the current state of * its reference bit won't affect page replacement. */ if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PTE1_SHIFT) ^ (uintptr_t)pmap) & (NPTE2_IN_PG - 1)) == 0 && !pte1_is_wired(opte1)) { pte1_clear_bit(pte1p, PTE1_A); pmap_tlb_flush(pmap, pv->pv_va); } rtval++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); } if (rtval >= PMAP_TS_REFERENCED_MAX) goto out; } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); small_mappings: if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) goto out; pv = pvf; do { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, pv->pv_va); KASSERT(pte1_is_link(pte1_load(pte1p)), ("%s: not found a link in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); if ((pte2_load(pte2p) & PTE2_A) != 0) { pte2_clear_bit(pte2p, PTE2_A); pmap_tlb_flush(pmap, pv->pv_va); rtval++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); } } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && rtval < PMAP_TS_REFERENCED_MAX); out: sched_unpin(); rw_wunlock(&pvh_global_lock); return (rtval); } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range * must have the wired attribute set. In contrast, invalid mappings * cannot have the wired attribute set, so they are ignored. * * The wired attribute of the page table entry is not a hardware feature, * so there is no need to invalidate any TLB entries. */ void pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t nextva; pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; boolean_t pv_lists_locked; if (pmap_is_current(pmap)) pv_lists_locked = FALSE; else { pv_lists_locked = TRUE; resume: rw_wlock(&pvh_global_lock); sched_pin(); } PMAP_LOCK(pmap); for (; sva < eva; sva = nextva) { nextva = pte1_trunc(sva + PTE1_SIZE); if (nextva < sva) nextva = eva; pte1p = pmap_pte1(pmap, sva); pte1 = pte1_load(pte1p); /* * Weed out invalid mappings. Note: we assume that L1 page * page table is always allocated, and in kernel virtual. */ if (pte1 == 0) continue; if (pte1_is_section(pte1)) { if (!pte1_is_wired(pte1)) panic("%s: pte1 %#x not wired", __func__, pte1); /* * Are we unwiring the entire large page? If not, * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { pte1_clear_bit(pte1p, PTE1_W); pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE; continue; } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { PMAP_UNLOCK(pmap); /* Repeat sva. */ goto resume; } sched_pin(); } if (!pmap_demote_pte1(pmap, pte1p, sva)) panic("%s: demotion failed", __func__); #ifdef INVARIANTS else { /* Update pte1 after demotion */ pte1 = pte1_load(pte1p); } #endif } } KASSERT(pte1_is_link(pte1), ("%s: pmap %p va %#x pte1 %#x at %p" " is not link", __func__, pmap, sva, pte1, pte1p)); /* * Limit our scan to either the end of the va represented * by the current L2 page table page, or to the end of the * range being protected. */ if (nextva > eva) nextva = eva; for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++, sva += PAGE_SIZE) { pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2)) continue; if (!pte2_is_wired(pte2)) panic("%s: pte2 %#x is missing PTE2_W", __func__, pte2); /* * PTE2_W must be cleared atomically. Although the pmap * lock synchronizes access to PTE2_W, another processor * could be changing PTE2_NM and/or PTE2_A concurrently. */ pte2_clear_bit(pte2p, PTE2_W); pmap->pm_stats.wired_count--; } } if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock); } PMAP_UNLOCK(pmap); } /* * Clear the write and modified bits in each of the given page's mappings. */ void pmap_remove_write(vm_page_t m) { struct md_page *pvh; pv_entry_t next_pv, pv; pmap_t pmap; pt1_entry_t *pte1p; pt2_entry_t *pte2p, opte2; vm_offset_t va; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return; rw_wlock(&pvh_global_lock); sched_pin(); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, va); if (!(pte1_load(pte1p) & PTE1_RO)) (void)pmap_demote_pte1(pmap, pte1p, va); PMAP_UNLOCK(pmap); } small_mappings: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, pv->pv_va); KASSERT(!pte1_is_section(pte1_load(pte1p)), ("%s: found" " a section in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); opte2 = pte2_load(pte2p); if (!(opte2 & PTE2_RO)) { pte2_store(pte2p, opte2 | PTE2_RO | PTE2_NM); if (pte2_is_dirty(opte2)) vm_page_dirty(m); pmap_tlb_flush(pmap, pv->pv_va); } PMAP_UNLOCK(pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); sched_unpin(); rw_wunlock(&pvh_global_lock); } /* * Apply the given advice to the specified range of addresses within the * given pmap. Depending on the advice, clear the referenced and/or * modified flags in each mapping and set the mapped page's dirty field. */ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { pt1_entry_t *pte1p, opte1; pt2_entry_t *pte2p, pte2; vm_offset_t pdnxt; vm_page_t m; boolean_t pv_lists_locked; if (advice != MADV_DONTNEED && advice != MADV_FREE) return; if (pmap_is_current(pmap)) pv_lists_locked = FALSE; else { pv_lists_locked = TRUE; resume: rw_wlock(&pvh_global_lock); sched_pin(); } PMAP_LOCK(pmap); for (; sva < eva; sva = pdnxt) { pdnxt = pte1_trunc(sva + PTE1_SIZE); if (pdnxt < sva) pdnxt = eva; pte1p = pmap_pte1(pmap, sva); opte1 = pte1_load(pte1p); if (!pte1_is_valid(opte1)) /* XXX */ continue; else if (pte1_is_section(opte1)) { if (!pte1_is_managed(opte1)) continue; if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { PMAP_UNLOCK(pmap); goto resume; } sched_pin(); } if (!pmap_demote_pte1(pmap, pte1p, sva)) { /* * The large page mapping was destroyed. */ continue; } /* * Unless the page mappings are wired, remove the * mapping to a single page so that a subsequent * access may repromote. Since the underlying L2 page * table is fully populated, this removal never * frees a L2 page table page. */ if (!pte1_is_wired(opte1)) { pte2p = pmap_pte2_quick(pmap, sva); KASSERT(pte2_is_valid(pte2_load(pte2p)), ("%s: invalid PTE2", __func__)); pmap_remove_pte2(pmap, pte2p, sva, NULL); } } if (pdnxt > eva) pdnxt = eva; for (pte2p = pmap_pte2_quick(pmap, sva); sva != pdnxt; pte2p++, sva += PAGE_SIZE) { pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2) || !pte2_is_managed(pte2)) continue; else if (pte2_is_dirty(pte2)) { if (advice == MADV_DONTNEED) { /* * Future calls to pmap_is_modified() * can be avoided by making the page * dirty now. */ m = PHYS_TO_VM_PAGE(pte2_pa(pte2)); vm_page_dirty(m); } pte2_set_bit(pte2p, PTE2_NM); pte2_clear_bit(pte2p, PTE2_A); } else if ((pte2 & PTE2_A) != 0) pte2_clear_bit(pte2p, PTE2_A); else continue; pmap_tlb_flush(pmap, sva); } } if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock); } PMAP_UNLOCK(pmap); } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_page_t m) { struct md_page *pvh; pv_entry_t next_pv, pv; pmap_t pmap; pt1_entry_t *pte1p, opte1; pt2_entry_t *pte2p, opte2; vm_offset_t va; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("%s: page %p is exclusive busy", __func__, m)); /* * If the page is not PGA_WRITEABLE, then no PTE2s can have PTE2_NM * cleared. If the object containing the page is locked and the page * is not exclusive busied, then PGA_WRITEABLE cannot be concurrently * set. */ if ((m->flags & PGA_WRITEABLE) == 0) return; rw_wlock(&pvh_global_lock); sched_pin(); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, va); opte1 = pte1_load(pte1p); if (!(opte1 & PTE1_RO)) { if (pmap_demote_pte1(pmap, pte1p, va) && !pte1_is_wired(opte1)) { /* * Write protect the mapping to a * single page so that a subsequent * write access may repromote. */ va += VM_PAGE_TO_PHYS(m) - pte1_pa(opte1); pte2p = pmap_pte2_quick(pmap, va); opte2 = pte2_load(pte2p); if ((opte2 & PTE2_V)) { pte2_set_bit(pte2p, PTE2_NM | PTE2_RO); vm_page_dirty(m); pmap_tlb_flush(pmap, va); } } } PMAP_UNLOCK(pmap); } small_mappings: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, pv->pv_va); KASSERT(!pte1_is_section(pte1_load(pte1p)), ("%s: found" " a section in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); if (pte2_is_dirty(pte2_load(pte2p))) { pte2_set_bit(pte2p, PTE2_NM); pmap_tlb_flush(pmap, pv->pv_va); } PMAP_UNLOCK(pmap); } sched_unpin(); rw_wunlock(&pvh_global_lock); } /* * Sets the memory attribute for the specified page. */ void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { struct sysmaps *sysmaps; vm_memattr_t oma; vm_paddr_t pa; oma = m->md.pat_mode; m->md.pat_mode = ma; CTR5(KTR_PMAP, "%s: page %p - 0x%08X oma: %d, ma: %d", __func__, m, VM_PAGE_TO_PHYS(m), oma, ma); if ((m->flags & PG_FICTITIOUS) != 0) return; #if 0 /* * If "m" is a normal page, flush it from the cache. * * First, try to find an existing mapping of the page by sf * buffer. sf_buf_invalidate_cache() modifies mapping and * flushes the cache. */ if (sf_buf_invalidate_cache(m, oma)) return; #endif /* * If page is not mapped by sf buffer, map the page * transient and do invalidation. */ if (ma != oma) { pa = VM_PAGE_TO_PHYS(m); sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (*sysmaps->CMAP2) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(pa, PTE2_AP_KRW, vm_memattr_to_pte2(ma))); dcache_wbinv_poc((vm_offset_t)sysmaps->CADDR2, pa, PAGE_SIZE); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } } /* * Miscellaneous support routines follow */ /* * Returns TRUE if the given page is mapped individually or as part of * a 1mpage. Otherwise, returns FALSE. */ boolean_t pmap_page_is_mapped(vm_page_t m) { boolean_t rv; if ((m->oflags & VPO_UNMANAGED) != 0) return (FALSE); rw_wlock(&pvh_global_lock); rv = !TAILQ_EMPTY(&m->md.pv_list) || ((m->flags & PG_FICTITIOUS) == 0 && !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); rw_wunlock(&pvh_global_lock); return (rv); } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { struct md_page *pvh; pv_entry_t pv; int loops = 0; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); rv = FALSE; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } } rw_wunlock(&pvh_global_lock); return (rv); } /* * pmap_zero_page zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. */ void pmap_zero_page(vm_page_t m) { struct sysmaps *sysmaps; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); pagezero(sysmaps->CADDR2); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } /* * pmap_zero_page_area zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. * * off and size may not cover an area beyond a single hardware page. */ void pmap_zero_page_area(vm_page_t m, int off, int size) { struct sysmaps *sysmaps; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); if (off == 0 && size == PAGE_SIZE) pagezero(sysmaps->CADDR2); else bzero(sysmaps->CADDR2 + off, size); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } /* * pmap_zero_page_idle zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. This * is intended to be called from the vm_pagezero process only and * outside of Giant. */ void pmap_zero_page_idle(vm_page_t m) { if (pte2_load(CMAP3) != 0) panic("%s: CMAP3 busy", __func__); sched_pin(); pte2_store(CMAP3, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); pagezero(CADDR3); pte2_clear(CMAP3); tlb_flush((vm_offset_t)CADDR3); sched_unpin(); } /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ void pmap_copy_page(vm_page_t src, vm_page_t dst) { struct sysmaps *sysmaps; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP1) != 0) panic("%s: CMAP1 busy", __func__); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP1, PTE2_KERN_NG(VM_PAGE_TO_PHYS(src), PTE2_AP_KR | PTE2_NM, vm_page_pte2_attr(src))); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(dst), PTE2_AP_KRW, vm_page_pte2_attr(dst))); bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE); pte2_clear(sysmaps->CMAP1); tlb_flush((vm_offset_t)sysmaps->CADDR1); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } int unmapped_buf_allowed = 1; void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { struct sysmaps *sysmaps; vm_page_t a_pg, b_pg; char *a_cp, *b_cp; vm_offset_t a_pg_offset, b_pg_offset; int cnt; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (*sysmaps->CMAP1 != 0) panic("pmap_copy_pages: CMAP1 busy"); if (*sysmaps->CMAP2 != 0) panic("pmap_copy_pages: CMAP2 busy"); while (xfersize > 0) { a_pg = ma[a_offset >> PAGE_SHIFT]; a_pg_offset = a_offset & PAGE_MASK; cnt = min(xfersize, PAGE_SIZE - a_pg_offset); b_pg = mb[b_offset >> PAGE_SHIFT]; b_pg_offset = b_offset & PAGE_MASK; cnt = min(cnt, PAGE_SIZE - b_pg_offset); pte2_store(sysmaps->CMAP1, PTE2_KERN_NG(VM_PAGE_TO_PHYS(a_pg), PTE2_AP_KR | PTE2_NM, vm_page_pte2_attr(a_pg))); tlb_flush_local((vm_offset_t)sysmaps->CADDR1); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(b_pg), PTE2_AP_KRW, vm_page_pte2_attr(b_pg))); tlb_flush_local((vm_offset_t)sysmaps->CADDR2); a_cp = sysmaps->CADDR1 + a_pg_offset; b_cp = sysmaps->CADDR2 + b_pg_offset; bcopy(a_cp, b_cp, cnt); a_offset += cnt; b_offset += cnt; xfersize -= cnt; } pte2_clear(sysmaps->CMAP1); tlb_flush((vm_offset_t)sysmaps->CADDR1); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } vm_offset_t pmap_quick_enter_page(vm_page_t m) { pt2_entry_t *pte2p; vm_offset_t qmap_addr; critical_enter(); qmap_addr = PCPU_GET(qmap_addr); pte2p = pt2map_entry(qmap_addr); KASSERT(pte2_load(pte2p) == 0, ("%s: PTE2 busy", __func__)); pte2_store(pte2p, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); return (qmap_addr); } void pmap_quick_remove_page(vm_offset_t addr) { pt2_entry_t *pte2p; vm_offset_t qmap_addr; qmap_addr = PCPU_GET(qmap_addr); pte2p = pt2map_entry(qmap_addr); KASSERT(addr == qmap_addr, ("%s: invalid address", __func__)); KASSERT(pte2_load(pte2p) != 0, ("%s: PTE2 not in use", __func__)); pte2_clear(pte2p); tlb_flush(qmap_addr); critical_exit(); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { struct spglist free; vm_offset_t addr; vm_offset_t end_addr = src_addr + len; vm_offset_t nextva; if (dst_addr != src_addr) return; if (!pmap_is_current(src_pmap)) return; rw_wlock(&pvh_global_lock); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); } else { PMAP_LOCK(src_pmap); PMAP_LOCK(dst_pmap); } sched_pin(); for (addr = src_addr; addr < end_addr; addr = nextva) { pt2_entry_t *src_pte2p, *dst_pte2p; vm_page_t dst_mpt2pg, src_mpt2pg; pt1_entry_t src_pte1; u_int pte1_idx; KASSERT(addr < VM_MAXUSER_ADDRESS, ("%s: invalid to pmap_copy page tables", __func__)); nextva = pte1_trunc(addr + PTE1_SIZE); if (nextva < addr) nextva = end_addr; pte1_idx = pte1_index(addr); src_pte1 = src_pmap->pm_pt1[pte1_idx]; if (pte1_is_section(src_pte1)) { if ((addr & PTE1_OFFSET) != 0 || (addr + PTE1_SIZE) > end_addr) continue; if (dst_pmap->pm_pt1[pte1_idx] == 0 && (!pte1_is_managed(src_pte1) || pmap_pv_insert_pte1(dst_pmap, addr, pte1_pa(src_pte1)))) { dst_pmap->pm_pt1[pte1_idx] = src_pte1 & ~PTE1_W; dst_pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE; pmap_pte1_mappings++; } continue; } else if (!pte1_is_link(src_pte1)) continue; src_mpt2pg = PHYS_TO_VM_PAGE(pte1_link_pa(src_pte1)); /* * We leave PT2s to be linked from PT1 even if they are not * referenced until all PT2s in a page are without reference. * * QQQ: It could be changed ... */ #if 0 /* single_pt2_link_is_cleared */ KASSERT(pt2_wirecount_get(src_mpt2pg, pte1_idx) > 0, ("%s: source page table page is unused", __func__)); #else if (pt2_wirecount_get(src_mpt2pg, pte1_idx) == 0) continue; #endif if (nextva > end_addr) nextva = end_addr; src_pte2p = pt2map_entry(addr); while (addr < nextva) { pt2_entry_t temp_pte2; temp_pte2 = pte2_load(src_pte2p); /* * we only virtual copy managed pages */ if (pte2_is_managed(temp_pte2)) { dst_mpt2pg = pmap_allocpte2(dst_pmap, addr, PMAP_ENTER_NOSLEEP); if (dst_mpt2pg == NULL) goto out; dst_pte2p = pmap_pte2_quick(dst_pmap, addr); if (!pte2_is_valid(pte2_load(dst_pte2p)) && pmap_try_insert_pv_entry(dst_pmap, addr, PHYS_TO_VM_PAGE(pte2_pa(temp_pte2)))) { /* * Clear the wired, modified, and * accessed (referenced) bits * during the copy. */ temp_pte2 &= ~(PTE2_W | PTE2_A); temp_pte2 |= PTE2_NM; pte2_store(dst_pte2p, temp_pte2); dst_pmap->pm_stats.resident_count++; } else { SLIST_INIT(&free); if (pmap_unwire_pt2(dst_pmap, addr, dst_mpt2pg, &free)) { pmap_tlb_flush(dst_pmap, addr); pmap_free_zero_pages(&free); } goto out; } if (pt2_wirecount_get(dst_mpt2pg, pte1_idx) >= pt2_wirecount_get(src_mpt2pg, pte1_idx)) break; } addr += PAGE_SIZE; src_pte2p++; } } out: sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more section mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { vm_offset_t pte1_offset; if (size < PTE1_SIZE) return; if (object != NULL && (object->flags & OBJ_COLORED) != 0) offset += ptoa(object->pg_color); pte1_offset = offset & PTE1_OFFSET; if (size - ((PTE1_SIZE - pte1_offset) & PTE1_OFFSET) < PTE1_SIZE || (*addr & PTE1_OFFSET) == pte1_offset) return; if ((*addr & PTE1_OFFSET) < pte1_offset) *addr = pte1_trunc(*addr) + pte1_offset; else *addr = pte1_roundup(*addr) + pte1_offset; } void pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; u_int cpuid, ttb; PDEBUG(9, printf("%s: td = %08x\n", __func__, (uint32_t)td)); critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); cpuid = PCPU_GET(cpuid); #if defined(SMP) CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else CPU_CLR(cpuid, &oldpmap->pm_active); CPU_SET(cpuid, &pmap->pm_active); #endif ttb = pmap_ttb_get(pmap); /* * pmap_activate is for the current thread on the current cpu */ td->td_pcb->pcb_pagedir = ttb; cp15_ttbr_set(ttb); PCPU_SET(curpmap, pmap); critical_exit(); } /* * Perform the pmap work for mincore. */ int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) { pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; vm_paddr_t pa; boolean_t managed; int val; PMAP_LOCK(pmap); retry: pte1p = pmap_pte1(pmap, addr); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1)) { pa = trunc_page(pte1_pa(pte1) | (addr & PTE1_OFFSET)); managed = pte1_is_managed(pte1); val = MINCORE_SUPER | MINCORE_INCORE; if (pte1_is_dirty(pte1)) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if (pte1 & PTE1_A) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; } else if (pte1_is_link(pte1)) { pte2p = pmap_pte2(pmap, addr); pte2 = pte2_load(pte2p); pmap_pte2_release(pte2p); pa = pte2_pa(pte2); managed = pte2_is_managed(pte2); val = MINCORE_INCORE; if (pte2_is_dirty(pte2)) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if (pte2 & PTE2_A) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; } else { managed = FALSE; val = 0; } if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) goto retry; } else PA_UNLOCK_COND(*locked_pa); PMAP_UNLOCK(pmap); return (val); } void pmap_kenter_device(vm_offset_t va, vm_size_t size, vm_paddr_t pa) { vm_offset_t sva; uint32_t l2attr; KASSERT((size & PAGE_MASK) == 0, ("%s: device mapping not page-sized", __func__)); sva = va; l2attr = vm_memattr_to_pte2(VM_MEMATTR_DEVICE); while (size != 0) { pmap_kenter_prot_attr(va, pa, PTE2_AP_KRW, l2attr); va += PAGE_SIZE; pa += PAGE_SIZE; size -= PAGE_SIZE; } tlb_flush_range(sva, va - sva); } void pmap_kremove_device(vm_offset_t va, vm_size_t size) { vm_offset_t sva; KASSERT((size & PAGE_MASK) == 0, ("%s: device mapping not page-sized", __func__)); sva = va; while (size != 0) { pmap_kremove(va); va += PAGE_SIZE; size -= PAGE_SIZE; } tlb_flush_range(sva, va - sva); } void pmap_set_pcb_pagedir(pmap_t pmap, struct pcb *pcb) { pcb->pcb_pagedir = pmap_ttb_get(pmap); } /* * Clean L1 data cache range by physical address. * The range must be within a single page. */ static void pmap_dcache_wb_pou(vm_paddr_t pa, vm_size_t size, uint32_t attr) { struct sysmaps *sysmaps; KASSERT(((pa & PAGE_MASK) + size) <= PAGE_SIZE, ("%s: not on single page", __func__)); sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (*sysmaps->CMAP3) panic("%s: CMAP3 busy", __func__); pte2_store(sysmaps->CMAP3, PTE2_KERN_NG(pa, PTE2_AP_KRW, attr)); dcache_wb_pou((vm_offset_t)sysmaps->CADDR3 + (pa & PAGE_MASK), size); pte2_clear(sysmaps->CMAP3); tlb_flush((vm_offset_t)sysmaps->CADDR3); sched_unpin(); mtx_unlock(&sysmaps->lock); } /* * Sync instruction cache range which is not mapped yet. */ void cache_icache_sync_fresh(vm_offset_t va, vm_paddr_t pa, vm_size_t size) { uint32_t len, offset; vm_page_t m; /* Write back d-cache on given address range. */ offset = pa & PAGE_MASK; for ( ; size != 0; size -= len, pa += len, offset = 0) { len = min(PAGE_SIZE - offset, size); m = PHYS_TO_VM_PAGE(pa); KASSERT(m != NULL, ("%s: vm_page_t is null for %#x", __func__, pa)); pmap_dcache_wb_pou(pa, len, vm_page_pte2_attr(m)); } /* * I-cache is VIPT. Only way how to flush all virtual mappings * on given physical address is to invalidate all i-cache. */ icache_inv_all(); } void pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t size) { /* Write back d-cache on given address range. */ if (va >= VM_MIN_KERNEL_ADDRESS) { dcache_wb_pou(va, size); } else { uint32_t len, offset; vm_paddr_t pa; vm_page_t m; offset = va & PAGE_MASK; for ( ; size != 0; size -= len, va += len, offset = 0) { pa = pmap_extract(pmap, va); /* offset is preserved */ len = min(PAGE_SIZE - offset, size); m = PHYS_TO_VM_PAGE(pa); KASSERT(m != NULL, ("%s: vm_page_t is null for %#x", __func__, pa)); pmap_dcache_wb_pou(pa, len, vm_page_pte2_attr(m)); } } /* * I-cache is VIPT. Only way how to flush all virtual mappings * on given physical address is to invalidate all i-cache. */ icache_inv_all(); } /* * The implementation of pmap_fault() uses IN_RANGE2() macro which * depends on the fact that given range size is a power of 2. */ CTASSERT(powerof2(NB_IN_PT1)); CTASSERT(powerof2(PT2MAP_SIZE)); #define IN_RANGE2(addr, start, size) \ ((vm_offset_t)(start) == ((vm_offset_t)(addr) & ~((size) - 1))) /* * Handle access and R/W emulation faults. */ int pmap_fault(pmap_t pmap, vm_offset_t far, uint32_t fsr, int idx, bool usermode) { pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; if (pmap == NULL) pmap = kernel_pmap; /* * In kernel, we should never get abort with FAR which is in range of * pmap->pm_pt1 or PT2MAP address spaces. If it happens, stop here * and print out a useful abort message and even get to the debugger * otherwise it likely ends with never ending loop of aborts. */ if (__predict_false(IN_RANGE2(far, pmap->pm_pt1, NB_IN_PT1))) { /* * All L1 tables should always be mapped and present. * However, we check only current one herein. For user mode, * only permission abort from malicious user is not fatal. * And alignment abort as it may have higher priority. */ if (!usermode || (idx != FAULT_ALIGN && idx != FAULT_PERM_L2)) { CTR4(KTR_PMAP, "%s: pmap %#x pm_pt1 %#x far %#x", __func__, pmap, pmap->pm_pt1, far); panic("%s: pm_pt1 abort", __func__); } return (KERN_INVALID_ADDRESS); } if (__predict_false(IN_RANGE2(far, PT2MAP, PT2MAP_SIZE))) { /* * PT2MAP should be always mapped and present in current * L1 table. However, only existing L2 tables are mapped * in PT2MAP. For user mode, only L2 translation abort and * permission abort from malicious user is not fatal. * And alignment abort as it may have higher priority. */ if (!usermode || (idx != FAULT_ALIGN && idx != FAULT_TRAN_L2 && idx != FAULT_PERM_L2)) { CTR4(KTR_PMAP, "%s: pmap %#x PT2MAP %#x far %#x", __func__, pmap, PT2MAP, far); panic("%s: PT2MAP abort", __func__); } return (KERN_INVALID_ADDRESS); } /* * A pmap lock is used below for handling of access and R/W emulation * aborts. They were handled by atomic operations before so some * analysis of new situation is needed to answer the following question: * Is it safe to use the lock even for these aborts? * * There may happen two cases in general: * * (1) Aborts while the pmap lock is locked already - this should not * happen as pmap lock is not recursive. However, under pmap lock only * internal kernel data should be accessed and such data should be * mapped with A bit set and NM bit cleared. If double abort happens, * then a mapping of data which has caused it must be fixed. Further, * all new mappings are always made with A bit set and the bit can be * cleared only on managed mappings. * * (2) Aborts while another lock(s) is/are locked - this already can * happen. However, there is no difference here if it's either access or * R/W emulation abort, or if it's some other abort. */ PMAP_LOCK(pmap); #ifdef SMP /* * Special treatment due to break-before-make approach done when * pte1 is updated for userland mapping during section promotion or * demotion. If not catched here, pmap_enter() can find a section * mapping on faulting address. That is not allowed. */ if (idx == FAULT_TRAN_L1 && usermode && cp15_ats1cur_check(far) == 0) { PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } #endif /* * Accesss bits for page and section. Note that the entry * is not in TLB yet, so TLB flush is not necessary. * * QQQ: This is hardware emulation, we do not call userret() * for aborts from user mode. */ if (idx == FAULT_ACCESS_L2) { pte2p = pt2map_entry(far); pte2 = pte2_load(pte2p); if (pte2_is_valid(pte2)) { pte2_store(pte2p, pte2 | PTE2_A); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } } if (idx == FAULT_ACCESS_L1) { pte1p = pmap_pte1(pmap, far); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1)) { pte1_store(pte1p, pte1 | PTE1_A); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } } /* * Handle modify bits for page and section. Note that the modify * bit is emulated by software. So PTEx_RO is software read only * bit and PTEx_NM flag is real hardware read only bit. * * QQQ: This is hardware emulation, we do not call userret() * for aborts from user mode. */ if ((fsr & FSR_WNR) && (idx == FAULT_PERM_L2)) { pte2p = pt2map_entry(far); pte2 = pte2_load(pte2p); if (pte2_is_valid(pte2) && !(pte2 & PTE2_RO) && (pte2 & PTE2_NM)) { pte2_store(pte2p, pte2 & ~PTE2_NM); tlb_flush(trunc_page(far)); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } } if ((fsr & FSR_WNR) && (idx == FAULT_PERM_L1)) { pte1p = pmap_pte1(pmap, far); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1) && !(pte1 & PTE1_RO) && (pte1 & PTE1_NM)) { pte1_store(pte1p, pte1 & ~PTE1_NM); tlb_flush(pte1_trunc(far)); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } } /* * QQQ: The previous code, mainly fast handling of access and * modify bits aborts, could be moved to ASM. Now we are * starting to deal with not fast aborts. */ #ifdef INVARIANTS /* * Read an entry in PT2TAB associated with both pmap and far. * It's safe because PT2TAB is always mapped. */ pte2 = pt2tab_load(pmap_pt2tab_entry(pmap, far)); if (pte2_is_valid(pte2)) { /* * Now, when we know that L2 page table is allocated, * we can use PT2MAP to get L2 page table entry. */ pte2 = pte2_load(pt2map_entry(far)); if (pte2_is_valid(pte2)) { /* * If L2 page table entry is valid, make sure that * L1 page table entry is valid too. Note that we * leave L2 page entries untouched when promoted. */ pte1 = pte1_load(pmap_pte1(pmap, far)); if (!pte1_is_valid(pte1)) { panic("%s: missing L1 page entry (%p, %#x)", __func__, pmap, far); } } } #endif PMAP_UNLOCK(pmap); return (KERN_FAILURE); } #if defined(PMAP_DEBUG) /* * Reusing of KVA used in pmap_zero_page function !!! */ static void pmap_zero_page_check(vm_page_t m) { uint32_t *p, *end; struct sysmaps *sysmaps; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); end = (uint32_t*)(sysmaps->CADDR2 + PAGE_SIZE); for (p = (uint32_t*)sysmaps->CADDR2; p < end; p++) if (*p != 0) panic("%s: page %p not zero, va: %p", __func__, m, sysmaps->CADDR2); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } int pmap_pid_dump(int pid) { pmap_t pmap; struct proc *p; int npte2 = 0; int i, j, index; sx_slock(&allproc_lock); FOREACH_PROC_IN_SYSTEM(p) { if (p->p_pid != pid || p->p_vmspace == NULL) continue; index = 0; pmap = vmspace_pmap(p->p_vmspace); for (i = 0; i < NPTE1_IN_PT1; i++) { pt1_entry_t pte1; pt2_entry_t *pte2p, pte2; vm_offset_t base, va; vm_paddr_t pa; vm_page_t m; base = i << PTE1_SHIFT; pte1 = pte1_load(&pmap->pm_pt1[i]); if (pte1_is_section(pte1)) { /* * QQQ: Do something here! */ } else if (pte1_is_link(pte1)) { for (j = 0; j < NPTE2_IN_PT2; j++) { va = base + (j << PAGE_SHIFT); if (va >= VM_MIN_KERNEL_ADDRESS) { if (index) { index = 0; printf("\n"); } sx_sunlock(&allproc_lock); return (npte2); } pte2p = pmap_pte2(pmap, va); pte2 = pte2_load(pte2p); pmap_pte2_release(pte2p); if (!pte2_is_valid(pte2)) continue; pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pa); printf("va: 0x%x, pa: 0x%x, h: %d, w:" " %d, f: 0x%x", va, pa, m->hold_count, m->wire_count, m->flags); npte2++; index++; if (index >= 2) { index = 0; printf("\n"); } else { printf(" "); } } } } } sx_sunlock(&allproc_lock); return (npte2); } #endif #ifdef DDB static pt2_entry_t * pmap_pte2_ddb(pmap_t pmap, vm_offset_t va) { pt1_entry_t pte1; vm_paddr_t pt2pg_pa; pte1 = pte1_load(pmap_pte1(pmap, va)); if (!pte1_is_link(pte1)) return (NULL); if (pmap_is_current(pmap)) return (pt2map_entry(va)); /* Note that L2 page table size is not equal to PAGE_SIZE. */ pt2pg_pa = trunc_page(pte1_link_pa(pte1)); if (pte2_pa(pte2_load(PMAP3)) != pt2pg_pa) { pte2_store(PMAP3, PTE2_KPT(pt2pg_pa)); #ifdef SMP PMAP3cpu = PCPU_GET(cpuid); #endif tlb_flush_local((vm_offset_t)PADDR3); } #ifdef SMP else if (PMAP3cpu != PCPU_GET(cpuid)) { PMAP3cpu = PCPU_GET(cpuid); tlb_flush_local((vm_offset_t)PADDR3); } #endif return (PADDR3 + (arm32_btop(va) & (NPTE2_IN_PG - 1))); } static void dump_pmap(pmap_t pmap) { printf("pmap %p\n", pmap); printf(" pm_pt1: %p\n", pmap->pm_pt1); printf(" pm_pt2tab: %p\n", pmap->pm_pt2tab); printf(" pm_active: 0x%08lX\n", pmap->pm_active.__bits[0]); } DB_SHOW_COMMAND(pmaps, pmap_list_pmaps) { pmap_t pmap; LIST_FOREACH(pmap, &allpmaps, pm_list) { dump_pmap(pmap); } } static int pte2_class(pt2_entry_t pte2) { int cls; cls = (pte2 >> 2) & 0x03; cls |= (pte2 >> 4) & 0x04; return (cls); } static void dump_section(pmap_t pmap, uint32_t pte1_idx) { } static void dump_link(pmap_t pmap, uint32_t pte1_idx, boolean_t invalid_ok) { uint32_t i; vm_offset_t va; pt2_entry_t *pte2p, pte2; vm_page_t m; va = pte1_idx << PTE1_SHIFT; pte2p = pmap_pte2_ddb(pmap, va); for (i = 0; i < NPTE2_IN_PT2; i++, pte2p++, va += PAGE_SIZE) { pte2 = pte2_load(pte2p); if (pte2 == 0) continue; if (!pte2_is_valid(pte2)) { printf(" 0x%08X: 0x%08X", va, pte2); if (!invalid_ok) printf(" - not valid !!!"); printf("\n"); continue; } m = PHYS_TO_VM_PAGE(pte2_pa(pte2)); printf(" 0x%08X: 0x%08X, TEX%d, s:%d, g:%d, m:%p", va , pte2, pte2_class(pte2), !!(pte2 & PTE2_S), !(pte2 & PTE2_NG), m); if (m != NULL) { printf(" v:%d h:%d w:%d f:0x%04X\n", m->valid, m->hold_count, m->wire_count, m->flags); } else { printf("\n"); } } } static __inline boolean_t is_pv_chunk_space(vm_offset_t va) { if ((((vm_offset_t)pv_chunkbase) <= va) && (va < ((vm_offset_t)pv_chunkbase + PAGE_SIZE * pv_maxchunks))) return (TRUE); return (FALSE); } DB_SHOW_COMMAND(pmap, pmap_pmap_print) { /* XXX convert args. */ pmap_t pmap = (pmap_t)addr; pt1_entry_t pte1; pt2_entry_t pte2; vm_offset_t va, eva; vm_page_t m; uint32_t i; boolean_t invalid_ok, dump_link_ok, dump_pv_chunk; if (have_addr) { pmap_t pm; LIST_FOREACH(pm, &allpmaps, pm_list) if (pm == pmap) break; if (pm == NULL) { printf("given pmap %p is not in allpmaps list\n", pmap); return; } } else pmap = PCPU_GET(curpmap); eva = (modif[0] == 'u') ? VM_MAXUSER_ADDRESS : 0xFFFFFFFF; dump_pv_chunk = FALSE; /* XXX evaluate from modif[] */ printf("pmap: 0x%08X\n", (uint32_t)pmap); printf("PT2MAP: 0x%08X\n", (uint32_t)PT2MAP); printf("pt2tab: 0x%08X\n", (uint32_t)pmap->pm_pt2tab); for(i = 0; i < NPTE1_IN_PT1; i++) { pte1 = pte1_load(&pmap->pm_pt1[i]); if (pte1 == 0) continue; va = i << PTE1_SHIFT; if (va >= eva) break; if (pte1_is_section(pte1)) { printf("0x%08X: Section 0x%08X, s:%d g:%d\n", va, pte1, !!(pte1 & PTE1_S), !(pte1 & PTE1_NG)); dump_section(pmap, i); } else if (pte1_is_link(pte1)) { dump_link_ok = TRUE; invalid_ok = FALSE; pte2 = pte2_load(pmap_pt2tab_entry(pmap, va)); m = PHYS_TO_VM_PAGE(pte1_link_pa(pte1)); printf("0x%08X: Link 0x%08X, pt2tab: 0x%08X m: %p", va, pte1, pte2, m); if (is_pv_chunk_space(va)) { printf(" - pv_chunk space"); if (dump_pv_chunk) invalid_ok = TRUE; else dump_link_ok = FALSE; } else if (m != NULL) printf(" w:%d w2:%u", m->wire_count, pt2_wirecount_get(m, pte1_index(va))); if (pte2 == 0) printf(" !!! pt2tab entry is ZERO"); else if (pte2_pa(pte1) != pte2_pa(pte2)) printf(" !!! pt2tab entry is DIFFERENT - m: %p", PHYS_TO_VM_PAGE(pte2_pa(pte2))); printf("\n"); if (dump_link_ok) dump_link(pmap, i, invalid_ok); } else printf("0x%08X: Invalid entry 0x%08X\n", va, pte1); } } static void dump_pt2tab(pmap_t pmap) { uint32_t i; pt2_entry_t pte2; vm_offset_t va; vm_paddr_t pa; vm_page_t m; printf("PT2TAB:\n"); for (i = 0; i < PT2TAB_ENTRIES; i++) { pte2 = pte2_load(&pmap->pm_pt2tab[i]); if (!pte2_is_valid(pte2)) continue; va = i << PT2TAB_SHIFT; pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pa); printf(" 0x%08X: 0x%08X, TEX%d, s:%d, m:%p", va, pte2, pte2_class(pte2), !!(pte2 & PTE2_S), m); if (m != NULL) printf(" , h: %d, w: %d, f: 0x%04X pidx: %lld", m->hold_count, m->wire_count, m->flags, m->pindex); printf("\n"); } } DB_SHOW_COMMAND(pmap_pt2tab, pmap_pt2tab_print) { /* XXX convert args. */ pmap_t pmap = (pmap_t)addr; pt1_entry_t pte1; pt2_entry_t pte2; vm_offset_t va; uint32_t i, start; if (have_addr) { printf("supported only on current pmap\n"); return; } pmap = PCPU_GET(curpmap); printf("curpmap: 0x%08X\n", (uint32_t)pmap); printf("PT2MAP: 0x%08X\n", (uint32_t)PT2MAP); printf("pt2tab: 0x%08X\n", (uint32_t)pmap->pm_pt2tab); start = pte1_index((vm_offset_t)PT2MAP); for (i = start; i < (start + NPT2_IN_PT2TAB); i++) { pte1 = pte1_load(&pmap->pm_pt1[i]); if (pte1 == 0) continue; va = i << PTE1_SHIFT; if (pte1_is_section(pte1)) { printf("0x%08X: Section 0x%08X, s:%d\n", va, pte1, !!(pte1 & PTE1_S)); dump_section(pmap, i); } else if (pte1_is_link(pte1)) { pte2 = pte2_load(pmap_pt2tab_entry(pmap, va)); printf("0x%08X: Link 0x%08X, pt2tab: 0x%08X\n", va, pte1, pte2); if (pte2 == 0) printf(" !!! pt2tab entry is ZERO\n"); } else printf("0x%08X: Invalid entry 0x%08X\n", va, pte1); } dump_pt2tab(pmap); } #endif Index: head/sys/arm/arm/swtch-v4.S =================================================================== --- head/sys/arm/arm/swtch-v4.S (revision 299068) +++ head/sys/arm/arm/swtch-v4.S (revision 299069) @@ -1,377 +1,377 @@ /* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ /*- * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * cpuswitch.S * * cpu switching functions * * Created : 15/10/94 * */ #include "assym.s" #include "opt_sched.h" #include #include #include #include #include __FBSDID("$FreeBSD$"); #define GET_PCPU(tmp, tmp2) \ ldr tmp, .Lcurpcpu #ifdef VFP .fpu vfp /* allow VFP instructions */ #endif .Lcurpcpu: .word _C_LABEL(__pcpu) .Lblocked_lock: .word _C_LABEL(blocked_lock) #define DOMAIN_CLIENT 0x01 .Lcpufuncs: .word _C_LABEL(cpufuncs) /* * cpu_throw(oldtd, newtd) * * Remove current thread state, then select the next thread to run * and load its state. * r0 = oldtd * r1 = newtd */ ENTRY(cpu_throw) mov r5, r1 /* * r0 = oldtd * r5 = newtd */ #ifdef VFP /* This thread is dying, disable */ bl _C_LABEL(vfp_discard) /* VFP without preserving state. */ #endif GET_PCPU(r7, r9) ldr r7, [r5, #(TD_PCB)] /* r7 = new thread's PCB */ /* Switch to lwp0 context */ ldr r9, .Lcpufuncs mov lr, pc ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] ldr r0, [r7, #(PCB_PL1VEC)] ldr r1, [r7, #(PCB_DACR)] /* * r0 = Pointer to L1 slot for vector_page (or NULL) * r1 = lwp0's DACR * r5 = lwp0 * r7 = lwp0's PCB * r9 = cpufuncs */ /* * Ensure the vector table is accessible by fixing up lwp0's L1 */ cmp r0, #0 /* No need to fixup vector table? */ ldrne r3, [r0] /* But if yes, fetch current value */ ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ cmpne r3, r2 /* Stuffing the same value? */ strne r2, [r0] /* Store if not. */ #ifdef PMAP_INCLUDE_PTE_SYNC /* * Need to sync the cache to make sure that last store is * visible to the MMU. */ movne r1, #4 movne lr, pc ldrne pc, [r9, #CF_DCACHE_WB_RANGE] #endif /* PMAP_INCLUDE_PTE_SYNC */ /* * Note: We don't do the same optimisation as cpu_switch() with * respect to avoiding flushing the TLB if we're switching to * the same L1 since this process' VM space may be about to go * away, so we don't want *any* turds left in the TLB. */ /* Switch the memory to the new process */ ldr r0, [r7, #(PCB_PAGEDIR)] mov lr, pc ldr pc, [r9, #CF_CONTEXT_SWITCH] GET_PCPU(r6, r4) /* Hook in a new pcb */ str r7, [r6, #PC_CURPCB] /* We have a new curthread now so make a note it */ str r5, [r6, #PC_CURTHREAD] /* Set the new tp */ ldr r6, [r5, #(TD_MD + MD_TP)] ldr r4, =ARM_TP_ADDRESS str r6, [r4] ldr r6, [r5, #(TD_MD + MD_RAS_START)] str r6, [r4, #4] /* ARM_RAS_START */ ldr r6, [r5, #(TD_MD + MD_RAS_END)] str r6, [r4, #8] /* ARM_RAS_END */ /* Restore all the saved registers and exit */ add r3, r7, #PCB_R4 ldmia r3, {r4-r12, sp, pc} END(cpu_throw) /* * cpu_switch(oldtd, newtd, lock) * * Save the current thread state, then select the next thread to run * and load its state. * r0 = oldtd * r1 = newtd * r2 = lock (new lock for old thread) */ ENTRY(cpu_switch) /* Interrupts are disabled. */ /* Save all the registers in the old thread's pcb. */ ldr r3, [r0, #(TD_PCB)] /* Restore all the saved registers and exit */ add r3, #(PCB_R4) stmia r3, {r4-r12, sp, lr, pc} mov r6, r2 /* Save the mutex */ /* rem: r0 = old lwp */ /* rem: interrupts are disabled */ /* Process is now on a processor. */ /* We have a new curthread now so make a note it */ GET_PCPU(r7, r2) str r1, [r7, #PC_CURTHREAD] /* Hook in a new pcb */ ldr r2, [r1, #TD_PCB] str r2, [r7, #PC_CURPCB] /* Stage two : Save old context */ /* Get the user structure for the old thread. */ ldr r2, [r0, #(TD_PCB)] mov r4, r0 /* Save the old thread. */ /* Store the old tp; userland can change it on armv4. */ ldr r3, =ARM_TP_ADDRESS ldr r9, [r3] str r9, [r0, #(TD_MD + MD_TP)] ldr r9, [r3, #4] str r9, [r0, #(TD_MD + MD_RAS_START)] ldr r9, [r3, #8] str r9, [r0, #(TD_MD + MD_RAS_END)] /* Set the new tp */ ldr r9, [r1, #(TD_MD + MD_TP)] str r9, [r3] ldr r9, [r1, #(TD_MD + MD_RAS_START)] str r9, [r3, #4] ldr r9, [r1, #(TD_MD + MD_RAS_END)] str r9, [r3, #8] /* Get the user structure for the new process in r9 */ ldr r9, [r1, #(TD_PCB)] /* rem: r2 = old PCB */ /* rem: r9 = new PCB */ /* rem: interrupts are enabled */ #ifdef VFP fmrx r0, fpexc /* If the VFP is enabled */ tst r0, #(VFPEXC_EN) /* the current thread has */ movne r1, #1 /* used it, so go save */ addne r0, r2, #(PCB_VFPSTATE) /* the state into the PCB */ blne _C_LABEL(vfp_store) /* and disable the VFP. */ #endif /* r0-r3 now free! */ /* Third phase : restore saved context */ /* rem: r2 = old PCB */ /* rem: r9 = new PCB */ ldr r5, [r9, #(PCB_DACR)] /* r5 = new DACR */ mov r2, #DOMAIN_CLIENT cmp r5, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ beq .Lcs_context_switched /* Yup. Don't flush cache */ mrc p15, 0, r0, c3, c0, 0 /* r0 = old DACR */ /* * Get the new L1 table pointer into r11. If we're switching to * an LWP with the same address space as the outgoing one, we can * skip the cache purge and the TTB load. * * To avoid data dep stalls that would happen anyway, we try * and get some useful work done in the mean time. */ mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ teq r10, r11 /* Same L1? */ cmpeq r0, r5 /* Same DACR? */ beq .Lcs_context_switched /* yes! */ /* - * Definately need to flush the cache. + * Definitely need to flush the cache. */ ldr r1, .Lcpufuncs mov lr, pc ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] .Lcs_cache_purge_skipped: /* rem: r6 = lock */ /* rem: r9 = new PCB */ /* rem: r10 = old L1 */ /* rem: r11 = new L1 */ mov r2, #0x00000000 ldr r7, [r9, #(PCB_PL1VEC)] /* * Ensure the vector table is accessible by fixing up the L1 */ cmp r7, #0 /* No need to fixup vector table? */ ldrne r2, [r7] /* But if yes, fetch current value */ ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ mcr p15, 0, r5, c3, c0, 0 /* Update DACR for new context */ cmpne r2, r0 /* Stuffing the same value? */ #ifndef PMAP_INCLUDE_PTE_SYNC strne r0, [r7] /* Nope, update it */ #else beq .Lcs_same_vector str r0, [r7] /* Otherwise, update it */ /* * Need to sync the cache to make sure that last store is * visible to the MMU. */ ldr r2, .Lcpufuncs mov r0, r7 mov r1, #4 mov lr, pc ldr pc, [r2, #CF_DCACHE_WB_RANGE] .Lcs_same_vector: #endif /* PMAP_INCLUDE_PTE_SYNC */ cmp r10, r11 /* Switching to the same L1? */ ldr r10, .Lcpufuncs beq .Lcs_same_l1 /* Yup. */ /* * Do a full context switch, including full TLB flush. */ mov r0, r11 mov lr, pc ldr pc, [r10, #CF_CONTEXT_SWITCH] b .Lcs_context_switched /* * We're switching to a different process in the same L1. * In this situation, we only need to flush the TLB for the * vector_page mapping, and even then only if r7 is non-NULL. */ .Lcs_same_l1: cmp r7, #0 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ movne lr, pc ldrne pc, [r10, #CF_TLB_FLUSHID_SE] .Lcs_context_switched: /* Release the old thread */ str r6, [r4, #TD_LOCK] /* XXXSCW: Safe to re-enable FIQs here */ /* rem: r9 = new PCB */ /* Restore all the saved registers and exit */ add r3, r9, #PCB_R4 ldmia r3, {r4-r12, sp, pc} END(cpu_switch) Index: head/sys/arm/at91/at91_cfata.c =================================================================== --- head/sys/arm/at91/at91_cfata.c (revision 299068) +++ head/sys/arm/at91/at91_cfata.c (revision 299069) @@ -1,281 +1,281 @@ /*- * Copyright (c) 2008 Deglitch Networks, Stanislav Sedov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Driver for the AT91RM9200 CompactFlash controller operating in a * common memory mode. Interrupts are driven by polling. The driver * implements an ATA bridge and attached ATA channel driver on top * of it. - * NOTE WELL: this driver uses polling mode. To achive an acceptable + * NOTE WELL: this driver uses polling mode. To achieve an acceptable * operating speed you will probably want to use HZ=2000 in kernel * config. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct at91_cfata_softc { device_t dev; struct resource *mem_res; struct resource irq; void (*isr_cb)(void *); void *isr_arg; struct callout tick; }; static int at91_cfata_detach(device_t dev); static void at91_cfata_callout(void *arg); static int at91_cfata_probe(device_t dev) { device_set_desc_copy(dev, "AT91RM9200 CompactFlash controller"); return (0); } static int at91_cfata_attach(device_t dev) { struct at91_cfata_softc *sc; int rid, error; sc = device_get_softc(dev); sc->dev = dev; rid = 0; error = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) return (ENOMEM); /* XXX: init CF controller? */ callout_init(&sc->tick, 1); /* Callout to poll the device. */ device_add_child(dev, "ata", -1); bus_generic_attach(dev); return (0); } static int at91_cfata_detach(device_t dev) { struct at91_cfata_softc *sc; sc = device_get_softc(dev); bus_generic_detach(sc->dev); if (sc->mem_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->mem_res), sc->mem_res); sc->mem_res = NULL; } return (0); } static struct resource * ata_at91_alloc_resource(device_t dev, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) { struct at91_cfata_softc *sc = device_get_softc(dev); KASSERT(type == SYS_RES_IRQ && *rid == ATA_IRQ_RID, ("[at91_cfata, %d]: illegal resource request (type %u rid %u)", __LINE__, type, *rid)); return (&sc->irq); } static int ata_at91_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { KASSERT(type == SYS_RES_IRQ && rid == ATA_IRQ_RID, ("[at91_cfata, %d]: illegal resource request (type %u rid %u)", __LINE__, type, rid)); return (0); } static int ata_at91_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filt, driver_intr_t *function, void *argument, void **cookiep) { struct at91_cfata_softc *sc = device_get_softc(dev); KASSERT(sc->isr_cb == NULL, ("[at91_cfata, %d]: overwriting the old handler", __LINE__)); sc->isr_cb = function; sc->isr_arg = argument; *cookiep = sc; callout_reset(&sc->tick, 1, at91_cfata_callout, sc); return (0); } static int ata_at91_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { struct at91_cfata_softc *sc = device_get_softc(dev); sc->isr_cb = NULL; sc->isr_arg = NULL; return (0); } static void at91_cfata_callout(void *arg) { struct at91_cfata_softc *sc; sc = (struct at91_cfata_softc *)arg; if (sc->isr_cb != NULL) sc->isr_cb(sc->isr_arg); callout_reset(&sc->tick, 1, at91_cfata_callout, sc); } static int at91_channel_probe(device_t dev) { struct ata_channel *ch = device_get_softc(dev); ch->unit = 0; ch->flags |= ATA_USE_16BIT | ATA_NO_SLAVE; device_set_desc_copy(dev, "ATA channel 0"); return (ata_probe(dev)); } static int at91_channel_attach(device_t dev) { struct at91_cfata_softc *sc = device_get_softc(device_get_parent(dev)); struct ata_channel *ch = device_get_softc(dev); int i; for (i = 0; i < ATA_MAX_RES; i++) ch->r_io[i].res = sc->mem_res; /* * CF+ Specification. * 6.1.3 Memory Mapped Addressing. */ ch->r_io[ATA_DATA].offset = 0x00; ch->r_io[ATA_FEATURE].offset = 0x01; ch->r_io[ATA_COUNT].offset = 0x02; ch->r_io[ATA_SECTOR].offset = 0x03; ch->r_io[ATA_CYL_LSB].offset = 0x04; ch->r_io[ATA_CYL_MSB].offset = 0x05; ch->r_io[ATA_DRIVE].offset = 0x06; ch->r_io[ATA_COMMAND].offset = 0x07; ch->r_io[ATA_ERROR].offset = 0x01; ch->r_io[ATA_IREASON].offset = 0x02; ch->r_io[ATA_STATUS].offset = 0x07; ch->r_io[ATA_ALTSTAT].offset = 0x0e; ch->r_io[ATA_CONTROL].offset = 0x0e; /* Should point at the base of registers. */ ch->r_io[ATA_IDX_ADDR].offset = 0x0; ata_generic_hw(dev); return (ata_attach(dev)); } static device_method_t at91_cfata_methods[] = { /* Device interface. */ DEVMETHOD(device_probe, at91_cfata_probe), DEVMETHOD(device_attach, at91_cfata_attach), DEVMETHOD(device_detach, at91_cfata_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* ATA bus methods. */ DEVMETHOD(bus_alloc_resource, ata_at91_alloc_resource), DEVMETHOD(bus_release_resource, ata_at91_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_setup_intr, ata_at91_setup_intr), DEVMETHOD(bus_teardown_intr, ata_at91_teardown_intr), { 0, 0 } }; devclass_t at91_cfata_devclass; static driver_t at91_cfata_driver = { "at91_cfata", at91_cfata_methods, sizeof(struct at91_cfata_softc), }; DRIVER_MODULE(at91_cfata, atmelarm, at91_cfata_driver, at91_cfata_devclass, 0, 0); MODULE_VERSION(at91_cfata, 1); MODULE_DEPEND(at91_cfata, ata, 1, 1, 1); /* * ATA channel driver. */ static device_method_t at91_channel_methods[] = { /* device interface */ DEVMETHOD(device_probe, at91_channel_probe), DEVMETHOD(device_attach, at91_channel_attach), DEVMETHOD(device_detach, ata_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, ata_suspend), DEVMETHOD(device_resume, ata_resume), { 0, 0 } }; driver_t at91_channel_driver = { "ata", at91_channel_methods, sizeof(struct ata_channel), }; DRIVER_MODULE(ata, at91_cfata, at91_channel_driver, ata_devclass, 0, 0); Index: head/sys/arm/at91/at91_machdep.c =================================================================== --- head/sys/arm/at91/at91_machdep.c (revision 299068) +++ head/sys/arm/at91/at91_machdep.c (revision 299069) @@ -1,685 +1,685 @@ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * machdep.c * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * This file needs a lot of work. * * Created : 17/09/94 */ #include "opt_kstack_pages.h" #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef MAXCPU #define MAXCPU 1 #endif /* Page table for mapping proc0 zero page */ #define KERNEL_PT_SYS 0 #define KERNEL_PT_KERN 1 #define KERNEL_PT_KERN_NUM 22 /* L2 table for mapping after kernel */ #define KERNEL_PT_AFKERNEL KERNEL_PT_KERN + KERNEL_PT_KERN_NUM #define KERNEL_PT_AFKERNEL_NUM 5 /* this should be evenly divisable by PAGE_SIZE / L2_TABLE_SIZE_REAL (or 4) */ #define NUM_KERNEL_PTS (KERNEL_PT_AFKERNEL + KERNEL_PT_AFKERNEL_NUM) struct pv_addr kernel_pt_table[NUM_KERNEL_PTS]; /* Static device mappings. */ const struct devmap_entry at91_devmap[] = { /* * Map the critical on-board devices. The interrupt vector at * 0xffff0000 makes it impossible to map them PA == VA, so we map all * 0xfffxxxxx addresses to 0xdffxxxxx. This covers all critical devices * on all members of the AT91SAM9 and AT91RM9200 families. */ { 0xdff00000, 0xfff00000, 0x00100000, }, /* There's a notion that we should do the rest of these lazily. */ /* * We can't just map the OHCI registers VA == PA, because * AT91xx_xxx_BASE belongs to the userland address space. * We could just choose a different virtual address, but a better * solution would probably be to just use pmap_mapdev() to allocate * KVA, as we don't need the OHCI controller before the vm * initialization is done. However, the AT91 resource allocation * system doesn't know how to use pmap_mapdev() yet. * Care must be taken to ensure PA and VM address do not overlap * between entries. */ { /* * Add the ohci controller, and anything else that might be * on this chip select for a VA/PA mapping. */ /* Internal Memory 1MB */ AT91RM92_OHCI_VA_BASE, AT91RM92_OHCI_BASE, 0x00100000, }, { /* CompactFlash controller. Portion of EBI CS4 1MB */ AT91RM92_CF_VA_BASE, AT91RM92_CF_BASE, 0x00100000, }, /* * The next two should be good for the 9260, 9261 and 9G20 since * addresses mapping is the same. */ { /* Internal Memory 1MB */ AT91SAM9G20_OHCI_VA_BASE, AT91SAM9G20_OHCI_BASE, 0x00100000, }, { /* EBI CS3 256MB */ AT91SAM9G20_NAND_VA_BASE, AT91SAM9G20_NAND_BASE, AT91SAM9G20_NAND_SIZE, }, /* * The next should be good for the 9G45. */ { /* Internal Memory 1MB */ AT91SAM9G45_OHCI_VA_BASE, AT91SAM9G45_OHCI_BASE, 0x00100000, }, { 0, 0, 0, } }; #ifdef LINUX_BOOT_ABI extern int membanks; extern int memstart[]; extern int memsize[]; #endif long at91_ramsize(void) { uint32_t cr, mdr, mr, *SDRAMC; int banks, rows, cols, bw; #ifdef LINUX_BOOT_ABI /* * If we found any ATAGs that were for memory, return the first bank. */ if (membanks > 0) return (memsize[0]); #endif if (at91_is_rm92()) { SDRAMC = (uint32_t *)(AT91_BASE + AT91RM92_SDRAMC_BASE); cr = SDRAMC[AT91RM92_SDRAMC_CR / 4]; mr = SDRAMC[AT91RM92_SDRAMC_MR / 4]; banks = (cr & AT91RM92_SDRAMC_CR_NB_4) ? 2 : 1; rows = ((cr & AT91RM92_SDRAMC_CR_NR_MASK) >> 2) + 11; cols = (cr & AT91RM92_SDRAMC_CR_NC_MASK) + 8; bw = (mr & AT91RM92_SDRAMC_MR_DBW_16) ? 1 : 2; } else if (at91_cpu_is(AT91_T_SAM9G45)) { SDRAMC = (uint32_t *)(AT91_BASE + AT91SAM9G45_DDRSDRC0_BASE); cr = SDRAMC[AT91SAM9G45_DDRSDRC_CR / 4]; mdr = SDRAMC[AT91SAM9G45_DDRSDRC_MDR / 4]; banks = 0; rows = ((cr & AT91SAM9G45_DDRSDRC_CR_NR_MASK) >> 2) + 11; cols = (cr & AT91SAM9G45_DDRSDRC_CR_NC_MASK) + 8; bw = (mdr & AT91SAM9G45_DDRSDRC_MDR_DBW_16) ? 1 : 2; /* Fix the calculation for DDR memory */ mdr &= AT91SAM9G45_DDRSDRC_MDR_MASK; if (mdr & AT91SAM9G45_DDRSDRC_MDR_LPDDR1 || mdr & AT91SAM9G45_DDRSDRC_MDR_DDR2) { /* The cols value is 1 higher for DDR */ cols += 1; /* DDR has 4 internal banks. */ banks = 2; } } else { /* * This should be good for the 9260, 9261, 9G20, 9G35 and 9X25 * as addresses and registers are the same. */ SDRAMC = (uint32_t *)(AT91_BASE + AT91SAM9G20_SDRAMC_BASE); cr = SDRAMC[AT91SAM9G20_SDRAMC_CR / 4]; mr = SDRAMC[AT91SAM9G20_SDRAMC_MR / 4]; banks = (cr & AT91SAM9G20_SDRAMC_CR_NB_4) ? 2 : 1; rows = ((cr & AT91SAM9G20_SDRAMC_CR_NR_MASK) >> 2) + 11; cols = (cr & AT91SAM9G20_SDRAMC_CR_NC_MASK) + 8; bw = (cr & AT91SAM9G20_SDRAMC_CR_DBW_16) ? 1 : 2; } return (1 << (cols + rows + banks + bw)); } static const char *soc_type_name[] = { [AT91_T_CAP9] = "at91cap9", [AT91_T_RM9200] = "at91rm9200", [AT91_T_SAM9260] = "at91sam9260", [AT91_T_SAM9261] = "at91sam9261", [AT91_T_SAM9263] = "at91sam9263", [AT91_T_SAM9G10] = "at91sam9g10", [AT91_T_SAM9G20] = "at91sam9g20", [AT91_T_SAM9G45] = "at91sam9g45", [AT91_T_SAM9N12] = "at91sam9n12", [AT91_T_SAM9RL] = "at91sam9rl", [AT91_T_SAM9X5] = "at91sam9x5", [AT91_T_NONE] = "UNKNOWN" }; static const char *soc_subtype_name[] = { [AT91_ST_NONE] = "UNKNOWN", [AT91_ST_RM9200_BGA] = "at91rm9200_bga", [AT91_ST_RM9200_PQFP] = "at91rm9200_pqfp", [AT91_ST_SAM9XE] = "at91sam9xe", [AT91_ST_SAM9G45] = "at91sam9g45", [AT91_ST_SAM9M10] = "at91sam9m10", [AT91_ST_SAM9G46] = "at91sam9g46", [AT91_ST_SAM9M11] = "at91sam9m11", [AT91_ST_SAM9G15] = "at91sam9g15", [AT91_ST_SAM9G25] = "at91sam9g25", [AT91_ST_SAM9G35] = "at91sam9g35", [AT91_ST_SAM9X25] = "at91sam9x25", [AT91_ST_SAM9X35] = "at91sam9x35", }; struct at91_soc_info soc_info; /* * Read the SoC ID from the CIDR register and try to match it against the * values we know. If we find a good one, we return true. If not, we * return false. When we find a good one, we also find the subtype * and CPU family. */ static int at91_try_id(uint32_t dbgu_base) { uint32_t socid; soc_info.cidr = *(volatile uint32_t *)(AT91_BASE + dbgu_base + DBGU_C1R); socid = soc_info.cidr & ~AT91_CPU_VERSION_MASK; soc_info.type = AT91_T_NONE; soc_info.subtype = AT91_ST_NONE; soc_info.family = (soc_info.cidr & AT91_CPU_FAMILY_MASK) >> 20; soc_info.exid = *(volatile uint32_t *)(AT91_BASE + dbgu_base + DBGU_C2R); switch (socid) { case AT91_CPU_CAP9: soc_info.type = AT91_T_CAP9; break; case AT91_CPU_RM9200: soc_info.type = AT91_T_RM9200; break; case AT91_CPU_SAM9XE128: case AT91_CPU_SAM9XE256: case AT91_CPU_SAM9XE512: case AT91_CPU_SAM9260: soc_info.type = AT91_T_SAM9260; if (soc_info.family == AT91_FAMILY_SAM9XE) soc_info.subtype = AT91_ST_SAM9XE; break; case AT91_CPU_SAM9261: soc_info.type = AT91_T_SAM9261; break; case AT91_CPU_SAM9263: soc_info.type = AT91_T_SAM9263; break; case AT91_CPU_SAM9G10: soc_info.type = AT91_T_SAM9G10; break; case AT91_CPU_SAM9G20: soc_info.type = AT91_T_SAM9G20; break; case AT91_CPU_SAM9G45: soc_info.type = AT91_T_SAM9G45; break; case AT91_CPU_SAM9N12: soc_info.type = AT91_T_SAM9N12; break; case AT91_CPU_SAM9RL64: soc_info.type = AT91_T_SAM9RL; break; case AT91_CPU_SAM9X5: soc_info.type = AT91_T_SAM9X5; break; default: return (0); } switch (soc_info.type) { case AT91_T_SAM9G45: switch (soc_info.exid) { case AT91_EXID_SAM9G45: soc_info.subtype = AT91_ST_SAM9G45; break; case AT91_EXID_SAM9G46: soc_info.subtype = AT91_ST_SAM9G46; break; case AT91_EXID_SAM9M10: soc_info.subtype = AT91_ST_SAM9M10; break; case AT91_EXID_SAM9M11: soc_info.subtype = AT91_ST_SAM9M11; break; } break; case AT91_T_SAM9X5: switch (soc_info.exid) { case AT91_EXID_SAM9G15: soc_info.subtype = AT91_ST_SAM9G15; break; case AT91_EXID_SAM9G25: soc_info.subtype = AT91_ST_SAM9G25; break; case AT91_EXID_SAM9G35: soc_info.subtype = AT91_ST_SAM9G35; break; case AT91_EXID_SAM9X25: soc_info.subtype = AT91_ST_SAM9X25; break; case AT91_EXID_SAM9X35: soc_info.subtype = AT91_ST_SAM9X35; break; } break; default: break; } /* * Disable interrupts in the DBGU unit... */ *(volatile uint32_t *)(AT91_BASE + dbgu_base + USART_IDR) = 0xffffffff; /* * Save the name for later... */ snprintf(soc_info.name, sizeof(soc_info.name), "%s%s%s", soc_type_name[soc_info.type], soc_info.subtype == AT91_ST_NONE ? "" : " subtype ", soc_info.subtype == AT91_ST_NONE ? "" : soc_subtype_name[soc_info.subtype]); /* * try to get the matching CPU support. */ soc_info.soc_data = at91_match_soc(soc_info.type, soc_info.subtype); soc_info.dbgu_base = AT91_BASE + dbgu_base; return (1); } void at91_soc_id(void) { if (!at91_try_id(AT91_DBGU0)) at91_try_id(AT91_DBGU1); } #ifdef ARM_MANY_BOARD /* likely belongs in arm/arm/machdep.c, but since board_init is still at91 only... */ SET_DECLARE(arm_board_set, const struct arm_board); /* Not yet fully functional, but enough to build ATMEL config */ static long board_init(void) { return -1; } #endif #ifndef FDT /* Physical and virtual addresses for some global pages */ struct pv_addr msgbufpv; struct pv_addr kernelstack; struct pv_addr systempage; struct pv_addr irqstack; struct pv_addr abtstack; struct pv_addr undstack; void * initarm(struct arm_boot_params *abp) { struct pv_addr kernel_l1pt; struct pv_addr dpcpu; int i; u_int l1pagetable; vm_offset_t freemempos; vm_offset_t afterkern; uint32_t memsize; vm_offset_t lastaddr; lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; set_cpufuncs(); pcpu0_init(); /* Do basic tuning, hz etc */ init_param1(); freemempos = (lastaddr + PAGE_MASK) & ~PAGE_MASK; /* Define a macro to simplify memory allocation */ #define valloc_pages(var, np) \ alloc_pages((var).pv_va, (np)); \ (var).pv_pa = (var).pv_va + (abp->abp_physaddr - KERNVIRTADDR); #define alloc_pages(var, np) \ (var) = freemempos; \ freemempos += (np * PAGE_SIZE); \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos += PAGE_SIZE; valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); for (i = 0; i < NUM_KERNEL_PTS; ++i) { if (!(i % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[i], L2_TABLE_SIZE / PAGE_SIZE); } else { kernel_pt_table[i].pv_va = freemempos - (i % (PAGE_SIZE / L2_TABLE_SIZE_REAL)) * L2_TABLE_SIZE_REAL; kernel_pt_table[i].pv_pa = kernel_pt_table[i].pv_va - KERNVIRTADDR + abp->abp_physaddr; } } /* * Allocate a page for the system page mapped to 0x00000000 * or 0xffff0000. This page will just contain the system vectors * and can be shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE * MAXCPU); valloc_pages(abtstack, ABT_STACK_SIZE * MAXCPU); valloc_pages(undstack, UND_STACK_SIZE * MAXCPU); valloc_pages(kernelstack, kstack_pages * MAXCPU); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Now we start construction of the L1 page table * We start by mapping the L2 page tables into the L1. * This means that we can replace L1 mappings later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* Map the L2 pages tables in the L1 page table */ pmap_link_l2pt(l1pagetable, ARM_VECTORS_HIGH, &kernel_pt_table[KERNEL_PT_SYS]); for (i = 0; i < KERNEL_PT_KERN_NUM; i++) pmap_link_l2pt(l1pagetable, KERNBASE + i * L1_S_SIZE, &kernel_pt_table[KERNEL_PT_KERN + i]); pmap_map_chunk(l1pagetable, KERNBASE, PHYSADDR, rounddown2(((uint32_t)lastaddr - KERNBASE) + PAGE_SIZE, PAGE_SIZE), VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); afterkern = round_page(rounddown2(lastaddr + L1_S_SIZE, L1_S_SIZE)); for (i = 0; i < KERNEL_PT_AFKERNEL_NUM; i++) { pmap_link_l2pt(l1pagetable, afterkern + i * L1_S_SIZE, &kernel_pt_table[KERNEL_PT_AFKERNEL + i]); } /* Map the vector page. */ pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map the DPCPU pages */ pmap_map_chunk(l1pagetable, dpcpu.pv_va, dpcpu.pv_pa, DPCPU_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map the stack pages */ pmap_map_chunk(l1pagetable, irqstack.pv_va, irqstack.pv_pa, IRQ_STACK_SIZE * PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, abtstack.pv_va, abtstack.pv_pa, ABT_STACK_SIZE * PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, undstack.pv_va, undstack.pv_pa, UND_STACK_SIZE * PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, kernelstack.pv_va, kernelstack.pv_pa, kstack_pages * PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, kernel_l1pt.pv_va, kernel_l1pt.pv_pa, L1_TABLE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, msgbufpv.pv_va, msgbufpv.pv_pa, msgbufsize, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); for (i = 0; i < NUM_KERNEL_PTS; ++i) { pmap_map_chunk(l1pagetable, kernel_pt_table[i].pv_va, kernel_pt_table[i].pv_pa, L2_TABLE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); } devmap_bootstrap(l1pagetable, at91_devmap); cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | DOMAIN_CLIENT); cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)); at91_soc_id(); /* * Initialize all the clocks, so that the console can work. We can only * do this if at91_soc_id() was able to fill in the support data. Even * if we can't init the clocks, still try to do a console init so we can * try to print the error message about missing soc support. There's a * chance the printf will work if the bootloader set up the DBGU. */ if (soc_info.soc_data != NULL) { soc_info.soc_data->soc_clock_init(); at91_pmc_init_clock(); } cninit(); if (soc_info.soc_data == NULL) printf("Warning: No soc support for %s found.\n", soc_info.name); memsize = board_init(); if (memsize == -1) { printf("board_init() failed, cannot determine ram size; " "assuming 16MB\n"); memsize = 16 * 1024 * 1024; } /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ cpu_control(CPU_CONTROL_MMU_ENABLE, CPU_CONTROL_MMU_ENABLE); cpu_setup(); set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); undefined_init(); init_proc0(kernelstack.pv_va); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_curmaxkvaddr = afterkern + L1_S_SIZE * (KERNEL_PT_KERN_NUM - 1); /* Always use the 256MB of KVA we have available between the kernel and devices */ vm_max_kernel_address = KERNVIRTADDR + (256 << 20); pmap_bootstrap(freemempos, &kernel_l1pt); msgbufp = (void*)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Add the physical ram we have available. * * Exclude the kernel, and all the things we allocated which immediately * follow the kernel, from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_hardware_region(PHYSADDR, memsize); arm_physmem_exclude_region(abp->abp_physaddr, virtual_avail - KERNVIRTADDR, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); } #endif /* * These functions are handled elsewhere, so make them nops here. */ void cpu_startprofclock(void) { } void cpu_stopprofclock(void) { } void cpu_initclocks(void) { } void DELAY(int n) { if (soc_info.soc_data) soc_info.soc_data->soc_delay(n); } void cpu_reset(void) { if (soc_info.soc_data) soc_info.soc_data->soc_reset(); while (1) continue; } Index: head/sys/arm/at91/at91_mci.c =================================================================== --- head/sys/arm/at91/at91_mci.c (revision 299068) +++ head/sys/arm/at91/at91_mci.c (revision 299069) @@ -1,1415 +1,1415 @@ /*- * Copyright (c) 2006 Bernd Walter. All rights reserved. * Copyright (c) 2006 M. Warner Losh. All rights reserved. * Copyright (c) 2010 Greg Ansley. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #include "mmcbr_if.h" #include "opt_at91.h" /* * About running the MCI bus above 25MHz * * Historically, the MCI bus has been run at 30MHz on systems with a 60MHz * master clock, in part due to a bug in dev/mmc.c making always request * 30MHz, and in part over clocking the bus because 15MHz was too slow. * Fixing that bug causes the mmc driver to request a 25MHz clock (as it * should) and the logic in at91_mci_update_ios() picks the highest speed that * doesn't exceed that limit. With a 60MHz MCK that would be 15MHz, and * that's a real performance buzzkill when you've been getting away with 30MHz * all along. * * By defining AT91_MCI_ALLOW_OVERCLOCK (or setting the allow_overclock=1 * device hint or sysctl) you can enable logic in at91_mci_update_ios() to * overlcock the SD bus a little by running it at MCK / 2 when the requested * speed is 25MHz and the next highest speed is 15MHz or less. This appears * to work on virtually all SD cards, since it is what this driver has been * doing prior to the introduction of this option, where the overclocking vs - * underclocking decision was automaticly "overclock". Modern SD cards can + * underclocking decision was automatically "overclock". Modern SD cards can * run at 45mhz/1-bit in standard mode (high speed mode enable commands not * sent) without problems. * * Speaking of high-speed mode, the rm9200 manual says the MCI device supports * the SD v1.0 specification and can run up to 50MHz. This is interesting in * that the SD v1.0 spec caps the speed at 25MHz; high speed mode was added in * the v1.10 spec. Furthermore, high speed mode doesn't just crank up the * clock, it alters the signal timing. The rm9200 MCI device doesn't support * these altered timings. So while speeds over 25MHz may work, they only work * in what the SD spec calls "default" speed mode, and it amounts to violating * the spec by overclocking the bus. * * If you also enable 4-wire mode it's possible transfers faster than 25MHz * will fail. On the AT91RM9200, due to bugs in the bus contention logic, if * you have the USB host device and OHCI driver enabled will fail. Even * underclocking to 15MHz, intermittant overrun and underrun errors occur. * Note that you don't even need to have usb devices attached to the system, * the errors begin to occur as soon as the OHCI driver sets the register bit * to enable periodic transfers. It appears (based on brief investigation) * that the usb host controller uses so much ASB bandwidth that sometimes the * DMA for MCI transfers doesn't get a bus grant in time and data gets * dropped. Adding even a modicum of network activity changes the symptom * from intermittant to very frequent. Members of the AT91SAM9 family have * corrected this problem, or are at least better about their use of the bus. */ #ifndef AT91_MCI_ALLOW_OVERCLOCK #define AT91_MCI_ALLOW_OVERCLOCK 1 #endif /* * Allocate 2 bounce buffers we'll use to endian-swap the data due to the rm9200 * erratum. We use a pair of buffers because when reading that lets us begin * endian-swapping the data in the first buffer while the DMA is reading into * the second buffer. (We can't use the same trick for writing because we might * not get all the data in the 2nd buffer swapped before the hardware needs it; * dealing with that would add complexity to the driver.) * * The buffers are sized at 16K each due to the way the busdma cache sync * operations work on arm. A dcache_inv_range() operation on a range larger * than 16K gets turned into a dcache_wbinv_all(). That needlessly flushes the * entire data cache, impacting overall system performance. */ #define BBCOUNT 2 #define BBSIZE (16*1024) #define MAX_BLOCKS ((BBSIZE*BBCOUNT)/512) static int mci_debug; struct at91_mci_softc { void *intrhand; /* Interrupt handle */ device_t dev; int sc_cap; #define CAP_HAS_4WIRE 1 /* Has 4 wire bus */ #define CAP_NEEDS_BYTESWAP 2 /* broken hardware needing bounce */ #define CAP_MCI1_REV2XX 4 /* MCI 1 rev 2.x */ int flags; #define PENDING_CMD 0x01 #define PENDING_STOP 0x02 #define CMD_MULTIREAD 0x10 #define CMD_MULTIWRITE 0x20 int has_4wire; int allow_overclock; struct resource *irq_res; /* IRQ resource */ struct resource *mem_res; /* Memory resource */ struct mtx sc_mtx; bus_dma_tag_t dmatag; struct mmc_host host; int bus_busy; struct mmc_request *req; struct mmc_command *curcmd; bus_dmamap_t bbuf_map[BBCOUNT]; char * bbuf_vaddr[BBCOUNT]; /* bounce bufs in KVA space */ uint32_t bbuf_len[BBCOUNT]; /* len currently queued for bounce buf */ uint32_t bbuf_curidx; /* which bbuf is the active DMA buffer */ uint32_t xfer_offset; /* offset so far into caller's buf */ }; /* bus entry points */ static int at91_mci_probe(device_t dev); static int at91_mci_attach(device_t dev); static int at91_mci_detach(device_t dev); static void at91_mci_intr(void *); /* helper routines */ static int at91_mci_activate(device_t dev); static void at91_mci_deactivate(device_t dev); static int at91_mci_is_mci1rev2xx(void); #define AT91_MCI_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define AT91_MCI_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define AT91_MCI_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \ "mci", MTX_DEF) #define AT91_MCI_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define AT91_MCI_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define AT91_MCI_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); static inline uint32_t RD4(struct at91_mci_softc *sc, bus_size_t off) { return (bus_read_4(sc->mem_res, off)); } static inline void WR4(struct at91_mci_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(sc->mem_res, off, val); } static void at91_bswap_buf(struct at91_mci_softc *sc, void * dptr, void * sptr, uint32_t memsize) { uint32_t * dst = (uint32_t *)dptr; uint32_t * src = (uint32_t *)sptr; uint32_t i; /* * If the hardware doesn't need byte-swapping, let bcopy() do the * work. Use bounce buffer even if we don't need byteswap, since - * buffer may straddle a page boundry, and we don't handle + * buffer may straddle a page boundary, and we don't handle * multi-segment transfers in hardware. Seen from 'bsdlabel -w' which * uses raw geom access to the volume. Greg Ansley (gja (at) * ansley.com) */ if (!(sc->sc_cap & CAP_NEEDS_BYTESWAP)) { memcpy(dptr, sptr, memsize); return; } /* * Nice performance boost for slightly unrolling this loop. * (But very little extra boost for further unrolling it.) */ for (i = 0; i < memsize; i += 16) { *dst++ = bswap32(*src++); *dst++ = bswap32(*src++); *dst++ = bswap32(*src++); *dst++ = bswap32(*src++); } /* Mop up the last 1-3 words, if any. */ for (i = 0; i < (memsize & 0x0F); i += 4) { *dst++ = bswap32(*src++); } } static void at91_mci_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error != 0) return; *(bus_addr_t *)arg = segs[0].ds_addr; } static void at91_mci_pdc_disable(struct at91_mci_softc *sc) { WR4(sc, PDC_PTCR, PDC_PTCR_TXTDIS | PDC_PTCR_RXTDIS); WR4(sc, PDC_RPR, 0); WR4(sc, PDC_RCR, 0); WR4(sc, PDC_RNPR, 0); WR4(sc, PDC_RNCR, 0); WR4(sc, PDC_TPR, 0); WR4(sc, PDC_TCR, 0); WR4(sc, PDC_TNPR, 0); WR4(sc, PDC_TNCR, 0); } /* * Reset the controller, then restore most of the current state. * * This is called after detecting an error. It's also called after stopping a * multi-block write, to un-wedge the device so that it will handle the NOTBUSY * signal correctly. See comments in at91_mci_stop_done() for more details. */ static void at91_mci_reset(struct at91_mci_softc *sc) { uint32_t mr; uint32_t sdcr; uint32_t dtor; uint32_t imr; at91_mci_pdc_disable(sc); /* save current state */ imr = RD4(sc, MCI_IMR); mr = RD4(sc, MCI_MR) & 0x7fff; sdcr = RD4(sc, MCI_SDCR); dtor = RD4(sc, MCI_DTOR); /* reset the controller */ WR4(sc, MCI_IDR, 0xffffffff); WR4(sc, MCI_CR, MCI_CR_MCIDIS | MCI_CR_SWRST); /* restore state */ WR4(sc, MCI_CR, MCI_CR_MCIEN|MCI_CR_PWSEN); WR4(sc, MCI_MR, mr); WR4(sc, MCI_SDCR, sdcr); WR4(sc, MCI_DTOR, dtor); WR4(sc, MCI_IER, imr); /* * Make sure sdio interrupts will fire. Not sure why reading * SR ensures that, but this is in the linux driver. */ RD4(sc, MCI_SR); } static void at91_mci_init(device_t dev) { struct at91_mci_softc *sc = device_get_softc(dev); uint32_t val; WR4(sc, MCI_CR, MCI_CR_MCIDIS | MCI_CR_SWRST); /* device into reset */ WR4(sc, MCI_IDR, 0xffffffff); /* Turn off interrupts */ WR4(sc, MCI_DTOR, MCI_DTOR_DTOMUL_1M | 1); val = MCI_MR_PDCMODE; val |= 0x34a; /* PWSDIV = 3; CLKDIV = 74 */ // if (sc->sc_cap & CAP_MCI1_REV2XX) // val |= MCI_MR_RDPROOF | MCI_MR_WRPROOF; WR4(sc, MCI_MR, val); #ifndef AT91_MCI_SLOT_B WR4(sc, MCI_SDCR, 0); /* SLOT A, 1 bit bus */ #else /* * XXX Really should add second "unit" but nobody using using * a two slot card that we know of. XXX */ WR4(sc, MCI_SDCR, 1); /* SLOT B, 1 bit bus */ #endif /* * Enable controller, including power-save. The slower clock * of the power-save mode is only in effect when there is no * transfer in progress, so it can be left in this mode all * the time. */ WR4(sc, MCI_CR, MCI_CR_MCIEN|MCI_CR_PWSEN); } static void at91_mci_fini(device_t dev) { struct at91_mci_softc *sc = device_get_softc(dev); WR4(sc, MCI_IDR, 0xffffffff); /* Turn off interrupts */ at91_mci_pdc_disable(sc); WR4(sc, MCI_CR, MCI_CR_MCIDIS | MCI_CR_SWRST); /* device into reset */ } static int at91_mci_probe(device_t dev) { #ifdef FDT if (!ofw_bus_is_compatible(dev, "atmel,hsmci")) return (ENXIO); #endif device_set_desc(dev, "MCI mmc/sd host bridge"); return (0); } static int at91_mci_attach(device_t dev) { struct at91_mci_softc *sc = device_get_softc(dev); struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; device_t child; int err, i; sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); sc->dev = dev; sc->sc_cap = 0; if (at91_is_rm92()) sc->sc_cap |= CAP_NEEDS_BYTESWAP; /* * MCI1 Rev 2 controllers need some workarounds, flag if so. */ if (at91_mci_is_mci1rev2xx()) sc->sc_cap |= CAP_MCI1_REV2XX; err = at91_mci_activate(dev); if (err) goto out; AT91_MCI_LOCK_INIT(sc); at91_mci_fini(dev); at91_mci_init(dev); /* * Allocate DMA tags and maps and bounce buffers. * * The parms in the tag_create call cause the dmamem_alloc call to * create each bounce buffer as a single contiguous buffer of BBSIZE * bytes aligned to a 4096 byte boundary. * * Do not use DMA_COHERENT for these buffers because that maps the * memory as non-cachable, which prevents cache line burst fills/writes, * which is something we need since we're trying to overlap the * byte-swapping with the DMA operations. */ err = bus_dma_tag_create(bus_get_dma_tag(dev), 4096, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, BBSIZE, 1, BBSIZE, 0, NULL, NULL, &sc->dmatag); if (err != 0) goto out; for (i = 0; i < BBCOUNT; ++i) { err = bus_dmamem_alloc(sc->dmatag, (void **)&sc->bbuf_vaddr[i], BUS_DMA_NOWAIT, &sc->bbuf_map[i]); if (err != 0) goto out; } /* * Activate the interrupt */ err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, at91_mci_intr, sc, &sc->intrhand); if (err) { AT91_MCI_LOCK_DESTROY(sc); goto out; } /* * Allow 4-wire to be initially set via #define. * Allow a device hint to override that. * Allow a sysctl to override that. */ #if defined(AT91_MCI_HAS_4WIRE) && AT91_MCI_HAS_4WIRE != 0 sc->has_4wire = 1; #endif resource_int_value(device_get_name(dev), device_get_unit(dev), "4wire", &sc->has_4wire); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "4wire", CTLFLAG_RW, &sc->has_4wire, 0, "has 4 wire SD Card bus"); if (sc->has_4wire) sc->sc_cap |= CAP_HAS_4WIRE; sc->allow_overclock = AT91_MCI_ALLOW_OVERCLOCK; resource_int_value(device_get_name(dev), device_get_unit(dev), "allow_overclock", &sc->allow_overclock); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "allow_overclock", CTLFLAG_RW, &sc->allow_overclock, 0, "Allow up to 30MHz clock for 25MHz request when next highest speed 15MHz or less."); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "debug", CTLFLAG_RWTUN, &mci_debug, 0, "enable debug output"); /* * Our real min freq is master_clock/512, but upper driver layers are * going to set the min speed during card discovery, and the right speed * for that is 400kHz, so advertise a safe value just under that. * * For max speed, while the rm9200 manual says the max is 50mhz, it also * says it supports only the SD v1.0 spec, which means the real limit is * 25mhz. On the other hand, historical use has been to slightly violate * the standard by running the bus at 30MHz. For more information on * that, see the comments at the top of this file. */ sc->host.f_min = 375000; sc->host.f_max = at91_master_clock / 2; if (sc->host.f_max > 25000000) sc->host.f_max = 25000000; sc->host.host_ocr = MMC_OCR_320_330 | MMC_OCR_330_340; sc->host.caps = 0; if (sc->sc_cap & CAP_HAS_4WIRE) sc->host.caps |= MMC_CAP_4_BIT_DATA; child = device_add_child(dev, "mmc", 0); device_set_ivars(dev, &sc->host); err = bus_generic_attach(dev); out: if (err) at91_mci_deactivate(dev); return (err); } static int at91_mci_detach(device_t dev) { struct at91_mci_softc *sc = device_get_softc(dev); at91_mci_fini(dev); at91_mci_deactivate(dev); bus_dmamem_free(sc->dmatag, sc->bbuf_vaddr[0], sc->bbuf_map[0]); bus_dmamem_free(sc->dmatag, sc->bbuf_vaddr[1], sc->bbuf_map[1]); bus_dma_tag_destroy(sc->dmatag); return (EBUSY); /* XXX */ } static int at91_mci_activate(device_t dev) { struct at91_mci_softc *sc; int rid; sc = device_get_softc(dev); rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) goto errout; rid = 0; sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res == NULL) goto errout; return (0); errout: at91_mci_deactivate(dev); return (ENOMEM); } static void at91_mci_deactivate(device_t dev) { struct at91_mci_softc *sc; sc = device_get_softc(dev); if (sc->intrhand) bus_teardown_intr(dev, sc->irq_res, sc->intrhand); sc->intrhand = NULL; bus_generic_detach(sc->dev); if (sc->mem_res) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->mem_res), sc->mem_res); sc->mem_res = NULL; if (sc->irq_res) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res), sc->irq_res); sc->irq_res = NULL; return; } static int at91_mci_is_mci1rev2xx(void) { switch (soc_info.type) { case AT91_T_SAM9260: case AT91_T_SAM9263: case AT91_T_CAP9: case AT91_T_SAM9G10: case AT91_T_SAM9G20: case AT91_T_SAM9RL: return(1); default: return (0); } } static int at91_mci_update_ios(device_t brdev, device_t reqdev) { struct at91_mci_softc *sc; struct mmc_ios *ios; uint32_t clkdiv; uint32_t freq; sc = device_get_softc(brdev); ios = &sc->host.ios; /* * Calculate our closest available clock speed that doesn't exceed the * requested speed. * * When overclocking is allowed, the requested clock is 25MHz, the * computed frequency is 15MHz or smaller and clockdiv is 1, use * clockdiv of 0 to double that. If less than 12.5MHz, double * regardless of the overclocking setting. * * Whatever we come up with, store it back into ios->clock so that the * upper layer drivers can report the actual speed of the bus. */ if (ios->clock == 0) { WR4(sc, MCI_CR, MCI_CR_MCIDIS); clkdiv = 0; } else { WR4(sc, MCI_CR, MCI_CR_MCIEN|MCI_CR_PWSEN); if ((at91_master_clock % (ios->clock * 2)) == 0) clkdiv = ((at91_master_clock / ios->clock) / 2) - 1; else clkdiv = (at91_master_clock / ios->clock) / 2; freq = at91_master_clock / ((clkdiv+1) * 2); if (clkdiv == 1 && ios->clock == 25000000 && freq <= 15000000) { if (sc->allow_overclock || freq <= 12500000) { clkdiv = 0; freq = at91_master_clock / ((clkdiv+1) * 2); } } ios->clock = freq; } if (ios->bus_width == bus_width_4) WR4(sc, MCI_SDCR, RD4(sc, MCI_SDCR) | MCI_SDCR_SDCBUS); else WR4(sc, MCI_SDCR, RD4(sc, MCI_SDCR) & ~MCI_SDCR_SDCBUS); WR4(sc, MCI_MR, (RD4(sc, MCI_MR) & ~MCI_MR_CLKDIV) | clkdiv); /* Do we need a settle time here? */ /* XXX We need to turn the device on/off here with a GPIO pin */ return (0); } static void at91_mci_start_cmd(struct at91_mci_softc *sc, struct mmc_command *cmd) { uint32_t cmdr, mr; struct mmc_data *data; sc->curcmd = cmd; data = cmd->data; /* XXX Upper layers don't always set this */ cmd->mrq = sc->req; /* Begin setting up command register. */ cmdr = cmd->opcode; if (sc->host.ios.bus_mode == opendrain) cmdr |= MCI_CMDR_OPDCMD; /* Set up response handling. Allow max timeout for responses. */ if (MMC_RSP(cmd->flags) == MMC_RSP_NONE) cmdr |= MCI_CMDR_RSPTYP_NO; else { cmdr |= MCI_CMDR_MAXLAT; if (cmd->flags & MMC_RSP_136) cmdr |= MCI_CMDR_RSPTYP_136; else cmdr |= MCI_CMDR_RSPTYP_48; } /* * If there is no data transfer, just set up the right interrupt mask * and start the command. * * The interrupt mask needs to be CMDRDY plus all non-data-transfer * errors. It's important to leave the transfer-related errors out, to * avoid spurious timeout or crc errors on a STOP command following a * multiblock read. When a multiblock read is in progress, sending a * STOP in the middle of a block occasionally triggers such errors, but * we're totally disinterested in them because we've already gotten all * the data we wanted without error before sending the STOP command. */ if (data == NULL) { uint32_t ier = MCI_SR_CMDRDY | MCI_SR_RTOE | MCI_SR_RENDE | MCI_SR_RCRCE | MCI_SR_RDIRE | MCI_SR_RINDE; at91_mci_pdc_disable(sc); if (cmd->opcode == MMC_STOP_TRANSMISSION) cmdr |= MCI_CMDR_TRCMD_STOP; /* Ignore response CRC on CMD2 and ACMD41, per standard. */ if (cmd->opcode == MMC_SEND_OP_COND || cmd->opcode == ACMD_SD_SEND_OP_COND) ier &= ~MCI_SR_RCRCE; if (mci_debug) printf("CMDR %x (opcode %d) ARGR %x no data\n", cmdr, cmd->opcode, cmd->arg); WR4(sc, MCI_ARGR, cmd->arg); WR4(sc, MCI_CMDR, cmdr); WR4(sc, MCI_IDR, 0xffffffff); WR4(sc, MCI_IER, ier); return; } /* There is data, set up the transfer-related parts of the command. */ if (data->flags & MMC_DATA_READ) cmdr |= MCI_CMDR_TRDIR; if (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) cmdr |= MCI_CMDR_TRCMD_START; if (data->flags & MMC_DATA_STREAM) cmdr |= MCI_CMDR_TRTYP_STREAM; else if (data->flags & MMC_DATA_MULTI) { cmdr |= MCI_CMDR_TRTYP_MULTIPLE; sc->flags |= (data->flags & MMC_DATA_READ) ? CMD_MULTIREAD : CMD_MULTIWRITE; } /* * Disable PDC until we're ready. * * Set block size and turn on PDC mode for dma xfer. * Note that the block size is the smaller of the amount of data to be * transferred, or 512 bytes. The 512 size is fixed by the standard; * smaller blocks are possible, but never larger. */ WR4(sc, PDC_PTCR, PDC_PTCR_RXTDIS | PDC_PTCR_TXTDIS); mr = RD4(sc,MCI_MR) & ~MCI_MR_BLKLEN; mr |= min(data->len, 512) << 16; WR4(sc, MCI_MR, mr | MCI_MR_PDCMODE|MCI_MR_PDCPADV); /* * Set up DMA. * * Use bounce buffers even if we don't need to byteswap, because doing * multi-block IO with large DMA buffers is way fast (compared to * single-block IO), even after incurring the overhead of also copying * from/to the caller's buffers (which may be in non-contiguous physical * pages). * * In an ideal non-byteswap world we could create a dma tag that allows * for discontiguous segments and do the IO directly from/to the * caller's buffer(s), using ENDRX/ENDTX interrupts to chain the * discontiguous buffers through the PDC. Someday. * * If a read is bigger than 2k, split it in half so that we can start * byte-swapping the first half while the second half is on the wire. * It would be best if we could split it into 8k chunks, but we can't * always keep up with the byte-swapping due to other system activity, * and if an RXBUFF interrupt happens while we're still handling the * byte-swap from the prior buffer (IE, we haven't returned from * handling the prior interrupt yet), then data will get dropped on the * floor and we can't easily recover from that. The right fix for that * would be to have the interrupt handling only keep the DMA flowing and * enqueue filled buffers to be byte-swapped in a non-interrupt context. * Even that won't work on the write side of things though; in that * context we have to have all the data ready to go before starting the * dma. * * XXX what about stream transfers? */ sc->xfer_offset = 0; sc->bbuf_curidx = 0; if (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) { uint32_t len; uint32_t remaining = data->len; bus_addr_t paddr; int err; if (remaining > (BBCOUNT*BBSIZE)) panic("IO read size exceeds MAXDATA\n"); if (data->flags & MMC_DATA_READ) { if (remaining > 2048) // XXX len = remaining / 2; else len = remaining; err = bus_dmamap_load(sc->dmatag, sc->bbuf_map[0], sc->bbuf_vaddr[0], len, at91_mci_getaddr, &paddr, BUS_DMA_NOWAIT); if (err != 0) panic("IO read dmamap_load failed\n"); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[0], BUS_DMASYNC_PREREAD); WR4(sc, PDC_RPR, paddr); WR4(sc, PDC_RCR, len / 4); sc->bbuf_len[0] = len; remaining -= len; if (remaining == 0) { sc->bbuf_len[1] = 0; } else { len = remaining; err = bus_dmamap_load(sc->dmatag, sc->bbuf_map[1], sc->bbuf_vaddr[1], len, at91_mci_getaddr, &paddr, BUS_DMA_NOWAIT); if (err != 0) panic("IO read dmamap_load failed\n"); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[1], BUS_DMASYNC_PREREAD); WR4(sc, PDC_RNPR, paddr); WR4(sc, PDC_RNCR, len / 4); sc->bbuf_len[1] = len; remaining -= len; } WR4(sc, PDC_PTCR, PDC_PTCR_RXTEN); } else { len = min(BBSIZE, remaining); at91_bswap_buf(sc, sc->bbuf_vaddr[0], data->data, len); err = bus_dmamap_load(sc->dmatag, sc->bbuf_map[0], sc->bbuf_vaddr[0], len, at91_mci_getaddr, &paddr, BUS_DMA_NOWAIT); if (err != 0) panic("IO write dmamap_load failed\n"); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[0], BUS_DMASYNC_PREWRITE); /* * Erratum workaround: PDC transfer length on a write * must not be smaller than 12 bytes (3 words); only * blklen bytes (set above) are actually transferred. */ WR4(sc, PDC_TPR,paddr); WR4(sc, PDC_TCR, (len < 12) ? 3 : len / 4); sc->bbuf_len[0] = len; remaining -= len; if (remaining == 0) { sc->bbuf_len[1] = 0; } else { len = remaining; at91_bswap_buf(sc, sc->bbuf_vaddr[1], ((char *)data->data)+BBSIZE, len); err = bus_dmamap_load(sc->dmatag, sc->bbuf_map[1], sc->bbuf_vaddr[1], len, at91_mci_getaddr, &paddr, BUS_DMA_NOWAIT); if (err != 0) panic("IO write dmamap_load failed\n"); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[1], BUS_DMASYNC_PREWRITE); WR4(sc, PDC_TNPR, paddr); WR4(sc, PDC_TNCR, (len < 12) ? 3 : len / 4); sc->bbuf_len[1] = len; remaining -= len; } /* do not enable PDC xfer until CMDRDY asserted */ } data->xfer_len = 0; /* XXX what's this? appears to be unused. */ } if (mci_debug) printf("CMDR %x (opcode %d) ARGR %x with data len %d\n", cmdr, cmd->opcode, cmd->arg, cmd->data->len); WR4(sc, MCI_ARGR, cmd->arg); WR4(sc, MCI_CMDR, cmdr); WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_CMDRDY); } static void at91_mci_next_operation(struct at91_mci_softc *sc) { struct mmc_request *req; req = sc->req; if (req == NULL) return; if (sc->flags & PENDING_CMD) { sc->flags &= ~PENDING_CMD; at91_mci_start_cmd(sc, req->cmd); return; } else if (sc->flags & PENDING_STOP) { sc->flags &= ~PENDING_STOP; at91_mci_start_cmd(sc, req->stop); return; } WR4(sc, MCI_IDR, 0xffffffff); sc->req = NULL; sc->curcmd = NULL; //printf("req done\n"); req->done(req); } static int at91_mci_request(device_t brdev, device_t reqdev, struct mmc_request *req) { struct at91_mci_softc *sc = device_get_softc(brdev); AT91_MCI_LOCK(sc); if (sc->req != NULL) { AT91_MCI_UNLOCK(sc); return (EBUSY); } //printf("new req\n"); sc->req = req; sc->flags = PENDING_CMD; if (sc->req->stop) sc->flags |= PENDING_STOP; at91_mci_next_operation(sc); AT91_MCI_UNLOCK(sc); return (0); } static int at91_mci_get_ro(device_t brdev, device_t reqdev) { return (0); } static int at91_mci_acquire_host(device_t brdev, device_t reqdev) { struct at91_mci_softc *sc = device_get_softc(brdev); int err = 0; AT91_MCI_LOCK(sc); while (sc->bus_busy) msleep(sc, &sc->sc_mtx, PZERO, "mciah", hz / 5); sc->bus_busy++; AT91_MCI_UNLOCK(sc); return (err); } static int at91_mci_release_host(device_t brdev, device_t reqdev) { struct at91_mci_softc *sc = device_get_softc(brdev); AT91_MCI_LOCK(sc); sc->bus_busy--; wakeup(sc); AT91_MCI_UNLOCK(sc); return (0); } static void at91_mci_read_done(struct at91_mci_softc *sc, uint32_t sr) { struct mmc_command *cmd = sc->curcmd; char * dataptr = (char *)cmd->data->data; uint32_t curidx = sc->bbuf_curidx; uint32_t len = sc->bbuf_len[curidx]; /* * We arrive here when a DMA transfer for a read is done, whether it's * a single or multi-block read. * * We byte-swap the buffer that just completed, and if that is the * last buffer that's part of this read then we move on to the next * operation, otherwise we wait for another ENDRX for the next bufer. */ bus_dmamap_sync(sc->dmatag, sc->bbuf_map[curidx], BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->dmatag, sc->bbuf_map[curidx]); at91_bswap_buf(sc, dataptr + sc->xfer_offset, sc->bbuf_vaddr[curidx], len); if (mci_debug) { printf("read done sr %x curidx %d len %d xfer_offset %d\n", sr, curidx, len, sc->xfer_offset); } sc->xfer_offset += len; sc->bbuf_curidx = !curidx; /* swap buffers */ /* * If we've transferred all the data, move on to the next operation. * * If we're still transferring the last buffer, RNCR is already zero but * we have to write a zero anyway to clear the ENDRX status so we don't * re-interrupt until the last buffer is done. */ if (sc->xfer_offset == cmd->data->len) { WR4(sc, PDC_PTCR, PDC_PTCR_RXTDIS | PDC_PTCR_TXTDIS); cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } else { WR4(sc, PDC_RNCR, 0); WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_ENDRX); } } static void at91_mci_write_done(struct at91_mci_softc *sc, uint32_t sr) { struct mmc_command *cmd = sc->curcmd; /* * We arrive here when the entire DMA transfer for a write is done, * whether it's a single or multi-block write. If it's multi-block we * have to immediately move on to the next operation which is to send * the stop command. If it's a single-block transfer we need to wait * for NOTBUSY, but if that's already asserted we can avoid another * interrupt and just move on to completing the request right away. */ WR4(sc, PDC_PTCR, PDC_PTCR_RXTDIS | PDC_PTCR_TXTDIS); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[sc->bbuf_curidx], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->dmatag, sc->bbuf_map[sc->bbuf_curidx]); if ((cmd->data->flags & MMC_DATA_MULTI) || (sr & MCI_SR_NOTBUSY)) { cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } else { WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_NOTBUSY); } } static void at91_mci_notbusy(struct at91_mci_softc *sc) { struct mmc_command *cmd = sc->curcmd; /* * We arrive here by either completion of a single-block write, or * completion of the stop command that ended a multi-block write (and, * I suppose, after a card-select or erase, but I haven't tested * those). Anyway, we're done and it's time to move on to the next * command. */ cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } static void at91_mci_stop_done(struct at91_mci_softc *sc, uint32_t sr) { struct mmc_command *cmd = sc->curcmd; /* * We arrive here after receiving CMDRDY for a MMC_STOP_TRANSMISSION * command. Depending on the operation being stopped, we may have to * do some unusual things to work around hardware bugs. */ /* * This is known to be true of at91rm9200 hardware; it may or may not * apply to more recent chips: * * After stopping a multi-block write, the NOTBUSY bit in MCI_SR does * not properly reflect the actual busy state of the card as signaled * on the DAT0 line; it always claims the card is not-busy. If we * believe that and let operations continue, following commands will * fail with response timeouts (except of course MMC_SEND_STATUS -- it * indicates the card is busy in the PRG state, which was the smoking * gun that showed MCI_SR NOTBUSY was not tracking DAT0 correctly). * * The atmel docs are emphatic: "This flag [NOTBUSY] must be used only * for Write Operations." I guess technically since we sent a stop * it's not a write operation anymore. But then just what did they * think it meant for the stop command to have "...an optional busy * signal transmitted on the data line" according to the SD spec? * * I tried a variety of things to un-wedge the MCI and get the status * register to reflect NOTBUSY correctly again, but the only thing * that worked was a full device reset. It feels like an awfully big * hammer, but doing a full reset after every multiblock write is * still faster than doing single-block IO (by almost two orders of * magnitude: 20KB/sec improves to about 1.8MB/sec best case). * * After doing the reset, wait for a NOTBUSY interrupt before * continuing with the next operation. * * This workaround breaks multiwrite on the rev2xx parts, but some other * workaround is needed. */ if ((sc->flags & CMD_MULTIWRITE) && (sc->sc_cap & CAP_NEEDS_BYTESWAP)) { at91_mci_reset(sc); WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_NOTBUSY); return; } /* * This is known to be true of at91rm9200 hardware; it may or may not * apply to more recent chips: * * After stopping a multi-block read, loop to read and discard any * data that coasts in after we sent the stop command. The docs don't * say anything about it, but empirical testing shows that 1-3 * additional words of data get buffered up in some unmentioned * internal fifo and if we don't read and discard them here they end * up on the front of the next read DMA transfer we do. * * This appears to be unnecessary for rev2xx parts. */ if ((sc->flags & CMD_MULTIREAD) && (sc->sc_cap & CAP_NEEDS_BYTESWAP)) { uint32_t sr; int count = 0; do { sr = RD4(sc, MCI_SR); if (sr & MCI_SR_RXRDY) { RD4(sc, MCI_RDR); ++count; } } while (sr & MCI_SR_RXRDY); at91_mci_reset(sc); } cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } static void at91_mci_cmdrdy(struct at91_mci_softc *sc, uint32_t sr) { struct mmc_command *cmd = sc->curcmd; int i; if (cmd == NULL) return; /* * We get here at the end of EVERY command. We retrieve the command * response (if any) then decide what to do next based on the command. */ if (cmd->flags & MMC_RSP_PRESENT) { for (i = 0; i < ((cmd->flags & MMC_RSP_136) ? 4 : 1); i++) { cmd->resp[i] = RD4(sc, MCI_RSPR + i * 4); if (mci_debug) printf("RSPR[%d] = %x sr=%x\n", i, cmd->resp[i], sr); } } /* * If this was a stop command, go handle the various special * conditions (read: bugs) that have to be dealt with following a stop. */ if (cmd->opcode == MMC_STOP_TRANSMISSION) { at91_mci_stop_done(sc, sr); return; } /* * If this command can continue to assert BUSY beyond the response then * we need to wait for NOTBUSY before the command is really done. * * Note that this may not work properly on the at91rm9200. It certainly * doesn't work for the STOP command that follows a multi-block write, * so post-stop CMDRDY is handled separately; see the special handling * in at91_mci_stop_done(). * * Beside STOP, there are other R1B-type commands that use the busy * signal after CMDRDY: CMD7 (card select), CMD28-29 (write protect), * CMD38 (erase). I haven't tested any of them, but I rather expect * them all to have the same sort of problem with MCI_SR not actually * reflecting the state of the DAT0-line busy indicator. So this code * may need to grow some sort of special handling for them too. (This * just in: CMD7 isn't a problem right now because dev/mmc.c incorrectly * sets the response flags to R1 rather than R1B.) XXX */ if ((cmd->flags & MMC_RSP_BUSY)) { WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_NOTBUSY); return; } /* * If there is a data transfer with this command, then... * - If it's a read, we need to wait for ENDRX. * - If it's a write, now is the time to enable the PDC, and we need * to wait for a BLKE that follows a TXBUFE, because if we're doing * a split transfer we get a BLKE after the first half (when TPR/TCR * get loaded from TNPR/TNCR). So first we wait for the TXBUFE, and * the handling for that interrupt will then invoke the wait for the * subsequent BLKE which indicates actual completion. */ if (cmd->data) { uint32_t ier; if (cmd->data->flags & MMC_DATA_READ) { ier = MCI_SR_ENDRX; } else { ier = MCI_SR_TXBUFE; WR4(sc, PDC_PTCR, PDC_PTCR_TXTEN); } WR4(sc, MCI_IER, MCI_SR_ERROR | ier); return; } /* * If we made it to here, we don't need to wait for anything more for * the current command, move on to the next command (will complete the * request if there is no next command). */ cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } static void at91_mci_intr(void *arg) { struct at91_mci_softc *sc = (struct at91_mci_softc*)arg; struct mmc_command *cmd = sc->curcmd; uint32_t sr, isr; AT91_MCI_LOCK(sc); sr = RD4(sc, MCI_SR); isr = sr & RD4(sc, MCI_IMR); if (mci_debug) printf("i 0x%x sr 0x%x\n", isr, sr); /* * All interrupts are one-shot; disable it now. * The next operation will re-enable whatever interrupts it wants. */ WR4(sc, MCI_IDR, isr); if (isr & MCI_SR_ERROR) { if (isr & (MCI_SR_RTOE | MCI_SR_DTOE)) cmd->error = MMC_ERR_TIMEOUT; else if (isr & (MCI_SR_RCRCE | MCI_SR_DCRCE)) cmd->error = MMC_ERR_BADCRC; else if (isr & (MCI_SR_OVRE | MCI_SR_UNRE)) cmd->error = MMC_ERR_FIFO; else cmd->error = MMC_ERR_FAILED; /* * CMD8 is used to probe for SDHC cards, a standard SD card * will get a response timeout; don't report it because it's a * normal and expected condition. One might argue that all * error reporting should be left to higher levels, but when * they report at all it's always EIO, which isn't very * helpful. XXX bootverbose? */ if (cmd->opcode != 8) { device_printf(sc->dev, "IO error; status MCI_SR = 0x%b cmd opcode = %d%s\n", sr, MCI_SR_BITSTRING, cmd->opcode, (cmd->opcode != 12) ? "" : (sc->flags & CMD_MULTIREAD) ? " after read" : " after write"); /* XXX not sure RTOE needs a full reset, just a retry */ at91_mci_reset(sc); } at91_mci_next_operation(sc); } else { if (isr & MCI_SR_TXBUFE) { // printf("TXBUFE\n"); /* * We need to wait for a BLKE that follows TXBUFE * (intermediate BLKEs might happen after ENDTXes if * we're chaining multiple buffers). If BLKE is also * asserted at the time we get TXBUFE, we can avoid * another interrupt and process it right away, below. */ if (sr & MCI_SR_BLKE) isr |= MCI_SR_BLKE; else WR4(sc, MCI_IER, MCI_SR_BLKE); } if (isr & MCI_SR_RXBUFF) { // printf("RXBUFF\n"); } if (isr & MCI_SR_ENDTX) { // printf("ENDTX\n"); } if (isr & MCI_SR_ENDRX) { // printf("ENDRX\n"); at91_mci_read_done(sc, sr); } if (isr & MCI_SR_NOTBUSY) { // printf("NOTBUSY\n"); at91_mci_notbusy(sc); } if (isr & MCI_SR_DTIP) { // printf("Data transfer in progress\n"); } if (isr & MCI_SR_BLKE) { // printf("Block transfer end\n"); at91_mci_write_done(sc, sr); } if (isr & MCI_SR_TXRDY) { // printf("Ready to transmit\n"); } if (isr & MCI_SR_RXRDY) { // printf("Ready to receive\n"); } if (isr & MCI_SR_CMDRDY) { // printf("Command ready\n"); at91_mci_cmdrdy(sc, sr); } } AT91_MCI_UNLOCK(sc); } static int at91_mci_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct at91_mci_softc *sc = device_get_softc(bus); switch (which) { default: return (EINVAL); case MMCBR_IVAR_BUS_MODE: *(int *)result = sc->host.ios.bus_mode; break; case MMCBR_IVAR_BUS_WIDTH: *(int *)result = sc->host.ios.bus_width; break; case MMCBR_IVAR_CHIP_SELECT: *(int *)result = sc->host.ios.chip_select; break; case MMCBR_IVAR_CLOCK: *(int *)result = sc->host.ios.clock; break; case MMCBR_IVAR_F_MIN: *(int *)result = sc->host.f_min; break; case MMCBR_IVAR_F_MAX: *(int *)result = sc->host.f_max; break; case MMCBR_IVAR_HOST_OCR: *(int *)result = sc->host.host_ocr; break; case MMCBR_IVAR_MODE: *(int *)result = sc->host.mode; break; case MMCBR_IVAR_OCR: *(int *)result = sc->host.ocr; break; case MMCBR_IVAR_POWER_MODE: *(int *)result = sc->host.ios.power_mode; break; case MMCBR_IVAR_VDD: *(int *)result = sc->host.ios.vdd; break; case MMCBR_IVAR_CAPS: if (sc->has_4wire) { sc->sc_cap |= CAP_HAS_4WIRE; sc->host.caps |= MMC_CAP_4_BIT_DATA; } else { sc->sc_cap &= ~CAP_HAS_4WIRE; sc->host.caps &= ~MMC_CAP_4_BIT_DATA; } *(int *)result = sc->host.caps; break; case MMCBR_IVAR_MAX_DATA: /* * Something is wrong with the 2x parts and multiblock, so * just do 1 block at a time for now, which really kills * performance. */ if (sc->sc_cap & CAP_MCI1_REV2XX) *(int *)result = 1; else *(int *)result = MAX_BLOCKS; break; } return (0); } static int at91_mci_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct at91_mci_softc *sc = device_get_softc(bus); switch (which) { default: return (EINVAL); case MMCBR_IVAR_BUS_MODE: sc->host.ios.bus_mode = value; break; case MMCBR_IVAR_BUS_WIDTH: sc->host.ios.bus_width = value; break; case MMCBR_IVAR_CHIP_SELECT: sc->host.ios.chip_select = value; break; case MMCBR_IVAR_CLOCK: sc->host.ios.clock = value; break; case MMCBR_IVAR_MODE: sc->host.mode = value; break; case MMCBR_IVAR_OCR: sc->host.ocr = value; break; case MMCBR_IVAR_POWER_MODE: sc->host.ios.power_mode = value; break; case MMCBR_IVAR_VDD: sc->host.ios.vdd = value; break; /* These are read-only */ case MMCBR_IVAR_CAPS: case MMCBR_IVAR_HOST_OCR: case MMCBR_IVAR_F_MIN: case MMCBR_IVAR_F_MAX: case MMCBR_IVAR_MAX_DATA: return (EINVAL); } return (0); } static device_method_t at91_mci_methods[] = { /* device_if */ DEVMETHOD(device_probe, at91_mci_probe), DEVMETHOD(device_attach, at91_mci_attach), DEVMETHOD(device_detach, at91_mci_detach), /* Bus interface */ DEVMETHOD(bus_read_ivar, at91_mci_read_ivar), DEVMETHOD(bus_write_ivar, at91_mci_write_ivar), /* mmcbr_if */ DEVMETHOD(mmcbr_update_ios, at91_mci_update_ios), DEVMETHOD(mmcbr_request, at91_mci_request), DEVMETHOD(mmcbr_get_ro, at91_mci_get_ro), DEVMETHOD(mmcbr_acquire_host, at91_mci_acquire_host), DEVMETHOD(mmcbr_release_host, at91_mci_release_host), DEVMETHOD_END }; static driver_t at91_mci_driver = { "at91_mci", at91_mci_methods, sizeof(struct at91_mci_softc), }; static devclass_t at91_mci_devclass; #ifdef FDT DRIVER_MODULE(at91_mci, simplebus, at91_mci_driver, at91_mci_devclass, NULL, NULL); #else DRIVER_MODULE(at91_mci, atmelarm, at91_mci_driver, at91_mci_devclass, NULL, NULL); #endif DRIVER_MODULE(mmc, at91_mci, mmc_driver, mmc_devclass, NULL, NULL); MODULE_DEPEND(at91_mci, mmc, 1, 1, 1); Index: head/sys/arm/at91/at91_reset.S =================================================================== --- head/sys/arm/at91/at91_reset.S (revision 299068) +++ head/sys/arm/at91/at91_reset.S (revision 299069) @@ -1,57 +1,57 @@ #include #include #include #include __FBSDID("$FreeBSD$"); #define SDRAM_TR (AT91_BASE + \ AT91SAM9G20_SDRAMC_BASE + AT91SAM9G20_SDRAMC_TR) #define SDRAM_LPR (AT91_BASE + \ AT91SAM9G20_SDRAMC_BASE + AT91SAM9G20_SDRAMC_LPR) #define RSTC_RCR (AT91_BASE + \ AT91SAM9G20_RSTC_BASE + RST_CR) /* * From AT91SAM9G20 Datasheet errata 44:3.5: * - * When User Reset occurs durring SDRAM read acces, eh SDRAM clock is turned + * When User Reset occurs during SDRAM read access, the SDRAM clock is turned * off while data are ready to be read on the data bus. The SDRAM maintains * the data until the clock restarts. * * If the User reset is programed to assert a general reset, the data * maintained by the SDRAM leads to a data bus conflict and adversly affects * the boot memories connected to the EBI: * + NAND Flash boot functionality, if the system boots out of internal ROM. * + NOR Flash boot, if the system boots on an external memory connected to * the EBI CS0. * * Assembly code is mandatory for the following sequnce as ARM * instructions need to be piplined. * */ ENTRY(cpu_reset_sam9g20) /* Disable IRQs */ mrs r0, cpsr orr r0, r0, #0x80 msr cpsr_c, r0 /* Change Refresh to block all data access */ ldr r0, =SDRAM_TR ldr r1, =1 str r1, [r0] /* Prepare power down command */ ldr r0, =SDRAM_LPR ldr r1, =2 /* Prepare proc_reset and periph reset */ ldr r2, =RSTC_RCR ldr r3, =0xA5000005 /* perform power down command */ str r1, [r0] /* Perfom proc_reset and periph reset (in the ARM pipeline) */ str r3, [r2] Index: head/sys/arm/at91/at91reg.h =================================================================== --- head/sys/arm/at91/at91reg.h (revision 299068) +++ head/sys/arm/at91/at91reg.h (revision 299069) @@ -1,90 +1,90 @@ /*- * Copyright (c) 2009 Greg Ansley All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ */ #ifndef _AT91REG_H_ #define _AT91REG_H_ #include "opt_at91.h" /* Where builtin peripherals start in KVM */ #define AT91_BASE 0xd0000000 /* Where builtin peripherals start PA */ #define AT91_PA_BASE 0xf0000000 /* A few things that we count on being the same - * throught the whole family of SOCs */ + * throughout the whole family of SOCs */ -/* SYSC System Controler */ +/* SYSC System Controller */ /* System Registers */ #define AT91_SYS_BASE 0xffff000 #define AT91_SYS_SIZE 0x1000 #define AT91_DBGU0 0x0ffff200 /* Most */ #define AT91_DBGU1 0x0fffee00 /* SAM9263, CAP9, and SAM9G45 */ #define AT91_DBGU_SIZE 0x200 #define DBGU_C1R (64) /* Chip ID1 Register */ #define DBGU_C2R (68) /* Chip ID2 Register */ #define DBGU_FNTR (72) /* Force NTRST Register */ #define AT91_CPU_VERSION_MASK 0x0000001f #define AT91_CPU_FAMILY_MASK 0x0ff00000 #define AT91_CPU_RM9200 0x09290780 #define AT91_CPU_SAM9260 0x019803a0 #define AT91_CPU_SAM9261 0x019703a0 #define AT91_CPU_SAM9263 0x019607a0 #define AT91_CPU_SAM9G10 0x819903a0 #define AT91_CPU_SAM9G20 0x019905a0 #define AT91_CPU_SAM9G45 0x819b05a0 #define AT91_CPU_SAM9N12 0x819a07a0 #define AT91_CPU_SAM9RL64 0x019b03a0 #define AT91_CPU_SAM9X5 0x819a05a0 #define AT91_CPU_SAM9XE128 0x329973a0 #define AT91_CPU_SAM9XE256 0x329a93a0 #define AT91_CPU_SAM9XE512 0x329aa3a0 #define AT91_CPU_CAP9 0x039a03a0 #define AT91_EXID_SAM9M11 0x00000001 #define AT91_EXID_SAM9M10 0x00000002 #define AT91_EXID_SAM9G46 0x00000003 #define AT91_EXID_SAM9G45 0x00000004 #define AT91_EXID_SAM9G15 0x00000000 #define AT91_EXID_SAM9G35 0x00000001 #define AT91_EXID_SAM9X35 0x00000002 #define AT91_EXID_SAM9G25 0x00000003 #define AT91_EXID_SAM9X25 0x00000004 #define AT91_IRQ_SYSTEM 1 #endif /* _AT91REG_H_ */ Index: head/sys/arm/at91/at91sam9260.c =================================================================== --- head/sys/arm/at91/at91sam9260.c (revision 299068) +++ head/sys/arm/at91/at91sam9260.c (revision 299069) @@ -1,218 +1,218 @@ /*- * Copyright (c) 2005 Olivier Houchard. All rights reserved. * Copyright (c) 2010 Greg Ansley. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include /* * Standard priority levels for the system. 0 is lowest and 7 is highest. * These values are the ones Atmel uses for its Linux port */ static const int at91_irq_prio[32] = { 7, /* Advanced Interrupt Controller */ 7, /* System Peripherals */ 1, /* Parallel IO Controller A */ 1, /* Parallel IO Controller B */ 1, /* Parallel IO Controller C */ 0, /* Analog-to-Digital Converter */ 5, /* USART 0 */ 5, /* USART 1 */ 5, /* USART 2 */ 0, /* Multimedia Card Interface */ 2, /* USB Device Port */ 6, /* Two-Wire Interface */ 5, /* Serial Peripheral Interface 0 */ 5, /* Serial Peripheral Interface 1 */ 5, /* Serial Synchronous Controller */ 0, /* (reserved) */ 0, /* (reserved) */ 0, /* Timer Counter 0 */ 0, /* Timer Counter 1 */ 0, /* Timer Counter 2 */ 2, /* USB Host port */ 3, /* Ethernet */ 0, /* Image Sensor Interface */ 5, /* USART 3 */ 5, /* USART 4 */ 5, /* USART 5 */ 0, /* Timer Counter 3 */ 0, /* Timer Counter 4 */ 0, /* Timer Counter 5 */ 0, /* Advanced Interrupt Controller IRQ0 */ 0, /* Advanced Interrupt Controller IRQ1 */ 0, /* Advanced Interrupt Controller IRQ2 */ }; static const uint32_t at91_pio_base[] = { AT91SAM9260_PIOA_BASE, AT91SAM9260_PIOB_BASE, AT91SAM9260_PIOC_BASE, }; #define DEVICE(_name, _id, _unit) \ { \ _name, _unit, \ AT91SAM9260_ ## _id ##_BASE, \ AT91SAM9260_ ## _id ## _SIZE, \ AT91SAM9260_IRQ_ ## _id \ } static const struct cpu_devs at91_devs[] = { DEVICE("at91_aic", AIC, 0), DEVICE("at91_pmc", PMC, 0), DEVICE("at91_wdt", WDT, 0), DEVICE("at91_rst", RSTC, 0), DEVICE("at91_pit", PIT, 0), DEVICE("at91_pio", PIOA, 0), DEVICE("at91_pio", PIOB, 1), DEVICE("at91_pio", PIOC, 2), DEVICE("at91_twi", TWI, 0), DEVICE("at91_mci", MCI, 0), DEVICE("uart", DBGU, 0), DEVICE("uart", USART0, 1), DEVICE("uart", USART1, 2), DEVICE("uart", USART2, 3), DEVICE("uart", USART3, 4), DEVICE("uart", USART4, 5), DEVICE("uart", USART5, 6), DEVICE("spi", SPI0, 0), DEVICE("spi", SPI1, 1), DEVICE("ate", EMAC, 0), DEVICE("macb", EMAC, 0), DEVICE("nand", NAND, 0), DEVICE("ohci", OHCI, 0), { 0, 0, 0, 0, 0 } }; /* * The following is unused currently since we don't ever set the PLLA * frequency of the device. */ static uint32_t at91_pll_outa(int freq) { uint32_t outa = 0; /* * Set OUTA, per the data sheet. See Table 40-15 titled * PLLA Characteristics in the SAM9260 doc. */ if (freq > 155000000) outa = 2 << 14; return ((1 << 29) | outa); } static uint32_t at91_pll_outb(int freq) { return (1 << 14); } static void at91_clock_init(void) { struct at91_pmc_clock *clk; /* Update USB device port clock info */ clk = at91_pmc_clock_ref("udpck"); clk->pmc_mask = PMC_SCER_UDP_SAM9; at91_pmc_clock_deref(clk); /* Update USB host port clock info */ clk = at91_pmc_clock_ref("uhpck"); clk->pmc_mask = PMC_SCER_UHP_SAM9; at91_pmc_clock_deref(clk); /* Each SOC has different PLL contraints */ clk = at91_pmc_clock_ref("plla"); clk->pll_min_in = SAM9260_PLL_A_MIN_IN_FREQ; /* 1 MHz */ clk->pll_max_in = SAM9260_PLL_A_MAX_IN_FREQ; /* 32 MHz */ clk->pll_min_out = SAM9260_PLL_A_MIN_OUT_FREQ; /* 80 MHz */ clk->pll_max_out = SAM9260_PLL_A_MAX_OUT_FREQ; /* 240 MHz */ clk->pll_mul_shift = SAM9260_PLL_A_MUL_SHIFT; clk->pll_mul_mask = SAM9260_PLL_A_MUL_MASK; clk->pll_div_shift = SAM9260_PLL_A_DIV_SHIFT; clk->pll_div_mask = SAM9260_PLL_A_DIV_MASK; clk->set_outb = at91_pll_outa; at91_pmc_clock_deref(clk); /* * Fudge MAX pll in frequence down below 3.0 MHz to ensure * PMC alogrithm choose the divisor that causes the input clock * to be near the optimal 2 MHz per datasheet. We know * we are going to be using this for the USB clock at 96 MHz. - * Causes no extra frequency deviation for all recomended crystal + * Causes no extra frequency deviation for all recommended crystal * values. See Note 1, table 40-16 SAM9260 doc. */ clk = at91_pmc_clock_ref("pllb"); clk->pll_min_in = SAM9260_PLL_B_MIN_IN_FREQ; /* 1 MHz */ clk->pll_max_in = SAM9260_PLL_B_MAX_IN_FREQ; /* 5 MHz */ clk->pll_max_in = 2999999; /* ~3 MHz */ clk->pll_min_out = SAM9260_PLL_B_MIN_OUT_FREQ; /* 70 MHz */ clk->pll_max_out = SAM9260_PLL_B_MAX_OUT_FREQ; /* 130 MHz */ clk->pll_mul_shift = SAM9260_PLL_B_MUL_SHIFT; clk->pll_mul_mask = SAM9260_PLL_B_MUL_MASK; clk->pll_div_shift = SAM9260_PLL_B_DIV_SHIFT; clk->pll_div_mask = SAM9260_PLL_B_DIV_MASK; clk->set_outb = at91_pll_outb; at91_pmc_clock_deref(clk); } static struct at91_soc_data soc_data = { .soc_delay = at91_pit_delay, .soc_reset = at91_rst_cpu_reset, .soc_clock_init = at91_clock_init, .soc_irq_prio = at91_irq_prio, .soc_children = at91_devs, .soc_pio_base = at91_pio_base, .soc_pio_count = nitems(at91_pio_base), }; AT91_SOC(AT91_T_SAM9260, &soc_data); Index: head/sys/arm/at91/if_ate.c =================================================================== --- head/sys/arm/at91/if_ate.c (revision 299068) +++ head/sys/arm/at91/if_ate.c (revision 299069) @@ -1,1531 +1,1531 @@ /*- * Copyright (c) 2006 M. Warner Losh. All rights reserved. * Copyright (c) 2009 Greg Ansley. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* TODO * * 1) Turn on the clock in pmc? Turn off? * 2) GPIO initializtion in board setup code. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET #include #include #include #include #endif #include #include #include #include #include "opt_at91.h" #include #include #include #ifdef FDT #include #include #include #endif #include "miibus_if.h" /* * Driver-specific flags. */ #define ATE_FLAG_DETACHING 0x01 #define ATE_FLAG_MULTICAST 0x02 /* * Old EMAC assumes whole packet fits in one buffer; * new EBACB assumes all receive buffers are 128 bytes */ #define RX_BUF_SIZE(sc) (sc->is_emacb ? 128 : MCLBYTES) /* * EMACB has an 11 bit counter for Rx/Tx Descriptors * for max total of 1024 decriptors each. */ #define ATE_MAX_RX_DESCR 1024 #define ATE_MAX_TX_DESCR 1024 /* How many buffers to allocate */ #define ATE_MAX_TX_BUFFERS 4 /* We have ping-pong tx buffers */ /* How much memory to use for rx buffers */ #define ATE_RX_MEMORY (ATE_MAX_RX_DESCR * 128) /* Actual number of descriptors we allocate */ #define ATE_NUM_RX_DESCR ATE_MAX_RX_DESCR #define ATE_NUM_TX_DESCR ATE_MAX_TX_BUFFERS #if ATE_NUM_TX_DESCR > ATE_MAX_TX_DESCR #error "Can't have more TX buffers that descriptors" #endif #if ATE_NUM_RX_DESCR > ATE_MAX_RX_DESCR #error "Can't have more RX buffers that descriptors" #endif /* Wrap indexes the same way the hardware does */ #define NEXT_RX_IDX(sc, cur) \ ((sc->rx_descs[cur].addr & ETH_WRAP_BIT) ? 0 : (cur + 1)) #define NEXT_TX_IDX(sc, cur) \ ((sc->tx_descs[cur].status & ETHB_TX_WRAP) ? 0 : (cur + 1)) struct ate_softc { struct ifnet *ifp; /* ifnet pointer */ struct mtx sc_mtx; /* Basically a perimeter lock */ device_t dev; /* Myself */ device_t miibus; /* My child miibus */ struct resource *irq_res; /* IRQ resource */ struct resource *mem_res; /* Memory resource */ struct callout tick_ch; /* Tick callout */ struct ifmib_iso_8802_3 mibdata; /* Stuff for network mgmt */ bus_dma_tag_t mtag; /* bus dma tag for mbufs */ bus_dma_tag_t rx_tag; bus_dma_tag_t rx_desc_tag; bus_dmamap_t rx_desc_map; bus_dmamap_t rx_map[ATE_MAX_RX_DESCR]; bus_addr_t rx_desc_phys; /* PA of rx descriptors */ eth_rx_desc_t *rx_descs; /* VA of rx descriptors */ void *rx_buf[ATE_NUM_RX_DESCR]; /* RX buffer space */ int rxhead; /* Current RX map/desc index */ uint32_t rx_buf_size; /* Size of Rx buffers */ bus_dma_tag_t tx_desc_tag; bus_dmamap_t tx_desc_map; bus_dmamap_t tx_map[ATE_MAX_TX_BUFFERS]; bus_addr_t tx_desc_phys; /* PA of tx descriptors */ eth_tx_desc_t *tx_descs; /* VA of tx descriptors */ int txhead; /* Current TX map/desc index */ int txtail; /* Current TX map/desc index */ struct mbuf *sent_mbuf[ATE_MAX_TX_BUFFERS]; /* Sent mbufs */ void *intrhand; /* Interrupt handle */ int flags; int if_flags; int use_rmii; int is_emacb; /* SAM9x hardware version */ }; static inline uint32_t RD4(struct ate_softc *sc, bus_size_t off) { return (bus_read_4(sc->mem_res, off)); } static inline void WR4(struct ate_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(sc->mem_res, off, val); } static inline void BARRIER(struct ate_softc *sc, bus_size_t off, bus_size_t len, int flags) { bus_barrier(sc->mem_res, off, len, flags); } #define ATE_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define ATE_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define ATE_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \ MTX_NETWORK_LOCK, MTX_DEF) #define ATE_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define ATE_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define ATE_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); static devclass_t ate_devclass; /* * ifnet entry points. */ static void ateinit_locked(void *); static void atestart_locked(struct ifnet *); static void ateinit(void *); static void atestart(struct ifnet *); static void atestop(struct ate_softc *); static int ateioctl(struct ifnet * ifp, u_long, caddr_t); /* * Bus entry points. */ static int ate_probe(device_t dev); static int ate_attach(device_t dev); static int ate_detach(device_t dev); static void ate_intr(void *); /* * Helper routines. */ static int ate_activate(device_t dev); static void ate_deactivate(struct ate_softc *sc); static int ate_ifmedia_upd(struct ifnet *ifp); static void ate_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr); static int ate_get_mac(struct ate_softc *sc, u_char *eaddr); static void ate_set_mac(struct ate_softc *sc, u_char *eaddr); static void ate_rxfilter(struct ate_softc *sc); static int ate_miibus_readreg(device_t dev, int phy, int reg); static int ate_miibus_writereg(device_t dev, int phy, int reg, int data); /* * The AT91 family of products has the ethernet interface called EMAC. * However, it isn't self identifying. It is anticipated that the parent bus * code will take care to only add ate devices where they really are. As * such, we do nothing here to identify the device and just set its name. * However, FDT makes it self-identifying. */ static int ate_probe(device_t dev) { #ifdef FDT if (!ofw_bus_is_compatible(dev, "cdns,at91rm9200-emac") && !ofw_bus_is_compatible(dev, "cdns,emac") && !ofw_bus_is_compatible(dev, "cdns,at32ap7000-macb")) return (ENXIO); #endif device_set_desc(dev, "EMAC"); return (0); } #ifdef FDT /* * We have to know if we're using MII or RMII attachment * for the MACB to talk to the PHY correctly. With FDT, * we must use rmii if there's a proprety phy-mode * equal to "rmii". Otherwise we MII mode is used. */ static void ate_set_rmii(struct ate_softc *sc) { phandle_t node; char prop[10]; ssize_t len; node = ofw_bus_get_node(sc->dev); memset(prop, 0 ,sizeof(prop)); len = OF_getproplen(node, "phy-mode"); if (len != 4) return; if (OF_getprop(node, "phy-mode", prop, len) != len) return; if (strncmp(prop, "rmii", 4) == 0) sc->use_rmii = 1; } #else /* * We have to know if we're using MII or RMII attachment * for the MACB to talk to the PHY correctly. Without FDT, * there's no good way to do this. So, if the config file * has 'option AT91_ATE_USE_RMII', then we'll force RMII. * Otherwise, we'll use what the bootloader setup. Either * it setup RMII or MII, in which case we'll get it right, * or it did nothing, and we'll fall back to MII and the * option would override if present. */ static void ate_set_rmii(struct ate_softc *sc) { /* Default to what boot rom did */ if (!sc->is_emacb) sc->use_rmii = (RD4(sc, ETH_CFG) & ETH_CFG_RMII) == ETH_CFG_RMII; else sc->use_rmii = (RD4(sc, ETHB_UIO) & ETHB_UIO_RMII) == ETHB_UIO_RMII; #ifdef AT91_ATE_USE_RMII /* Compile time override */ sc->use_rmii = 1; #endif } #endif static int ate_attach(device_t dev) { struct ate_softc *sc; struct ifnet *ifp = NULL; struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; u_char eaddr[ETHER_ADDR_LEN]; uint32_t rnd; int rid, err; sc = device_get_softc(dev); sc->dev = dev; ATE_LOCK_INIT(sc); rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) { device_printf(dev, "could not allocate memory resources.\n"); err = ENOMEM; goto out; } rid = 0; sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res == NULL) { device_printf(dev, "could not allocate interrupt resources.\n"); err = ENOMEM; goto out; } /* New or old version, chooses buffer size. */ #ifdef FDT sc->is_emacb = ofw_bus_is_compatible(dev, "cdns,at32ap7000-macb"); #else sc->is_emacb = at91_is_sam9() || at91_is_sam9xe(); #endif sc->rx_buf_size = RX_BUF_SIZE(sc); err = ate_activate(dev); if (err) goto out; ate_set_rmii(sc); /* Sysctls */ sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "rmii", CTLFLAG_RW, &sc->use_rmii, 0, "rmii in use"); /* Calling atestop before ifp is set is OK. */ ATE_LOCK(sc); atestop(sc); ATE_UNLOCK(sc); callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0); if ((err = ate_get_mac(sc, eaddr)) != 0) { /* No MAC address configured. Generate the random one. */ if (bootverbose) device_printf(dev, "Generating random ethernet address.\n"); rnd = arc4random(); /* * Set OUI to convenient locally assigned address. 'b' * is 0x62, which has the locally assigned bit set, and * the broadcast/multicast bit clear. */ eaddr[0] = 'b'; eaddr[1] = 's'; eaddr[2] = 'd'; eaddr[3] = (rnd >> 16) & 0xff; eaddr[4] = (rnd >> 8) & 0xff; eaddr[5] = (rnd >> 0) & 0xff; } sc->ifp = ifp = if_alloc(IFT_ETHER); err = mii_attach(dev, &sc->miibus, ifp, ate_ifmedia_upd, ate_ifmedia_sts, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 0); if (err != 0) { device_printf(dev, "attaching PHYs failed\n"); goto out; } /* * XXX: Clear the isolate bit, or we won't get up, * at least on the HL201 */ ate_miibus_writereg(dev, 0, 0, 0x3000); ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_capabilities |= IFCAP_VLAN_MTU; ifp->if_capenable |= IFCAP_VLAN_MTU; /* The hw bits already set. */ ifp->if_start = atestart; ifp->if_ioctl = ateioctl; ifp->if_init = ateinit; ifp->if_baudrate = 10000000; IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN); ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; IFQ_SET_READY(&ifp->if_snd); ifp->if_linkmib = &sc->mibdata; ifp->if_linkmiblen = sizeof(sc->mibdata); sc->mibdata.dot3Compliance = DOT3COMPLIANCE_COLLS; sc->if_flags = ifp->if_flags; ether_ifattach(ifp, eaddr); /* Activate the interrupt. */ err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE, NULL, ate_intr, sc, &sc->intrhand); if (err) { device_printf(dev, "could not establish interrupt handler.\n"); ether_ifdetach(ifp); goto out; } out: if (err) ate_detach(dev); return (err); } static int ate_detach(device_t dev) { struct ate_softc *sc; struct ifnet *ifp; sc = device_get_softc(dev); KASSERT(sc != NULL, ("[ate: %d]: sc is NULL", __LINE__)); ifp = sc->ifp; if (device_is_attached(dev)) { ATE_LOCK(sc); sc->flags |= ATE_FLAG_DETACHING; atestop(sc); ATE_UNLOCK(sc); callout_drain(&sc->tick_ch); ether_ifdetach(ifp); } if (sc->miibus != NULL) { device_delete_child(dev, sc->miibus); sc->miibus = NULL; } bus_generic_detach(sc->dev); ate_deactivate(sc); if (sc->intrhand != NULL) { bus_teardown_intr(dev, sc->irq_res, sc->intrhand); sc->intrhand = NULL; } if (ifp != NULL) { if_free(ifp); sc->ifp = NULL; } if (sc->mem_res != NULL) { bus_release_resource(dev, SYS_RES_IOPORT, rman_get_rid(sc->mem_res), sc->mem_res); sc->mem_res = NULL; } if (sc->irq_res != NULL) { bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res), sc->irq_res); sc->irq_res = NULL; } ATE_LOCK_DESTROY(sc); return (0); } static void ate_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error != 0) return; *(bus_addr_t *)arg = segs[0].ds_addr; } static void ate_load_rx_buf(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { struct ate_softc *sc; if (error != 0) return; sc = (struct ate_softc *)arg; bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_PREWRITE); sc->rx_descs[sc->rxhead].addr = segs[0].ds_addr; sc->rx_descs[sc->rxhead].status = 0; bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_POSTWRITE); } static uint32_t ate_mac_hash(const uint8_t *buf) { uint32_t index = 0; for (int i = 0; i < 48; i++) { index ^= ((buf[i >> 3] >> (i & 7)) & 1) << (i % 6); } return (index); } /* * Compute the multicast filter for this device. */ static int ate_setmcast(struct ate_softc *sc) { uint32_t index; uint32_t mcaf[2]; u_char *af = (u_char *) mcaf; struct ifmultiaddr *ifma; struct ifnet *ifp; ifp = sc->ifp; if ((ifp->if_flags & IFF_PROMISC) != 0) return (0); if ((ifp->if_flags & IFF_ALLMULTI) != 0) { WR4(sc, ETH_HSL, 0xffffffff); WR4(sc, ETH_HSH, 0xffffffff); return (1); } /* Compute the multicast hash. */ mcaf[0] = 0; mcaf[1] = 0; if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; index = ate_mac_hash(LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); af[index >> 3] |= 1 << (index & 7); } if_maddr_runlock(ifp); /* * Write the hash to the hash register. This card can also * accept unicast packets as well as multicast packets using this * register for easier bridging operations, but we don't take * advantage of that. Locks here are to avoid LOR with the * if_maddr_rlock, but might not be strictly necessary. */ WR4(sc, ETH_HSL, mcaf[0]); WR4(sc, ETH_HSH, mcaf[1]); return (mcaf[0] || mcaf[1]); } static int ate_activate(device_t dev) { struct ate_softc *sc; int i; sc = device_get_softc(dev); /* Allocate DMA tags and maps for TX mbufs */ if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, MCLBYTES, 0, busdma_lock_mutex, &sc->sc_mtx, &sc->mtag)) goto errout; for (i = 0; i < ATE_MAX_TX_BUFFERS; i++) { if ( bus_dmamap_create(sc->mtag, 0, &sc->tx_map[i])) goto errout; } /* DMA tag and map for the RX descriptors. */ if (bus_dma_tag_create(bus_get_dma_tag(dev), sizeof(eth_rx_desc_t), 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, ATE_NUM_RX_DESCR * sizeof(eth_rx_desc_t), 1, ATE_NUM_RX_DESCR * sizeof(eth_rx_desc_t), 0, busdma_lock_mutex, &sc->sc_mtx, &sc->rx_desc_tag)) goto errout; if (bus_dmamem_alloc(sc->rx_desc_tag, (void **)&sc->rx_descs, BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->rx_desc_map) != 0) goto errout; if (bus_dmamap_load(sc->rx_desc_tag, sc->rx_desc_map, sc->rx_descs, ATE_NUM_RX_DESCR * sizeof(eth_rx_desc_t), ate_getaddr, &sc->rx_desc_phys, 0) != 0) goto errout; /* Allocate DMA tags and maps for RX. buffers */ if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, sc->rx_buf_size, 1, sc->rx_buf_size, 0, busdma_lock_mutex, &sc->sc_mtx, &sc->rx_tag)) goto errout; /* * Allocate our RX buffers. * This chip has a RX structure that's filled in. * XXX On MACB (SAM9 part) we should receive directly into mbuf * to avoid the copy. XXX */ sc->rxhead = 0; for (sc->rxhead = 0; sc->rxhead < ATE_RX_MEMORY/sc->rx_buf_size; sc->rxhead++) { if (bus_dmamem_alloc(sc->rx_tag, (void **)&sc->rx_buf[sc->rxhead], BUS_DMA_NOWAIT, &sc->rx_map[sc->rxhead]) != 0) goto errout; if (bus_dmamap_load(sc->rx_tag, sc->rx_map[sc->rxhead], sc->rx_buf[sc->rxhead], sc->rx_buf_size, ate_load_rx_buf, sc, 0) != 0) { printf("bus_dmamem_load\n"); goto errout; } bus_dmamap_sync(sc->rx_tag, sc->rx_map[sc->rxhead], BUS_DMASYNC_PREREAD); } /* * For the last buffer, set the wrap bit so the controller * restarts from the first descriptor. */ sc->rx_descs[--sc->rxhead].addr |= ETH_WRAP_BIT; sc->rxhead = 0; /* Flush the memory for the EMAC rx descriptor. */ bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_PREWRITE); /* Write the descriptor queue address. */ WR4(sc, ETH_RBQP, sc->rx_desc_phys); /* * DMA tag and map for the TX descriptors. */ if (bus_dma_tag_create(bus_get_dma_tag(dev), sizeof(eth_tx_desc_t), 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, ATE_MAX_TX_BUFFERS * sizeof(eth_tx_desc_t), 1, ATE_MAX_TX_BUFFERS * sizeof(eth_tx_desc_t), 0, busdma_lock_mutex, &sc->sc_mtx, &sc->tx_desc_tag) != 0) goto errout; if (bus_dmamem_alloc(sc->tx_desc_tag, (void **)&sc->tx_descs, BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->tx_desc_map) != 0) goto errout; if (bus_dmamap_load(sc->tx_desc_tag, sc->tx_desc_map, sc->tx_descs, ATE_MAX_TX_BUFFERS * sizeof(eth_tx_desc_t), ate_getaddr, &sc->tx_desc_phys, 0) != 0) goto errout; - /* Initilize descriptors; mark all empty */ + /* Initialize descriptors; mark all empty */ for (i = 0; i < ATE_MAX_TX_BUFFERS; i++) { sc->tx_descs[i].addr =0; sc->tx_descs[i].status = ETHB_TX_USED; sc->sent_mbuf[i] = NULL; } /* Mark last entry to cause wrap when indexing through */ sc->tx_descs[ATE_MAX_TX_BUFFERS - 1].status = ETHB_TX_WRAP | ETHB_TX_USED; /* Flush the memory for the EMAC tx descriptor. */ bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_PREWRITE); sc->txhead = sc->txtail = 0; if (sc->is_emacb) { /* Write the descriptor queue address. */ WR4(sc, ETHB_TBQP, sc->tx_desc_phys); /* EMACB: Enable transceiver input clock */ WR4(sc, ETHB_UIO, RD4(sc, ETHB_UIO) | ETHB_UIO_CLKE); } return (0); errout: return (ENOMEM); } static void ate_deactivate(struct ate_softc *sc) { int i; KASSERT(sc != NULL, ("[ate, %d]: sc is NULL!", __LINE__)); if (sc->mtag != NULL) { for (i = 0; i < ATE_MAX_TX_BUFFERS; i++) { if (sc->sent_mbuf[i] != NULL) { bus_dmamap_sync(sc->mtag, sc->tx_map[i], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mtag, sc->tx_map[i]); m_freem(sc->sent_mbuf[i]); } bus_dmamap_destroy(sc->mtag, sc->tx_map[i]); sc->sent_mbuf[i] = NULL; sc->tx_map[i] = NULL; } bus_dma_tag_destroy(sc->mtag); } if (sc->rx_desc_tag != NULL) { if (sc->rx_descs != NULL) { if (sc->rx_desc_phys != 0) { bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->rx_desc_tag, sc->rx_desc_map); sc->rx_desc_phys = 0; } } } if (sc->rx_tag != NULL) { for (i = 0; sc->rx_buf[i] != NULL; i++) { if (sc->rx_descs[i].addr != 0) { bus_dmamap_sync(sc->rx_tag, sc->rx_map[i], BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->rx_tag, sc->rx_map[i]); sc->rx_descs[i].addr = 0; } bus_dmamem_free(sc->rx_tag, sc->rx_buf[i], sc->rx_map[i]); sc->rx_buf[i] = NULL; } bus_dma_tag_destroy(sc->rx_tag); } if (sc->rx_desc_tag != NULL) { if (sc->rx_descs != NULL) bus_dmamem_free(sc->rx_desc_tag, sc->rx_descs, sc->rx_desc_map); bus_dma_tag_destroy(sc->rx_desc_tag); sc->rx_descs = NULL; sc->rx_desc_tag = NULL; } if (sc->is_emacb) WR4(sc, ETHB_UIO, RD4(sc, ETHB_UIO) & ~ETHB_UIO_CLKE); } /* * Change media according to request. */ static int ate_ifmedia_upd(struct ifnet *ifp) { struct ate_softc *sc = ifp->if_softc; struct mii_data *mii; mii = device_get_softc(sc->miibus); ATE_LOCK(sc); mii_mediachg(mii); ATE_UNLOCK(sc); return (0); } /* * Notify the world which media we're using. */ static void ate_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) { struct ate_softc *sc = ifp->if_softc; struct mii_data *mii; mii = device_get_softc(sc->miibus); ATE_LOCK(sc); mii_pollstat(mii); ifmr->ifm_active = mii->mii_media_active; ifmr->ifm_status = mii->mii_media_status; ATE_UNLOCK(sc); } static void ate_stat_update(struct ate_softc *sc, int active) { uint32_t reg; /* * The speed and full/half-duplex state needs to be reflected * in the ETH_CFG register. */ reg = RD4(sc, ETH_CFG); reg &= ~(ETH_CFG_SPD | ETH_CFG_FD); if (IFM_SUBTYPE(active) != IFM_10_T) reg |= ETH_CFG_SPD; if (active & IFM_FDX) reg |= ETH_CFG_FD; WR4(sc, ETH_CFG, reg); } static void ate_tick(void *xsc) { struct ate_softc *sc = xsc; struct ifnet *ifp = sc->ifp; struct mii_data *mii; int active; uint32_t c; /* * The KB920x boot loader tests ETH_SR & ETH_SR_LINK and will ask * the MII if there's a link if this bit is clear. Not sure if we * should do the same thing here or not. */ ATE_ASSERT_LOCKED(sc); if (sc->miibus != NULL) { mii = device_get_softc(sc->miibus); active = mii->mii_media_active; mii_tick(mii); if (mii->mii_media_status & IFM_ACTIVE && active != mii->mii_media_active) ate_stat_update(sc, mii->mii_media_active); } /* * Update the stats as best we can. When we're done, clear * the status counters and start over. We're supposed to read these * registers often enough that they won't overflow. Hopefully * once a second is often enough. Some don't map well to * the dot3Stats mib, so for those we just count them as general * errors. Stats for iframes, ibutes, oframes and obytes are * collected elsewhere. These registers zero on a read to prevent * races. For all the collision stats, also update the collision * stats for the interface. */ sc->mibdata.dot3StatsAlignmentErrors += RD4(sc, ETH_ALE); sc->mibdata.dot3StatsFCSErrors += RD4(sc, ETH_SEQE); c = RD4(sc, ETH_SCOL); if_inc_counter(ifp, IFCOUNTER_COLLISIONS, c); sc->mibdata.dot3StatsSingleCollisionFrames += c; c = RD4(sc, ETH_MCOL); sc->mibdata.dot3StatsMultipleCollisionFrames += c; if_inc_counter(ifp, IFCOUNTER_COLLISIONS, c); sc->mibdata.dot3StatsSQETestErrors += RD4(sc, ETH_SQEE); sc->mibdata.dot3StatsDeferredTransmissions += RD4(sc, ETH_DTE); c = RD4(sc, ETH_LCOL); sc->mibdata.dot3StatsLateCollisions += c; if_inc_counter(ifp, IFCOUNTER_COLLISIONS, c); c = RD4(sc, ETH_ECOL); sc->mibdata.dot3StatsExcessiveCollisions += c; if_inc_counter(ifp, IFCOUNTER_COLLISIONS, c); sc->mibdata.dot3StatsCarrierSenseErrors += RD4(sc, ETH_CSE); sc->mibdata.dot3StatsFrameTooLongs += RD4(sc, ETH_ELR); sc->mibdata.dot3StatsInternalMacReceiveErrors += RD4(sc, ETH_DRFC); /* * Not sure where to lump these, so count them against the errors * for the interface. */ if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, RD4(sc, ETH_TUE)); if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, RD4(sc, ETH_CDE) + RD4(sc, ETH_RJB) + RD4(sc, ETH_USF)); /* Schedule another timeout one second from now. */ callout_reset(&sc->tick_ch, hz, ate_tick, sc); } static void ate_set_mac(struct ate_softc *sc, u_char *eaddr) { WR4(sc, ETH_SA1L, (eaddr[3] << 24) | (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]); WR4(sc, ETH_SA1H, (eaddr[5] << 8) | (eaddr[4])); } static int ate_get_mac(struct ate_softc *sc, u_char *eaddr) { bus_size_t sa_low_reg[] = { ETH_SA1L, ETH_SA2L, ETH_SA3L, ETH_SA4L }; bus_size_t sa_high_reg[] = { ETH_SA1H, ETH_SA2H, ETH_SA3H, ETH_SA4H }; uint32_t low, high; int i; /* * The boot loader may setup the MAC with an address(es), grab the * first MAC address from the SA[1-4][HL] registers. */ for (i = 0; i < 4; i++) { low = RD4(sc, sa_low_reg[i]); high = RD4(sc, sa_high_reg[i]); if ((low | (high & 0xffff)) != 0) { eaddr[0] = low & 0xff; eaddr[1] = (low >> 8) & 0xff; eaddr[2] = (low >> 16) & 0xff; eaddr[3] = (low >> 24) & 0xff; eaddr[4] = high & 0xff; eaddr[5] = (high >> 8) & 0xff; return (0); } } return (ENXIO); } static void ate_intr(void *xsc) { struct ate_softc *sc = xsc; struct ifnet *ifp = sc->ifp; struct mbuf *mb; eth_rx_desc_t *rxdhead; uint32_t status, reg, idx; int remain, count, done; status = RD4(sc, ETH_ISR); if (status == 0) return; if (status & ETH_ISR_RCOM) { bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_POSTREAD); rxdhead = &sc->rx_descs[sc->rxhead]; while (rxdhead->addr & ETH_CPU_OWNER) { if (!sc->is_emacb) { /* * Simulate SAM9 FIRST/LAST bits for RM9200. * RM9200 EMAC has only on Rx buffer per packet. - * But sometime we are handed a zero lenght packet. + * But sometime we are handed a zero length packet. */ if ((rxdhead->status & ETH_LEN_MASK) == 0) rxdhead->status = 0; /* Mark error */ else rxdhead->status |= ETH_BUF_FIRST | ETH_BUF_LAST; } if ((rxdhead->status & ETH_BUF_FIRST) == 0) { /* Something went wrong during RX so release back to EMAC all buffers of invalid packets. */ rxdhead->status = 0; rxdhead->addr &= ~ETH_CPU_OWNER; sc->rxhead = NEXT_RX_IDX(sc, sc->rxhead); rxdhead = &sc->rx_descs[sc->rxhead]; continue; } /* Find end of packet or start of next */ idx = sc->rxhead; if ((sc->rx_descs[idx].status & ETH_BUF_LAST) == 0) { idx = NEXT_RX_IDX(sc, idx); while ((sc->rx_descs[idx].addr & ETH_CPU_OWNER) && ((sc->rx_descs[idx].status & (ETH_BUF_FIRST|ETH_BUF_LAST))== 0)) idx = NEXT_RX_IDX(sc, idx); } /* Packet NOT yet completely in memory; we are done */ if ((sc->rx_descs[idx].addr & ETH_CPU_OWNER) == 0 || ((sc->rx_descs[idx].status & (ETH_BUF_FIRST|ETH_BUF_LAST))== 0)) break; /* Packets with no end descriptor are invalid. */ if ((sc->rx_descs[idx].status & ETH_BUF_LAST) == 0) { rxdhead->status &= ~ETH_BUF_FIRST; continue; } /* FCS is not coppied into mbuf. */ remain = (sc->rx_descs[idx].status & ETH_LEN_MASK) - 4; /* Get an appropriately sized mbuf. */ mb = m_get2(remain + ETHER_ALIGN, M_NOWAIT, MT_DATA, M_PKTHDR); if (mb == NULL) { if_inc_counter(sc->ifp, IFCOUNTER_IQDROPS, 1); rxdhead->status = 0; continue; } mb->m_data += ETHER_ALIGN; mb->m_pkthdr.rcvif = ifp; WR4(sc, ETH_RSR, RD4(sc, ETH_RSR)); /* Reset status */ /* Now we process the buffers that make up the packet */ do { /* Last buffer may just be 1-4 bytes of FCS so remain - * may be zero for last decriptor. */ + * may be zero for last descriptor. */ if (remain > 0) { /* Make sure we get the current bytes */ bus_dmamap_sync(sc->rx_tag, sc->rx_map[sc->rxhead], BUS_DMASYNC_POSTREAD); count = MIN(remain, sc->rx_buf_size); /* XXX Performance robbing copy. Could - * recieve directly to mbufs if not an + * receive directly to mbufs if not an * RM9200. And even then we could likely * copy just the protocol headers. XXX */ m_append(mb, count, sc->rx_buf[sc->rxhead]); remain -= count; } done = (rxdhead->status & ETH_BUF_LAST) != 0; /* Return the descriptor to the EMAC */ rxdhead->status = 0; rxdhead->addr &= ~ETH_CPU_OWNER; bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_PREWRITE); /* Move on to next descriptor with wrap */ sc->rxhead = NEXT_RX_IDX(sc, sc->rxhead); rxdhead = &sc->rx_descs[sc->rxhead]; } while (!done); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); (*ifp->if_input)(ifp, mb); } } if (status & ETH_ISR_TCOM) { bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_POSTREAD); ATE_LOCK(sc); /* XXX TSR register should be cleared */ if (!sc->is_emacb) { /* Simulate Transmit descriptor table */ /* First packet done */ if (sc->txtail < sc->txhead) sc->tx_descs[sc->txtail].status |= ETHB_TX_USED; /* Second Packet done */ if (sc->txtail + 1 < sc->txhead && RD4(sc, ETH_TSR) & ETH_TSR_IDLE) sc->tx_descs[sc->txtail + 1].status |= ETHB_TX_USED; } while ((sc->tx_descs[sc->txtail].status & ETHB_TX_USED) && sc->sent_mbuf[sc->txtail] != NULL) { bus_dmamap_sync(sc->mtag, sc->tx_map[sc->txtail], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mtag, sc->tx_map[sc->txtail]); m_freem(sc->sent_mbuf[sc->txtail]); sc->tx_descs[sc->txtail].addr = 0; sc->sent_mbuf[sc->txtail] = NULL; if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); sc->txtail = NEXT_TX_IDX(sc, sc->txtail); } /* Flush descriptors to EMAC */ bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_PREWRITE); /* * We're no longer busy, so clear the busy flag and call the * start routine to xmit more packets. */ sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; atestart_locked(sc->ifp); ATE_UNLOCK(sc); } if (status & ETH_ISR_RBNA) { /* Workaround RM9200 Errata #11 */ if (bootverbose) device_printf(sc->dev, "RBNA workaround\n"); reg = RD4(sc, ETH_CTL); WR4(sc, ETH_CTL, reg & ~ETH_CTL_RE); BARRIER(sc, ETH_CTL, 4, BUS_SPACE_BARRIER_WRITE); WR4(sc, ETH_CTL, reg | ETH_CTL_RE); } /* XXX need to work around SAM9260 errata 43.2.4.1: * disable the mac, reset tx buffer, enable mac on TUND */ } /* * Reset and initialize the chip. */ static void ateinit_locked(void *xsc) { struct ate_softc *sc = xsc; struct ifnet *ifp = sc->ifp; struct mii_data *mii; uint8_t eaddr[ETHER_ADDR_LEN]; uint32_t reg; ATE_ASSERT_LOCKED(sc); /* * XXX TODO(3) * we need to turn on the EMAC clock in the pmc. With the * default boot loader, this is already turned on. However, we * need to think about how best to turn it on/off as the interface * is brought up/down, as well as dealing with the mii bus... * * We also need to multiplex the pins correctly (in board_xxx.c). */ /* * There are two different ways that the mii bus is connected * to this chip mii or rmii. */ if (!sc->is_emacb) { /* RM9200 */ reg = RD4(sc, ETH_CFG); if (sc->use_rmii) reg |= ETH_CFG_RMII; else reg &= ~ETH_CFG_RMII; WR4(sc, ETH_CFG, reg); } else { /* SAM9 */ reg = ETHB_UIO_CLKE; reg |= (sc->use_rmii) ? ETHB_UIO_RMII : 0; WR4(sc, ETHB_UIO, reg); } ate_rxfilter(sc); /* * Set the chip MAC address. */ bcopy(IF_LLADDR(ifp), eaddr, ETHER_ADDR_LEN); ate_set_mac(sc, eaddr); /* Make sure we know state of TX queue */ sc->txhead = sc->txtail = 0; if (sc->is_emacb) { /* Write the descriptor queue address. */ WR4(sc, ETHB_TBQP, sc->tx_desc_phys); } /* * Turn on MACs and interrupt processing. */ WR4(sc, ETH_CTL, RD4(sc, ETH_CTL) | ETH_CTL_TE | ETH_CTL_RE); WR4(sc, ETH_IER, ETH_ISR_RCOM | ETH_ISR_TCOM | ETH_ISR_RBNA); /* Enable big packets. */ WR4(sc, ETH_CFG, RD4(sc, ETH_CFG) | ETH_CFG_BIG); /* * Set 'running' flag, and clear output active flag * and attempt to start the output. */ ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; mii = device_get_softc(sc->miibus); mii_pollstat(mii); ate_stat_update(sc, mii->mii_media_active); atestart_locked(ifp); callout_reset(&sc->tick_ch, hz, ate_tick, sc); } /* * Dequeue packets and transmit. */ static void atestart_locked(struct ifnet *ifp) { struct ate_softc *sc = ifp->if_softc; struct mbuf *m, *mdefrag; bus_dma_segment_t segs[1]; int nseg, e; ATE_ASSERT_LOCKED(sc); if (ifp->if_drv_flags & IFF_DRV_OACTIVE) return; while (sc->tx_descs[sc->txhead].status & ETHB_TX_USED) { /* * Check to see if there's room to put another packet into the * xmit queue. The old EMAC version has a ping-pong buffer for * xmit packets. We use OACTIVE to indicate "we can stuff more * into our buffers (clear) or not (set)." */ /* RM9200 has only two hardware entries */ if (!sc->is_emacb && (RD4(sc, ETH_TSR) & ETH_TSR_BNQ) == 0) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; return; } IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) break; e = bus_dmamap_load_mbuf_sg(sc->mtag, sc->tx_map[sc->txhead], m, segs, &nseg, 0); if (e == EFBIG) { mdefrag = m_defrag(m, M_NOWAIT); if (mdefrag == NULL) { IFQ_DRV_PREPEND(&ifp->if_snd, m); return; } m = mdefrag; e = bus_dmamap_load_mbuf_sg(sc->mtag, sc->tx_map[sc->txhead], m, segs, &nseg, 0); } if (e != 0) { m_freem(m); continue; } /* * There's a small race between the loop in ate_intr finishing * and the check above to see if the packet was finished, as well * as when atestart gets called via other paths. Lose the race * gracefully and free the mbuf... */ if (sc->sent_mbuf[sc->txhead] != NULL) { bus_dmamap_sync(sc->mtag, sc->tx_map[sc->txtail], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mtag, sc->tx_map[sc->txtail]); m_free(sc->sent_mbuf[sc->txhead]); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); } sc->sent_mbuf[sc->txhead] = m; bus_dmamap_sync(sc->mtag, sc->tx_map[sc->txhead], BUS_DMASYNC_PREWRITE); /* Tell the hardware to xmit the packet. */ if (!sc->is_emacb) { WR4(sc, ETH_TAR, segs[0].ds_addr); BARRIER(sc, ETH_TAR, 4, BUS_SPACE_BARRIER_WRITE); WR4(sc, ETH_TCR, segs[0].ds_len); } else { bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_POSTWRITE); sc->tx_descs[sc->txhead].addr = segs[0].ds_addr; sc->tx_descs[sc->txhead].status = segs[0].ds_len | (sc->tx_descs[sc->txhead].status & ETHB_TX_WRAP) | ETHB_TX_BUF_LAST; bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_PREWRITE); WR4(sc, ETH_CTL, RD4(sc, ETH_CTL) | ETHB_CTL_TGO); } sc->txhead = NEXT_TX_IDX(sc, sc->txhead); /* Tap off here if there is a bpf listener. */ BPF_MTAP(ifp, m); } if ((sc->tx_descs[sc->txhead].status & ETHB_TX_USED) == 0) ifp->if_drv_flags |= IFF_DRV_OACTIVE; } static void ateinit(void *xsc) { struct ate_softc *sc = xsc; ATE_LOCK(sc); ateinit_locked(sc); ATE_UNLOCK(sc); } static void atestart(struct ifnet *ifp) { struct ate_softc *sc = ifp->if_softc; ATE_LOCK(sc); atestart_locked(ifp); ATE_UNLOCK(sc); } /* * Turn off interrupts, and stop the NIC. Can be called with sc->ifp NULL, * so be careful. */ static void atestop(struct ate_softc *sc) { struct ifnet *ifp; int i; ATE_ASSERT_LOCKED(sc); ifp = sc->ifp; if (ifp) { //ifp->if_timer = 0; ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); } callout_stop(&sc->tick_ch); /* * Enable some parts of the MAC that are needed always (like the * MII bus. This turns off the RE and TE bits, which will remain * off until ateinit() is called to turn them on. With RE and TE * turned off, there's no DMA to worry about after this write. */ WR4(sc, ETH_CTL, ETH_CTL_MPE); /* * Turn off all the configured options and revert to defaults. */ /* Make sure thate the MDIO clk is less than * 2.5 Mhz. Can no longer default to /32 since * SAM9 family may have MCK > 80 Mhz */ if (at91_master_clock <= 2000000) WR4(sc, ETH_CFG, ETH_CFG_CLK_8); else if (at91_master_clock <= 4000000) WR4(sc, ETH_CFG, ETH_CFG_CLK_16); else if (at91_master_clock <= 800000) WR4(sc, ETH_CFG, ETH_CFG_CLK_32); else WR4(sc, ETH_CFG, ETH_CFG_CLK_64); /* * Turn off all the interrupts, and ack any pending ones by reading * the ISR. */ WR4(sc, ETH_IDR, 0xffffffff); RD4(sc, ETH_ISR); /* * Clear out the Transmit and Receiver Status registers of any * errors they may be reporting */ WR4(sc, ETH_TSR, 0xffffffff); WR4(sc, ETH_RSR, 0xffffffff); /* Release TX resources. */ for (i = 0; i < ATE_MAX_TX_BUFFERS; i++) { if (sc->sent_mbuf[i] != NULL) { bus_dmamap_sync(sc->mtag, sc->tx_map[i], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mtag, sc->tx_map[i]); m_freem(sc->sent_mbuf[i]); sc->sent_mbuf[i] = NULL; } } /* Turn off transeiver input clock */ if (sc->is_emacb) WR4(sc, ETHB_UIO, RD4(sc, ETHB_UIO) & ~ETHB_UIO_CLKE); /* * XXX we should power down the EMAC if it isn't in use, after * putting it into loopback mode. This saves about 400uA according * to the datasheet. */ } static void ate_rxfilter(struct ate_softc *sc) { struct ifnet *ifp; uint32_t reg; int enabled; KASSERT(sc != NULL, ("[ate, %d]: sc is NULL!", __LINE__)); ATE_ASSERT_LOCKED(sc); ifp = sc->ifp; /* Wipe out old filter settings. */ reg = RD4(sc, ETH_CFG); reg &= ~(ETH_CFG_CAF | ETH_CFG_MTI | ETH_CFG_UNI); reg |= ETH_CFG_NBC; sc->flags &= ~ATE_FLAG_MULTICAST; /* Set new parameters. */ if ((ifp->if_flags & IFF_BROADCAST) != 0) reg &= ~ETH_CFG_NBC; if ((ifp->if_flags & IFF_PROMISC) != 0) { reg |= ETH_CFG_CAF; } else { enabled = ate_setmcast(sc); if (enabled != 0) { reg |= ETH_CFG_MTI; sc->flags |= ATE_FLAG_MULTICAST; } } WR4(sc, ETH_CFG, reg); } static int ateioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct ate_softc *sc = ifp->if_softc; struct mii_data *mii; struct ifreq *ifr = (struct ifreq *)data; int drv_flags, flags; int mask, error, enabled; error = 0; flags = ifp->if_flags; drv_flags = ifp->if_drv_flags; switch (cmd) { case SIOCSIFFLAGS: ATE_LOCK(sc); if ((flags & IFF_UP) != 0) { if ((drv_flags & IFF_DRV_RUNNING) != 0) { if (((flags ^ sc->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) != 0) ate_rxfilter(sc); } else { if ((sc->flags & ATE_FLAG_DETACHING) == 0) ateinit_locked(sc); } } else if ((drv_flags & IFF_DRV_RUNNING) != 0) { ifp->if_drv_flags &= ~IFF_DRV_RUNNING; atestop(sc); } sc->if_flags = flags; ATE_UNLOCK(sc); break; case SIOCADDMULTI: case SIOCDELMULTI: if ((drv_flags & IFF_DRV_RUNNING) != 0) { ATE_LOCK(sc); enabled = ate_setmcast(sc); if (enabled != (sc->flags & ATE_FLAG_MULTICAST)) ate_rxfilter(sc); ATE_UNLOCK(sc); } break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: mii = device_get_softc(sc->miibus); error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); break; case SIOCSIFCAP: mask = ifp->if_capenable ^ ifr->ifr_reqcap; if (mask & IFCAP_VLAN_MTU) { ATE_LOCK(sc); if (ifr->ifr_reqcap & IFCAP_VLAN_MTU) { WR4(sc, ETH_CFG, RD4(sc, ETH_CFG) | ETH_CFG_BIG); ifp->if_capenable |= IFCAP_VLAN_MTU; } else { WR4(sc, ETH_CFG, RD4(sc, ETH_CFG) & ~ETH_CFG_BIG); ifp->if_capenable &= ~IFCAP_VLAN_MTU; } ATE_UNLOCK(sc); } default: error = ether_ioctl(ifp, cmd, data); break; } return (error); } static void ate_child_detached(device_t dev, device_t child) { struct ate_softc *sc; sc = device_get_softc(dev); if (child == sc->miibus) sc->miibus = NULL; } /* * MII bus support routines. */ static int ate_miibus_readreg(device_t dev, int phy, int reg) { struct ate_softc *sc; int val; /* - * XXX if we implement agressive power savings, then we need + * XXX if we implement aggressive power savings, then we need * XXX to make sure that the clock to the emac is on here */ sc = device_get_softc(dev); DELAY(1); /* Hangs w/o this delay really 30.5us atm */ WR4(sc, ETH_MAN, ETH_MAN_REG_RD(phy, reg)); while ((RD4(sc, ETH_SR) & ETH_SR_IDLE) == 0) continue; val = RD4(sc, ETH_MAN) & ETH_MAN_VALUE_MASK; return (val); } static int ate_miibus_writereg(device_t dev, int phy, int reg, int data) { struct ate_softc *sc; /* - * XXX if we implement agressive power savings, then we need + * XXX if we implement aggressive power savings, then we need * XXX to make sure that the clock to the emac is on here */ sc = device_get_softc(dev); WR4(sc, ETH_MAN, ETH_MAN_REG_WR(phy, reg, data)); while ((RD4(sc, ETH_SR) & ETH_SR_IDLE) == 0) continue; return (0); } static device_method_t ate_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ate_probe), DEVMETHOD(device_attach, ate_attach), DEVMETHOD(device_detach, ate_detach), /* Bus interface */ DEVMETHOD(bus_child_detached, ate_child_detached), /* MII interface */ DEVMETHOD(miibus_readreg, ate_miibus_readreg), DEVMETHOD(miibus_writereg, ate_miibus_writereg), DEVMETHOD_END }; static driver_t ate_driver = { "ate", ate_methods, sizeof(struct ate_softc), }; #ifdef FDT DRIVER_MODULE(ate, simplebus, ate_driver, ate_devclass, NULL, NULL); #else DRIVER_MODULE(ate, atmelarm, ate_driver, ate_devclass, NULL, NULL); #endif DRIVER_MODULE(miibus, ate, miibus_driver, miibus_devclass, NULL, NULL); MODULE_DEPEND(ate, miibus, 1, 1, 1); MODULE_DEPEND(ate, ether, 1, 1, 1); Index: head/sys/arm/at91/if_atereg.h =================================================================== --- head/sys/arm/at91/if_atereg.h (revision 299068) +++ head/sys/arm/at91/if_atereg.h (revision 299069) @@ -1,214 +1,214 @@ /*- * Copyright (c) 2006 M. Warner Losh. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* $FreeBSD$ */ #ifndef ARM_AT91_IF_ATEREG_H #define ARM_AT91_IF_ATEREG_H -/* deines begining ETHB_ are EMACB (newer SAM9 hardware) versions only */ +/* Defines beginning ETHB_ are EMACB (newer SAM9 hardware) versions only. */ #define ETH_CTL 0x00 /* EMAC Control Register */ #define ETH_CFG 0x04 /* EMAC Configuration Register */ #define ETH_SR 0x08 /* EMAC STatus Register */ #define ETH_TAR 0x0c /* EMAC Transmit Address Register */ #define ETH_TCR 0x10 /* EMAC Transmit Control Register */ #define ETH_TSR 0x14 /* EMAC Transmit Status Register */ #define ETH_RBQP 0x18 /* EMAC Receive Buffer Queue Pointer */ #define ETHB_TBQP 0x1c /* reserved */ #define ETH_RSR 0x20 /* EMAC Receive Status Register */ #define ETH_ISR 0x24 /* EMAC Interrupt Status Register */ #define ETH_IER 0x28 /* EMAC Interrupt Enable Register */ #define ETH_IDR 0x2c /* EMAC Interrupt Disable Register */ #define ETH_IMR 0x30 /* EMAC Interrupt Mask Register */ #define ETH_MAN 0x34 /* EMAC PHY Maintenance Register */ /* 0x38 reserved */ /* 0x3c reserved */ #define ETH_FRA 0x40 /* Frames Transmitted OK Register */ #define ETH_SCOL 0x44 /* Single Collision Frame Register */ #define ETH_MCOL 0x48 /* Multiple Collision Frame Register */ #define ETH_OK 0x4c /* Frames Received OK Register */ #define ETH_SEQE 0x50 /* Frame Check Sequence Error Reg */ #define ETH_ALE 0x54 /* Alignment Error Register */ #define ETH_DTE 0x58 /* Deferred Transmittion Frame Reg */ #define ETH_LCOL 0x5c /* Late Collision Register */ #define ETH_ECOL 0x60 /* Excessive Collision Register */ #define ETH_TUE 0x64 /* Transmit Underrun Error Register */ #define ETH_CSE 0x68 /* Carrier Sense Error Register */ #define ETH_DRFC 0x6c /* Discarded RX Frame Register */ #define ETH_ROV 0x68 /* Receive Overrun Register */ #define ETH_CDE 0x64 /* Code Error Register */ #define ETH_ELR 0x78 /* Excessive Length Error Register */ #define ETH_RJB 0x7c /* Receive Jabber Register */ #define ETH_USF 0x80 /* Undersize Frame Register */ #define ETH_SQEE 0x84 /* SQE Test Error Register */ /* 0x88 reserved */ /* 0x8c reserved */ #define ETH_HSL 0x90 /* EMAC Hash Address Low [31:0] */ #define ETH_HSH 0x94 /* EMAC Hash Address High [63:32] */ #define ETH_SA1L 0x98 /* EMAC Specific Address 1 Low */ #define ETH_SA1H 0x9c /* EMAC Specific Address 1 High */ #define ETH_SA2L 0xa0 /* EMAC Specific Address 2 Low */ #define ETH_SA2H 0xa4 /* EMAC Specific Address 2 High */ #define ETH_SA3L 0xa8 /* EMAC Specific Address 3 Low */ #define ETH_SA3H 0xac /* EMAC Specific Address 3 High */ #define ETH_SA4L 0xb0 /* EMAC Specific Address 4 Low */ #define ETH_SA4H 0xb4 /* EMAC Specific Address 4 High */ #define ETHB_TID 0xb8 /* EMAC Type ID Checking */ #define ETHB_UIO 0xC0 /* EMAC User I/O Reg */ /* ETH_CTL */ #define ETH_CTL_LB (1U << 0) /* LB: Loopback */ #define ETH_CTL_LBL (1U << 1) /* LBL: Loopback Local */ #define ETH_CTL_RE (1U << 2) /* RE: Receive Enable */ #define ETH_CTL_TE (1U << 3) /* TE: Transmit Enable */ #define ETH_CTL_MPE (1U << 4) /* MPE: Management Port Enable */ #define ETH_CTL_CSR (1U << 5) /* CSR: Clear Statistics Registers */ #define ETH_CTL_ISR (1U << 6) /* ISR: Incremenet Statistics Regs */ #define ETH_CTL_WES (1U << 7) /* WES: Write Enable Statistics regs */ #define ETH_CTL_BP (1U << 8) /* BP: Back Pressure */ #define ETHB_CTL_TGO (1U << 9) /* TGO: Transmitter Start */ #define ETHB_CTL_TSTP (1U << 10) /* TSTP: Transmitter Stop */ /* ETH_CFG */ #define ETH_CFG_SPD (1U << 0) /* SPD: Speed 1 == 100: 0 == 10 */ #define ETH_CFG_FD (1U << 1) /* FD: Full duplex */ #define ETH_CFG_BR (1U << 2) /* BR: Bit Rate (optional?) */ /* bit 3 reserved */ #define ETH_CFG_CAF (1U << 4) /* CAF: Copy All Frames */ #define ETH_CFG_NBC (1U << 5) /* NBC: No Broadcast */ #define ETH_CFG_MTI (1U << 6) /* MTI: Multicast Hash Enable */ #define ETH_CFG_UNI (1U << 7) /* UNI: Unicast Hash Enable */ #define ETH_CFG_BIG (1U << 8) /* BIG: Receive 1522 Bytes */ #define ETH_CFG_EAE (1U << 9) /* EAE: External Address Match En */ #define ETH_CFG_CLK_8 (0U << 10) /* CLK: Clock / 8 */ #define ETH_CFG_CLK_16 (1U << 10) /* CLK: Clock / 16 */ #define ETH_CFG_CLK_32 (2U << 10) /* CLK: Clock / 32 */ #define ETH_CFG_CLK_64 (3U << 10) /* CLK: Clock / 64 */ #define ETH_CFG_RTY (1U << 12) /* RTY: Retry Test*/ #define ETH_CFG_RMII (1U << 13) /* RMII: Reduce MII */ #define ETHB_CFG_JBO (1U << 3) /* JBO: Jumbo Frames */ #define ETHB_CFG_PAE (1U << 13) /* PAE: Pause Enable */ #define ETHB_CFG_RBOF_0 (0U << 14) /* RBOF: Rx Buffer Offset */ #define ETHB_CFG_RBOF_1 (1U << 14) /* RBOF: Rx Buffer Offset */ #define ETHB_CFG_RBOF_2 (3U << 14) /* RBOF: Rx Buffer Offset */ #define ETHB_CFG_RBOF_3 (3U << 14) /* RBOF: Rx Buffer Offset */ #define ETHB_CFG_RCLE (1U << 16) /* RCLE: Rx Length Check Enable */ #define ETHB_CFG_DRFC (1U << 17) /* DRFC: Discard Rx FCS */ #define ETHB_CFG_RHD (1U << 18) /* RHD: RX TX'ed frame in half-duplex */ #define ETHB_CFG_IFCS (1U << 19) /* IFCS: Ignore bad RX FCS */ /* ETH_SR */ #define ETH_SR_LINK (1U << 0) /* Reserved! */ #define ETH_SR_MDIO (1U << 1) /* MDIO pin status */ #define ETH_SR_IDLE (1U << 2) /* IDLE (PHY logic) */ /* ETH_TCR */ #define ETH_TCR_NCRC (1U << 15) /* NCRC: No CRC */ /* ETH_TSR */ #define ETH_TSR_OVR (1U << 0) /* OVR: Ethernet Transmit Overrun */ #define ETH_TSR_COL (1U << 1) /* COL: Collision Occurred */ #define ETH_TSR_RLE (1U << 2) /* RLE: Retry Limit Exceeded */ #define ETH_TSR_IDLE (1U << 3) /* IDLE: Transmitter Idle */ #define ETH_TSR_BNQ (1U << 4) /* BNQ: Enet Tran Buff not Queued */ #define ETH_TSR_COMP (1U << 5) /* COMP: Transmit Complete */ #define ETH_TSR_UND (1U << 6) /* UND: Transmit Underrun */ #define ETH_TSR_WR_MASK (0x67) /* write 1 to clear bits */ /* ETH_RSR */ #define ETH_RSR_BNA (1U << 0) /* BNA: Buffer Not Available */ #define ETH_RSR_REC (1U << 1) /* REC: Frame Received */ #define ETH_RSR_OVR (1U << 2) /* OVR: RX Overrun */ /* ETH_ISR */ #define ETH_ISR_DONE (1U << 0) /* DONE: Management Done */ #define ETH_ISR_RCOM (1U << 1) /* RCOM: Receive Complete */ #define ETH_ISR_RBNA (1U << 2) /* RBNA: Receive Buffer Not Avail */ #define ETH_ISR_TOVR (1U << 3) /* TOVR: Transmit Buffer Overrun */ #define ETH_ISR_TUND (1U << 4) /* TUND: Transmit Buffer Underrun */ #define ETH_ISR_RTRY (1U << 5) /* RTRY: Retry Limit */ #define ETH_ISR_TBRE (1U << 6) /* TBRE: Trasnmit Buffer Reg empty */ #define ETH_ISR_TCOM (1U << 7) /* TCOM: Transmit Complete */ #define ETH_ISR_TIDLE (1U << 8) /* TIDLE: Transmit Idle */ #define ETH_ISR_LINK (1U << 9) /* LINK: Link pin delta (optional) */ #define ETH_ISR_ROVR (1U << 10) /* ROVR: RX Overrun */ #define ETH_ISR_ABT (1U << 11) /* ABT: Abort */ /* ETHB_UIO */ #define ETHB_UIO_RMII (1U << 0) /* RMII: Reduce MII */ #define ETHB_UIO_CLKE (1U << 1) /* CLKE: Clock Enable */ /* ETH_MAN */ #define ETH_MAN_BITS 0x40020000 /* HIGH and CODE bits */ #define ETH_MAN_READ (2U << 28) #define ETH_MAN_WRITE (1U << 28) #define ETH_MAN_PHYA_BIT 23 #define ETH_MAN_REGA_BIT 18 #define ETH_MAN_VALUE_MASK 0xffffU #define ETH_MAN_REG_WR(phy, reg, val) \ (ETH_MAN_BITS | ETH_MAN_WRITE | ((phy) << ETH_MAN_PHYA_BIT) | \ ((reg) << ETH_MAN_REGA_BIT) | ((val) & ETH_MAN_VALUE_MASK)) #define ETH_MAN_REG_RD(phy, reg) \ (ETH_MAN_BITS | ETH_MAN_READ | ((phy) << ETH_MAN_PHYA_BIT) | \ ((reg) << ETH_MAN_REGA_BIT)) typedef struct { uint32_t addr; #define ETH_CPU_OWNER (1U << 0) #define ETH_WRAP_BIT (1U << 1) #define ETH_ADR_MASK ~(EHT_CPU_OWNER | ETH_WRAP_BIT) uint32_t status; #define ETH_LEN_MASK 0x7ff #define ETH_BUF_FIRST (1U << 14) /* Packet matched addr 4 */ #define ETH_BUF_LAST (1U << 15) /* Packet matched addr 4 */ #define ETH_MAC_LOCAL_4 (1U << 23) /* Packet matched addr 4 */ #define ETH_MAC_LOCAL_3 (1U << 24) /* Packet matched addr 3 */ #define ETH_MAC_LOCAL_2 (1U << 25) /* Packet matched addr 2 */ #define ETH_MAC_LOCAL_1 (1U << 26) /* Packet matched addr 1 */ -#define ETH_MAC_UNK (1U << 27) /* Unkown source address RFU */ +#define ETH_MAC_UNK (1U << 27) /* Unknown source address RFU */ #define ETH_MAC_EXT (1U << 28) /* External Address */ #define ETH_MAC_UCAST (1U << 29) /* Unicast hash match */ #define ETH_MAC_MCAST (1U << 30) /* Multicast hash match */ #define ETH_MAC_ONES (1U << 31) /* Global all ones bcast addr */ } eth_rx_desc_t; typedef struct { uint32_t addr; uint32_t status; #define ETHB_TX_LEN_MASK 0x7ff #define ETHB_TX_BUF_LAST (1U << 15) /* Last buffer in packet */ #define ETHB_TX_NOCRC (1U << 16) /* Don't xmit CRC*/ #define ETHB_TX_BUFE (1U << 27) /* Buffers exhausted mid frame */ #define ETHB_TX_TUND (1U << 28) /* Transmit Underrun */ #define ETHB_TX_RTRYE (1U << 29) /* Re-try limit exceeded */ #define ETHB_TX_WRAP (1U << 30) /* Last descritor in list */ #define ETHB_TX_USED (1U << 31) /* Packet Transmitted */ } eth_tx_desc_t; #endif /* ARM_AT91_IF_ATEREG_H */ Index: head/sys/arm/broadcom/bcm2835/bcm2835_audio.c =================================================================== --- head/sys/arm/broadcom/bcm2835/bcm2835_audio.c (revision 299068) +++ head/sys/arm/broadcom/bcm2835/bcm2835_audio.c (revision 299069) @@ -1,895 +1,895 @@ /*- * Copyright (c) 2015 Oleksandr Tymoshenko * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_snd.h" #endif #include #include #include "mixer_if.h" #include "interface/compat/vchi_bsd.h" #include "interface/vchi/vchi.h" #include "interface/vchiq_arm/vchiq.h" #include "vc_vchi_audioserv_defs.h" SND_DECLARE_FILE("$FreeBSD$"); #define DEST_AUTO 0 #define DEST_HEADPHONES 1 #define DEST_HDMI 2 #define VCHIQ_AUDIO_PACKET_SIZE 4000 #define VCHIQ_AUDIO_BUFFER_SIZE 128000 #define VCHIQ_AUDIO_PREBUFFER 10 /* Number of pre-buffered audio messages */ #define VCHIQ_AUDIO_MAX_VOLUME /* volume in terms of 0.01dB */ #define VCHIQ_AUDIO_VOLUME_MIN -10239 #define VCHIQ_AUDIO_VOLUME(db100) (uint32_t)(-((db100) << 8)/100) /* dB levels with 5% volume step */ static int db_levels[] = { VCHIQ_AUDIO_VOLUME_MIN, -4605, -3794, -3218, -2772, -2407, -2099, -1832, -1597, -1386, -1195, -1021, -861, -713, -575, -446, -325, -210, -102, 0, }; static uint32_t bcm2835_audio_playfmt[] = { SND_FORMAT(AFMT_U8, 1, 0), SND_FORMAT(AFMT_U8, 2, 0), SND_FORMAT(AFMT_S8, 1, 0), SND_FORMAT(AFMT_S8, 2, 0), SND_FORMAT(AFMT_S16_LE, 1, 0), SND_FORMAT(AFMT_S16_LE, 2, 0), SND_FORMAT(AFMT_U16_LE, 1, 0), SND_FORMAT(AFMT_U16_LE, 2, 0), 0 }; static struct pcmchan_caps bcm2835_audio_playcaps = {8000, 48000, bcm2835_audio_playfmt, 0}; struct bcm2835_audio_info; #define PLAYBACK_IDLE 0 #define PLAYBACK_STARTING 1 #define PLAYBACK_PLAYING 2 #define PLAYBACK_STOPPING 3 struct bcm2835_audio_chinfo { struct bcm2835_audio_info *parent; struct pcm_channel *channel; struct snd_dbuf *buffer; uint32_t fmt, spd, blksz; uint32_t complete_pos; uint32_t free_buffer; uint32_t buffered_ptr; int playback_state; int prebuffered; }; struct bcm2835_audio_info { device_t dev; unsigned int bufsz; struct bcm2835_audio_chinfo pch; uint32_t dest, volume; struct mtx *lock; struct intr_config_hook intr_hook; /* VCHI data */ struct mtx vchi_lock; VCHI_INSTANCE_T vchi_instance; VCHI_CONNECTION_T *vchi_connection; VCHI_SERVICE_HANDLE_T vchi_handle; struct mtx data_lock; struct cv data_cv; /* Unloadign module */ int unloading; }; #define bcm2835_audio_lock(_ess) snd_mtxlock((_ess)->lock) #define bcm2835_audio_unlock(_ess) snd_mtxunlock((_ess)->lock) #define bcm2835_audio_lock_assert(_ess) snd_mtxassert((_ess)->lock) #define VCHIQ_VCHI_LOCK(sc) mtx_lock(&(sc)->vchi_lock) #define VCHIQ_VCHI_UNLOCK(sc) mtx_unlock(&(sc)->vchi_lock) static const char * dest_description(uint32_t dest) { switch (dest) { case DEST_AUTO: return "AUTO"; break; case DEST_HEADPHONES: return "HEADPHONES"; break; case DEST_HDMI: return "HDMI"; break; default: return "UNKNOWN"; break; } } static void bcm2835_audio_callback(void *param, const VCHI_CALLBACK_REASON_T reason, void *msg_handle) { struct bcm2835_audio_info *sc = (struct bcm2835_audio_info *)param; int32_t status; uint32_t msg_len; VC_AUDIO_MSG_T m; if (reason != VCHI_CALLBACK_MSG_AVAILABLE) return; status = vchi_msg_dequeue(sc->vchi_handle, &m, sizeof m, &msg_len, VCHI_FLAGS_NONE); if (m.type == VC_AUDIO_MSG_TYPE_RESULT) { if (m.u.result.success) { device_printf(sc->dev, "msg type %08x failed\n", m.type); } } else if (m.type == VC_AUDIO_MSG_TYPE_COMPLETE) { struct bcm2835_audio_chinfo *ch = m.u.complete.cookie; int count = m.u.complete.count & 0xffff; int perr = (m.u.complete.count & (1U << 30)) != 0; ch->complete_pos = (ch->complete_pos + count) % sndbuf_getsize(ch->buffer); ch->free_buffer += count; chn_intr(sc->pch.channel); if (perr || ch->free_buffer >= VCHIQ_AUDIO_PACKET_SIZE) cv_signal(&sc->data_cv); } else printf("%s: unknown m.type: %d\n", __func__, m.type); } /* VCHIQ stuff */ static void bcm2835_audio_init(struct bcm2835_audio_info *sc) { int status; /* Initialize and create a VCHI connection */ status = vchi_initialise(&sc->vchi_instance); if (status != 0) { printf("vchi_initialise failed: %d\n", status); return; } status = vchi_connect(NULL, 0, sc->vchi_instance); if (status != 0) { printf("vchi_connect failed: %d\n", status); return; } SERVICE_CREATION_T params = { VCHI_VERSION_EX(VC_AUDIOSERV_VER, VC_AUDIOSERV_MIN_VER), VC_AUDIO_SERVER_NAME, /* 4cc service code */ sc->vchi_connection, /* passed in fn pointers */ 0, /* rx fifo size */ 0, /* tx fifo size */ bcm2835_audio_callback, /* service callback */ sc, /* service callback parameter */ 1, 1, 0 /* want crc check on bulk transfers */ }; status = vchi_service_open(sc->vchi_instance, ¶ms, &sc->vchi_handle); if (status == 0) /* Finished with the service for now */ vchi_service_release(sc->vchi_handle); else sc->vchi_handle = VCHIQ_SERVICE_HANDLE_INVALID; } static void bcm2835_audio_release(struct bcm2835_audio_info *sc) { int success; if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); success = vchi_service_close(sc->vchi_handle); if (success != 0) printf("vchi_service_close failed: %d\n", success); sc->vchi_handle = VCHIQ_SERVICE_HANDLE_INVALID; } vchi_disconnect(sc->vchi_instance); } static void bcm2835_audio_reset_channel(struct bcm2835_audio_chinfo *ch) { ch->free_buffer = VCHIQ_AUDIO_BUFFER_SIZE; ch->playback_state = 0; ch->buffered_ptr = 0; ch->complete_pos = 0; ch->prebuffered = 0; sndbuf_reset(ch->buffer); } static void bcm2835_audio_start(struct bcm2835_audio_chinfo *ch) { VC_AUDIO_MSG_T m; int ret; struct bcm2835_audio_info *sc = ch->parent; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); bcm2835_audio_reset_channel(ch); m.type = VC_AUDIO_MSG_TYPE_START; ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_stop(struct bcm2835_audio_chinfo *ch) { VC_AUDIO_MSG_T m; int ret; struct bcm2835_audio_info *sc = ch->parent; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); m.type = VC_AUDIO_MSG_TYPE_STOP; m.u.stop.draining = 0; ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_open(struct bcm2835_audio_info *sc) { VC_AUDIO_MSG_T m; int ret; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); m.type = VC_AUDIO_MSG_TYPE_OPEN; ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_update_controls(struct bcm2835_audio_info *sc) { VC_AUDIO_MSG_T m; int ret, db; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); m.type = VC_AUDIO_MSG_TYPE_CONTROL; m.u.control.dest = sc->dest; if (sc->volume > 99) sc->volume = 99; db = db_levels[sc->volume/5]; m.u.control.volume = VCHIQ_AUDIO_VOLUME(db); ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_update_params(struct bcm2835_audio_info *sc, struct bcm2835_audio_chinfo *ch) { VC_AUDIO_MSG_T m; int ret; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); m.type = VC_AUDIO_MSG_TYPE_CONFIG; m.u.config.channels = AFMT_CHANNEL(ch->fmt); m.u.config.samplerate = ch->spd; m.u.config.bps = AFMT_BIT(ch->fmt); ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static __inline uint32_t vchiq_unbuffered_bytes(struct bcm2835_audio_chinfo *ch) { uint32_t size, ready, readyptr, readyend; size = sndbuf_getsize(ch->buffer); readyptr = sndbuf_getreadyptr(ch->buffer); ready = sndbuf_getready(ch->buffer); readyend = readyptr + ready; /* Normal case */ if (ch->buffered_ptr >= readyptr) { if (readyend > ch->buffered_ptr) return readyend - ch->buffered_ptr; else return 0; } else { /* buffered_ptr overflow */ if (readyend > ch->buffered_ptr + size) return readyend - ch->buffered_ptr - size; else return 0; } } static void bcm2835_audio_write_samples(struct bcm2835_audio_chinfo *ch) { struct bcm2835_audio_info *sc = ch->parent; VC_AUDIO_MSG_T m; void *buf; uint32_t count, size; int ret; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle == VCHIQ_SERVICE_HANDLE_INVALID) { VCHIQ_VCHI_UNLOCK(sc); return; } vchi_service_use(sc->vchi_handle); size = sndbuf_getsize(ch->buffer); count = vchiq_unbuffered_bytes(ch); buf = (uint8_t*)sndbuf_getbuf(ch->buffer) + ch->buffered_ptr; if (ch->buffered_ptr + count > size) count = size - ch->buffered_ptr; if (count < VCHIQ_AUDIO_PACKET_SIZE) goto done; count = min(count, ch->free_buffer); count -= count % VCHIQ_AUDIO_PACKET_SIZE; m.type = VC_AUDIO_MSG_TYPE_WRITE; m.u.write.count = count; m.u.write.max_packet = VCHIQ_AUDIO_PACKET_SIZE; m.u.write.callback = NULL; m.u.write.cookie = ch; if (buf) m.u.write.silence = 0; else m.u.write.silence = 1; ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); if (buf) { while (count > 0) { int bytes = MIN((int)m.u.write.max_packet, (int)count); ch->free_buffer -= bytes; ch->buffered_ptr += bytes; ch->buffered_ptr = ch->buffered_ptr % size; ret = vchi_msg_queue(sc->vchi_handle, buf, bytes, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed: %d\n", __func__, ret); buf = (char *)buf + bytes; count -= bytes; } } done: vchi_service_release(sc->vchi_handle); VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_worker(void *data) { struct bcm2835_audio_info *sc = (struct bcm2835_audio_info *)data; struct bcm2835_audio_chinfo *ch = &sc->pch; mtx_lock(&sc->data_lock); while(1) { if (sc->unloading) break; if (ch->playback_state == PLAYBACK_IDLE) { cv_wait_sig(&sc->data_cv, &sc->data_lock); continue; } if (ch->playback_state == PLAYBACK_STOPPING) { bcm2835_audio_reset_channel(&sc->pch); ch->playback_state = PLAYBACK_IDLE; continue; } if (ch->free_buffer < vchiq_unbuffered_bytes(ch)) { cv_timedwait_sig(&sc->data_cv, &sc->data_lock, 10); continue; } bcm2835_audio_write_samples(ch); if (ch->playback_state == PLAYBACK_STARTING) { ch->prebuffered++; if (ch->prebuffered == VCHIQ_AUDIO_PREBUFFER) { bcm2835_audio_start(ch); ch->playback_state = PLAYBACK_PLAYING; } } } mtx_unlock(&sc->data_lock); kproc_exit(0); } static void bcm2835_audio_create_worker(struct bcm2835_audio_info *sc) { struct proc *newp; if (kproc_create(bcm2835_audio_worker, (void*)sc, &newp, 0, 0, "bcm2835_audio_worker") != 0) { printf("failed to create bcm2835_audio_worker\n"); } } /* -------------------------------------------------------------------- */ /* channel interface for VCHI audio */ static void * bcmchan_init(kobj_t obj, void *devinfo, struct snd_dbuf *b, struct pcm_channel *c, int dir) { struct bcm2835_audio_info *sc = devinfo; struct bcm2835_audio_chinfo *ch = &sc->pch; void *buffer; if (dir == PCMDIR_REC) return NULL; ch->parent = sc; ch->channel = c; ch->buffer = b; /* default values */ ch->spd = 44100; ch->fmt = SND_FORMAT(AFMT_S16_LE, 2, 0); ch->blksz = VCHIQ_AUDIO_PACKET_SIZE; buffer = malloc(sc->bufsz, M_DEVBUF, M_WAITOK | M_ZERO); if (sndbuf_setup(ch->buffer, buffer, sc->bufsz) != 0) { free(buffer, M_DEVBUF); return NULL; } bcm2835_audio_update_params(sc, ch); return ch; } static int bcmchan_free(kobj_t obj, void *data) { struct bcm2835_audio_chinfo *ch = data; void *buffer; buffer = sndbuf_getbuf(ch->buffer); if (buffer) free(buffer, M_DEVBUF); return (0); } static int bcmchan_setformat(kobj_t obj, void *data, uint32_t format) { struct bcm2835_audio_chinfo *ch = data; struct bcm2835_audio_info *sc = ch->parent; bcm2835_audio_lock(sc); ch->fmt = format; bcm2835_audio_update_params(sc, ch); bcm2835_audio_unlock(sc); return 0; } static uint32_t bcmchan_setspeed(kobj_t obj, void *data, uint32_t speed) { struct bcm2835_audio_chinfo *ch = data; struct bcm2835_audio_info *sc = ch->parent; bcm2835_audio_lock(sc); ch->spd = speed; bcm2835_audio_update_params(sc, ch); bcm2835_audio_unlock(sc); return ch->spd; } static uint32_t bcmchan_setblocksize(kobj_t obj, void *data, uint32_t blocksize) { struct bcm2835_audio_chinfo *ch = data; return ch->blksz; } static int bcmchan_trigger(kobj_t obj, void *data, int go) { struct bcm2835_audio_chinfo *ch = data; struct bcm2835_audio_info *sc = ch->parent; if (!PCMTRIG_COMMON(go)) return (0); bcm2835_audio_lock(sc); switch (go) { case PCMTRIG_START: ch->playback_state = PLAYBACK_STARTING; /* wakeup worker thread */ cv_signal(&sc->data_cv); break; case PCMTRIG_STOP: case PCMTRIG_ABORT: ch->playback_state = PLAYBACK_STOPPING; bcm2835_audio_stop(ch); break; default: break; } bcm2835_audio_unlock(sc); return 0; } static uint32_t bcmchan_getptr(kobj_t obj, void *data) { struct bcm2835_audio_chinfo *ch = data; struct bcm2835_audio_info *sc = ch->parent; uint32_t ret; bcm2835_audio_lock(sc); ret = ch->complete_pos - (ch->complete_pos % VCHIQ_AUDIO_PACKET_SIZE); bcm2835_audio_unlock(sc); return ret; } static struct pcmchan_caps * bcmchan_getcaps(kobj_t obj, void *data) { return &bcm2835_audio_playcaps; } static kobj_method_t bcmchan_methods[] = { KOBJMETHOD(channel_init, bcmchan_init), KOBJMETHOD(channel_free, bcmchan_free), KOBJMETHOD(channel_setformat, bcmchan_setformat), KOBJMETHOD(channel_setspeed, bcmchan_setspeed), KOBJMETHOD(channel_setblocksize, bcmchan_setblocksize), KOBJMETHOD(channel_trigger, bcmchan_trigger), KOBJMETHOD(channel_getptr, bcmchan_getptr), KOBJMETHOD(channel_getcaps, bcmchan_getcaps), KOBJMETHOD_END }; CHANNEL_DECLARE(bcmchan); /************************************************************/ static int bcmmix_init(struct snd_mixer *m) { mix_setdevs(m, SOUND_MASK_VOLUME); return (0); } static int bcmmix_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct bcm2835_audio_info *sc = mix_getdevinfo(m); switch (dev) { case SOUND_MIXER_VOLUME: sc->volume = left; bcm2835_audio_update_controls(sc); break; default: break; } return left | (left << 8); } static kobj_method_t bcmmixer_methods[] = { KOBJMETHOD(mixer_init, bcmmix_init), KOBJMETHOD(mixer_set, bcmmix_set), KOBJMETHOD_END }; MIXER_DECLARE(bcmmixer); static int sysctl_bcm2835_audio_dest(SYSCTL_HANDLER_ARGS) { struct bcm2835_audio_info *sc = arg1; int val; int err; val = sc->dest; err = sysctl_handle_int(oidp, &val, 0, req); if (err || !req->newptr) /* error || read request */ return (err); if ((val < 0) || (val > 2)) return (EINVAL); sc->dest = val; device_printf(sc->dev, "destination set to %s\n", dest_description(val)); bcm2835_audio_update_controls(sc); return (0); } static void vchi_audio_sysctl_init(struct bcm2835_audio_info *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid *tree_node; struct sysctl_oid_list *tree; /* * Add system sysctl tree/handlers. */ ctx = device_get_sysctl_ctx(sc->dev); tree_node = device_get_sysctl_tree(sc->dev); tree = SYSCTL_CHILDREN(tree_node); SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "dest", CTLFLAG_RW | CTLTYPE_UINT, sc, sizeof(*sc), sysctl_bcm2835_audio_dest, "IU", "audio destination, " "0 - auto, 1 - headphones, 2 - HDMI"); } static void bcm2835_audio_identify(driver_t *driver, device_t parent) { BUS_ADD_CHILD(parent, 0, "pcm", 0); } static int bcm2835_audio_probe(device_t dev) { device_set_desc(dev, "VCHIQ audio"); return (BUS_PROBE_DEFAULT); } static void bcm2835_audio_delayed_init(void *xsc) { struct bcm2835_audio_info *sc; char status[SND_STATUSLEN]; sc = xsc; config_intrhook_disestablish(&sc->intr_hook); bcm2835_audio_init(sc); bcm2835_audio_open(sc); sc->volume = 75; sc->dest = DEST_AUTO; if (mixer_init(sc->dev, &bcmmixer_class, sc)) { device_printf(sc->dev, "mixer_init failed\n"); goto no; } if (pcm_register(sc->dev, sc, 1, 1)) { device_printf(sc->dev, "pcm_register failed\n"); goto no; } pcm_addchan(sc->dev, PCMDIR_PLAY, &bcmchan_class, sc); snprintf(status, SND_STATUSLEN, "at VCHIQ"); pcm_setstatus(sc->dev, status); bcm2835_audio_reset_channel(&sc->pch); bcm2835_audio_create_worker(sc); vchi_audio_sysctl_init(sc); no: ; } static int bcm2835_audio_attach(device_t dev) { struct bcm2835_audio_info *sc; sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK | M_ZERO); sc->dev = dev; sc->bufsz = VCHIQ_AUDIO_BUFFER_SIZE; sc->lock = snd_mtxcreate(device_get_nameunit(dev), "bcm2835_audio softc"); mtx_init(&sc->vchi_lock, "bcm2835_audio", "vchi_lock", MTX_DEF); mtx_init(&sc->data_lock, "data_mtx", "data_mtx", MTX_DEF); cv_init(&sc->data_cv, "data_cv"); sc->vchi_handle = VCHIQ_SERVICE_HANDLE_INVALID; /* * We need interrupts enabled for VCHI to work properly, - * so delay intialization until it happens + * so delay initialization until it happens. */ sc->intr_hook.ich_func = bcm2835_audio_delayed_init; sc->intr_hook.ich_arg = sc; if (config_intrhook_establish(&sc->intr_hook) != 0) goto no; return 0; no: return ENXIO; } static int bcm2835_audio_detach(device_t dev) { int r; struct bcm2835_audio_info *sc; sc = pcm_getdevinfo(dev); /* Stop worker thread */ sc->unloading = 1; cv_signal(&sc->data_cv); r = pcm_unregister(dev); if (r) return r; mtx_destroy(&sc->vchi_lock); mtx_destroy(&sc->data_lock); cv_destroy(&sc->data_cv); bcm2835_audio_release(sc); if (sc->lock) { snd_mtxfree(sc->lock); sc->lock = NULL; } free(sc, M_DEVBUF); return 0; } static device_method_t bcm2835_audio_methods[] = { /* Device interface */ DEVMETHOD(device_identify, bcm2835_audio_identify), DEVMETHOD(device_probe, bcm2835_audio_probe), DEVMETHOD(device_attach, bcm2835_audio_attach), DEVMETHOD(device_detach, bcm2835_audio_detach), { 0, 0 } }; static driver_t bcm2835_audio_driver = { "pcm", bcm2835_audio_methods, PCM_SOFTC_SIZE, }; DRIVER_MODULE(bcm2835_audio, vchiq, bcm2835_audio_driver, pcm_devclass, 0, 0); MODULE_DEPEND(bcm2835_audio, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER); MODULE_DEPEND(bcm2835_audio, vchiq, 1, 1, 1); MODULE_VERSION(bcm2835_audio, 1); Index: head/sys/arm/broadcom/bcm2835/bcm2835_dma.c =================================================================== --- head/sys/arm/broadcom/bcm2835/bcm2835_dma.c (revision 299068) +++ head/sys/arm/broadcom/bcm2835/bcm2835_dma.c (revision 299069) @@ -1,766 +1,766 @@ /* * Copyright (c) 2013 Daisuke Aoyama * Copyright (c) 2013 Oleksandr Tymoshenko * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bcm2835_dma.h" #include "bcm2835_vcbus.h" #define MAX_REG 9 /* private flags */ #define BCM_DMA_CH_USED 0x00000001 #define BCM_DMA_CH_FREE 0x40000000 #define BCM_DMA_CH_UNMAP 0x80000000 /* Register Map (4.2.1.2) */ #define BCM_DMA_CS(n) (0x100*(n) + 0x00) #define CS_ACTIVE (1 << 0) #define CS_END (1 << 1) #define CS_INT (1 << 2) #define CS_DREQ (1 << 3) #define CS_ISPAUSED (1 << 4) #define CS_ISHELD (1 << 5) #define CS_ISWAIT (1 << 6) #define CS_ERR (1 << 8) #define CS_WAITWRT (1 << 28) #define CS_DISDBG (1 << 29) #define CS_ABORT (1 << 30) #define CS_RESET (1U << 31) #define BCM_DMA_CBADDR(n) (0x100*(n) + 0x04) #define BCM_DMA_INFO(n) (0x100*(n) + 0x08) #define INFO_INT_EN (1 << 0) #define INFO_TDMODE (1 << 1) #define INFO_WAIT_RESP (1 << 3) #define INFO_D_INC (1 << 4) #define INFO_D_WIDTH (1 << 5) #define INFO_D_DREQ (1 << 6) #define INFO_S_INC (1 << 8) #define INFO_S_WIDTH (1 << 9) #define INFO_S_DREQ (1 << 10) #define INFO_WAITS_SHIFT (21) #define INFO_PERMAP_SHIFT (16) #define INFO_PERMAP_MASK (0x1f << INFO_PERMAP_SHIFT) #define BCM_DMA_SRC(n) (0x100*(n) + 0x0C) #define BCM_DMA_DST(n) (0x100*(n) + 0x10) #define BCM_DMA_LEN(n) (0x100*(n) + 0x14) #define BCM_DMA_STRIDE(n) (0x100*(n) + 0x18) #define BCM_DMA_CBNEXT(n) (0x100*(n) + 0x1C) #define BCM_DMA_DEBUG(n) (0x100*(n) + 0x20) #define DEBUG_ERROR_MASK (7) #define BCM_DMA_INT_STATUS 0xfe0 #define BCM_DMA_ENABLE 0xff0 /* relative offset from BCM_VC_DMA0_BASE (p.39) */ #define BCM_DMA_CH(n) (0x100*(n)) /* channels used by GPU */ #define BCM_DMA_CH_BULK 0 #define BCM_DMA_CH_FAST1 2 #define BCM_DMA_CH_FAST2 3 #define BCM_DMA_CH_GPU_MASK ((1 << BCM_DMA_CH_BULK) | \ (1 << BCM_DMA_CH_FAST1) | \ (1 << BCM_DMA_CH_FAST2)) /* DMA Control Block - 256bit aligned (p.40) */ struct bcm_dma_cb { uint32_t info; /* Transfer Information */ uint32_t src; /* Source Address */ uint32_t dst; /* Destination Address */ uint32_t len; /* Transfer Length */ uint32_t stride; /* 2D Mode Stride */ uint32_t next; /* Next Control Block Address */ uint32_t rsvd1; /* Reserved */ uint32_t rsvd2; /* Reserved */ }; #ifdef DEBUG static void bcm_dma_cb_dump(struct bcm_dma_cb *cb); static void bcm_dma_reg_dump(int ch); #endif /* DMA channel private info */ struct bcm_dma_ch { int ch; uint32_t flags; struct bcm_dma_cb * cb; uint32_t vc_cb; bus_dmamap_t dma_map; void (*intr_func)(int, void *); void * intr_arg; }; struct bcm_dma_softc { device_t sc_dev; struct mtx sc_mtx; struct resource * sc_mem; struct resource * sc_irq[BCM_DMA_CH_MAX]; void * sc_intrhand[BCM_DMA_CH_MAX]; struct bcm_dma_ch sc_dma_ch[BCM_DMA_CH_MAX]; bus_dma_tag_t sc_dma_tag; }; static struct bcm_dma_softc *bcm_dma_sc = NULL; static uint32_t bcm_dma_channel_mask; static void bcm_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) { bus_addr_t *addr; if (err) return; addr = (bus_addr_t*)arg; *addr = PHYS_TO_VCBUS(segs[0].ds_addr); } static void bcm_dma_reset(device_t dev, int ch) { struct bcm_dma_softc *sc = device_get_softc(dev); struct bcm_dma_cb *cb; uint32_t cs; int count; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return; cs = bus_read_4(sc->sc_mem, BCM_DMA_CS(ch)); if (cs & CS_ACTIVE) { /* pause current task */ bus_write_4(sc->sc_mem, BCM_DMA_CS(ch), 0); count = 1000; do { cs = bus_read_4(sc->sc_mem, BCM_DMA_CS(ch)); } while (!(cs & CS_ISPAUSED) && (count-- > 0)); if (!(cs & CS_ISPAUSED)) { device_printf(dev, "Can't abort DMA transfer at channel %d\n", ch); } bus_write_4(sc->sc_mem, BCM_DMA_CBNEXT(ch), 0); /* Complete everything, clear interrupt */ bus_write_4(sc->sc_mem, BCM_DMA_CS(ch), CS_ABORT | CS_INT | CS_END| CS_ACTIVE); } /* clear control blocks */ bus_write_4(sc->sc_mem, BCM_DMA_CBADDR(ch), 0); bus_write_4(sc->sc_mem, BCM_DMA_CBNEXT(ch), 0); /* Reset control block */ cb = sc->sc_dma_ch[ch].cb; bzero(cb, sizeof(*cb)); cb->info = INFO_WAIT_RESP; } static int bcm_dma_init(device_t dev) { struct bcm_dma_softc *sc = device_get_softc(dev); uint32_t reg; struct bcm_dma_ch *ch; void *cb_virt; vm_paddr_t cb_phys; int err; int i; /* * Only channels set in bcm_dma_channel_mask can be controlled by us. * The others are out of our control as well as the corresponding bits * in both BCM_DMA_ENABLE and BCM_DMA_INT_STATUS global registers. As * these registers are RW ones, there is no safe way how to write only * the bits which can be controlled by us. * * Fortunately, after reset, all channels are enabled in BCM_DMA_ENABLE * register and all statuses are cleared in BCM_DMA_INT_STATUS one. * Not touching these registers is a trade off between correct * initialization which does not count on anything and not messing up * something we have no control over. */ reg = bus_read_4(sc->sc_mem, BCM_DMA_ENABLE); if ((reg & bcm_dma_channel_mask) != bcm_dma_channel_mask) device_printf(dev, "channels are not enabled\n"); reg = bus_read_4(sc->sc_mem, BCM_DMA_INT_STATUS); if ((reg & bcm_dma_channel_mask) != 0) device_printf(dev, "statuses are not cleared\n"); /* Allocate DMA chunks control blocks */ /* p.40 of spec - control block should be 32-bit aligned */ err = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, sizeof(struct bcm_dma_cb), 1, sizeof(struct bcm_dma_cb), BUS_DMA_ALLOCNOW, NULL, NULL, &sc->sc_dma_tag); if (err) { device_printf(dev, "failed allocate DMA tag\n"); return (err); } /* setup initial settings */ for (i = 0; i < BCM_DMA_CH_MAX; i++) { ch = &sc->sc_dma_ch[i]; bzero(ch, sizeof(struct bcm_dma_ch)); ch->ch = i; ch->flags = BCM_DMA_CH_UNMAP; if ((bcm_dma_channel_mask & (1 << i)) == 0) continue; err = bus_dmamem_alloc(sc->sc_dma_tag, &cb_virt, BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, &ch->dma_map); if (err) { device_printf(dev, "cannot allocate DMA memory\n"); break; } /* * Least alignment for busdma-allocated stuff is cache - * line size, so just make sure nothing stupid happend + * line size, so just make sure nothing stupid happened * and we got properly aligned address */ if ((uintptr_t)cb_virt & 0x1f) { device_printf(dev, "DMA address is not 32-bytes aligned: %p\n", (void*)cb_virt); break; } err = bus_dmamap_load(sc->sc_dma_tag, ch->dma_map, cb_virt, sizeof(struct bcm_dma_cb), bcm_dmamap_cb, &cb_phys, BUS_DMA_WAITOK); if (err) { device_printf(dev, "cannot load DMA memory\n"); break; } ch->cb = cb_virt; ch->vc_cb = cb_phys; ch->flags = BCM_DMA_CH_FREE; ch->cb->info = INFO_WAIT_RESP; /* reset DMA engine */ bus_write_4(sc->sc_mem, BCM_DMA_CS(i), CS_RESET); } return (0); } /* * Allocate DMA channel for further use, returns channel # or * BCM_DMA_CH_INVALID */ int bcm_dma_allocate(int req_ch) { struct bcm_dma_softc *sc = bcm_dma_sc; int ch = BCM_DMA_CH_INVALID; int i; if (req_ch >= BCM_DMA_CH_MAX) return (BCM_DMA_CH_INVALID); /* Auto(req_ch < 0) or CH specified */ mtx_lock(&sc->sc_mtx); if (req_ch < 0) { for (i = 0; i < BCM_DMA_CH_MAX; i++) { if (sc->sc_dma_ch[i].flags & BCM_DMA_CH_FREE) { ch = i; sc->sc_dma_ch[ch].flags &= ~BCM_DMA_CH_FREE; sc->sc_dma_ch[ch].flags |= BCM_DMA_CH_USED; break; } } } else { if (sc->sc_dma_ch[req_ch].flags & BCM_DMA_CH_FREE) { ch = req_ch; sc->sc_dma_ch[ch].flags &= ~BCM_DMA_CH_FREE; sc->sc_dma_ch[ch].flags |= BCM_DMA_CH_USED; } } mtx_unlock(&sc->sc_mtx); return (ch); } /* * Frees allocated channel. Returns 0 on success, -1 otherwise */ int bcm_dma_free(int ch) { struct bcm_dma_softc *sc = bcm_dma_sc; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); mtx_lock(&sc->sc_mtx); if (sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED) { sc->sc_dma_ch[ch].flags |= BCM_DMA_CH_FREE; sc->sc_dma_ch[ch].flags &= ~BCM_DMA_CH_USED; sc->sc_dma_ch[ch].intr_func = NULL; sc->sc_dma_ch[ch].intr_arg = NULL; /* reset DMA engine */ bcm_dma_reset(sc->sc_dev, ch); } mtx_unlock(&sc->sc_mtx); return (0); } /* * Assign handler function for channel interrupt * Returns 0 on success, -1 otherwise */ int bcm_dma_setup_intr(int ch, void (*func)(int, void *), void *arg) { struct bcm_dma_softc *sc = bcm_dma_sc; struct bcm_dma_cb *cb; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (-1); sc->sc_dma_ch[ch].intr_func = func; sc->sc_dma_ch[ch].intr_arg = arg; cb = sc->sc_dma_ch[ch].cb; cb->info |= INFO_INT_EN; return (0); } /* * Setup DMA source parameters * ch - channel number * dreq - hardware DREQ # or BCM_DMA_DREQ_NONE if * source is physical memory * inc_addr - BCM_DMA_INC_ADDR if source address * should be increased after each access or * BCM_DMA_SAME_ADDR if address should remain * the same * width - size of read operation, BCM_DMA_32BIT * for 32bit bursts, BCM_DMA_128BIT for 128 bits * * Returns 0 on success, -1 otherwise */ int bcm_dma_setup_src(int ch, int dreq, int inc_addr, int width) { struct bcm_dma_softc *sc = bcm_dma_sc; uint32_t info; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (-1); info = sc->sc_dma_ch[ch].cb->info; info &= ~INFO_PERMAP_MASK; info |= (dreq << INFO_PERMAP_SHIFT) & INFO_PERMAP_MASK; if (dreq) info |= INFO_S_DREQ; else info &= ~INFO_S_DREQ; if (width == BCM_DMA_128BIT) info |= INFO_S_WIDTH; else info &= ~INFO_S_WIDTH; if (inc_addr == BCM_DMA_INC_ADDR) info |= INFO_S_INC; else info &= ~INFO_S_INC; sc->sc_dma_ch[ch].cb->info = info; return (0); } /* * Setup DMA destination parameters * ch - channel number * dreq - hardware DREQ # or BCM_DMA_DREQ_NONE if * destination is physical memory * inc_addr - BCM_DMA_INC_ADDR if source address * should be increased after each access or * BCM_DMA_SAME_ADDR if address should remain * the same * width - size of write operation, BCM_DMA_32BIT * for 32bit bursts, BCM_DMA_128BIT for 128 bits * * Returns 0 on success, -1 otherwise */ int bcm_dma_setup_dst(int ch, int dreq, int inc_addr, int width) { struct bcm_dma_softc *sc = bcm_dma_sc; uint32_t info; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (-1); info = sc->sc_dma_ch[ch].cb->info; info &= ~INFO_PERMAP_MASK; info |= (dreq << INFO_PERMAP_SHIFT) & INFO_PERMAP_MASK; if (dreq) info |= INFO_D_DREQ; else info &= ~INFO_D_DREQ; if (width == BCM_DMA_128BIT) info |= INFO_D_WIDTH; else info &= ~INFO_D_WIDTH; if (inc_addr == BCM_DMA_INC_ADDR) info |= INFO_D_INC; else info &= ~INFO_D_INC; sc->sc_dma_ch[ch].cb->info = info; return (0); } #ifdef DEBUG void bcm_dma_cb_dump(struct bcm_dma_cb *cb) { printf("DMA CB "); printf("INFO: %8.8x ", cb->info); printf("SRC: %8.8x ", cb->src); printf("DST: %8.8x ", cb->dst); printf("LEN: %8.8x ", cb->len); printf("\n"); printf("STRIDE: %8.8x ", cb->stride); printf("NEXT: %8.8x ", cb->next); printf("RSVD1: %8.8x ", cb->rsvd1); printf("RSVD2: %8.8x ", cb->rsvd2); printf("\n"); } void bcm_dma_reg_dump(int ch) { struct bcm_dma_softc *sc = bcm_dma_sc; int i; uint32_t reg; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return; printf("DMA%d: ", ch); for (i = 0; i < MAX_REG; i++) { reg = bus_read_4(sc->sc_mem, BCM_DMA_CH(ch) + i*4); printf("%8.8x ", reg); } printf("\n"); } #endif /* * Start DMA transaction * ch - channel number * src, dst - source and destination address in * ARM physical memory address space. - * len - amount of bytes to be transfered + * len - amount of bytes to be transferred * * Returns 0 on success, -1 otherwise */ int bcm_dma_start(int ch, vm_paddr_t src, vm_paddr_t dst, int len) { struct bcm_dma_softc *sc = bcm_dma_sc; struct bcm_dma_cb *cb; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (-1); cb = sc->sc_dma_ch[ch].cb; if (BCM2835_ARM_IS_IO(src)) cb->src = IO_TO_VCBUS(src); else cb->src = PHYS_TO_VCBUS(src); if (BCM2835_ARM_IS_IO(dst)) cb->dst = IO_TO_VCBUS(dst); else cb->dst = PHYS_TO_VCBUS(dst); cb->len = len; bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_ch[ch].dma_map, BUS_DMASYNC_PREWRITE); bus_write_4(sc->sc_mem, BCM_DMA_CBADDR(ch), sc->sc_dma_ch[ch].vc_cb); bus_write_4(sc->sc_mem, BCM_DMA_CS(ch), CS_ACTIVE); #ifdef DEBUG bcm_dma_cb_dump(sc->sc_dma_ch[ch].cb); bcm_dma_reg_dump(ch); #endif return (0); } /* * Get length requested for DMA transaction * ch - channel number * * Returns size of transaction, 0 if channel is invalid */ uint32_t bcm_dma_length(int ch) { struct bcm_dma_softc *sc = bcm_dma_sc; struct bcm_dma_cb *cb; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (0); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (0); cb = sc->sc_dma_ch[ch].cb; return (cb->len); } static void bcm_dma_intr(void *arg) { struct bcm_dma_softc *sc = bcm_dma_sc; struct bcm_dma_ch *ch = (struct bcm_dma_ch *)arg; uint32_t cs, debug; /* my interrupt? */ cs = bus_read_4(sc->sc_mem, BCM_DMA_CS(ch->ch)); if (!(cs & (CS_INT | CS_ERR))) { device_printf(sc->sc_dev, "unexpected DMA intr CH=%d, CS=%x\n", ch->ch, cs); return; } /* running? */ if (!(ch->flags & BCM_DMA_CH_USED)) { device_printf(sc->sc_dev, "unused DMA intr CH=%d, CS=%x\n", ch->ch, cs); return; } if (cs & CS_ERR) { debug = bus_read_4(sc->sc_mem, BCM_DMA_DEBUG(ch->ch)); device_printf(sc->sc_dev, "DMA error %d on CH%d\n", debug & DEBUG_ERROR_MASK, ch->ch); bus_write_4(sc->sc_mem, BCM_DMA_DEBUG(ch->ch), debug & DEBUG_ERROR_MASK); bcm_dma_reset(sc->sc_dev, ch->ch); } if (cs & CS_INT) { /* acknowledge interrupt */ bus_write_4(sc->sc_mem, BCM_DMA_CS(ch->ch), CS_INT | CS_END); /* Prepare for possible access to len field */ bus_dmamap_sync(sc->sc_dma_tag, ch->dma_map, BUS_DMASYNC_POSTWRITE); /* save callback function and argument */ if (ch->intr_func) ch->intr_func(ch->ch, ch->intr_arg); } } static int bcm_dma_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "broadcom,bcm2835-dma")) return (ENXIO); device_set_desc(dev, "BCM2835 DMA Controller"); return (BUS_PROBE_DEFAULT); } static int bcm_dma_attach(device_t dev) { struct bcm_dma_softc *sc = device_get_softc(dev); phandle_t node; int rid, err = 0; int i; sc->sc_dev = dev; if (bcm_dma_sc) return (ENXIO); for (i = 0; i < BCM_DMA_CH_MAX; i++) { sc->sc_irq[i] = NULL; sc->sc_intrhand[i] = NULL; } /* Get DMA channel mask. */ node = ofw_bus_get_node(sc->sc_dev); if (OF_getencprop(node, "brcm,dma-channel-mask", &bcm_dma_channel_mask, sizeof(bcm_dma_channel_mask)) == -1 && OF_getencprop(node, "broadcom,channels", &bcm_dma_channel_mask, sizeof(bcm_dma_channel_mask)) == -1) { device_printf(dev, "could not get channel mask property\n"); return (ENXIO); } /* Mask out channels used by GPU. */ bcm_dma_channel_mask &= ~BCM_DMA_CH_GPU_MASK; /* DMA0 - DMA14 */ rid = 0; sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem == NULL) { device_printf(dev, "could not allocate memory resource\n"); return (ENXIO); } /* IRQ DMA0 - DMA11 XXX NOT USE DMA12(spurious?) */ for (rid = 0; rid < BCM_DMA_CH_MAX; rid++) { if ((bcm_dma_channel_mask & (1 << rid)) == 0) continue; sc->sc_irq[rid] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->sc_irq[rid] == NULL) { device_printf(dev, "cannot allocate interrupt\n"); err = ENXIO; goto fail; } if (bus_setup_intr(dev, sc->sc_irq[rid], INTR_TYPE_MISC | INTR_MPSAFE, NULL, bcm_dma_intr, &sc->sc_dma_ch[rid], &sc->sc_intrhand[rid])) { device_printf(dev, "cannot setup interrupt handler\n"); err = ENXIO; goto fail; } } mtx_init(&sc->sc_mtx, "bcmdma", "bcmdma", MTX_DEF); bcm_dma_sc = sc; err = bcm_dma_init(dev); if (err) goto fail; return (err); fail: if (sc->sc_mem) bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem); for (i = 0; i < BCM_DMA_CH_MAX; i++) { if (sc->sc_intrhand[i]) bus_teardown_intr(dev, sc->sc_irq[i], sc->sc_intrhand[i]); if (sc->sc_irq[i]) bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq[i]); } return (err); } static device_method_t bcm_dma_methods[] = { DEVMETHOD(device_probe, bcm_dma_probe), DEVMETHOD(device_attach, bcm_dma_attach), { 0, 0 } }; static driver_t bcm_dma_driver = { "bcm_dma", bcm_dma_methods, sizeof(struct bcm_dma_softc), }; static devclass_t bcm_dma_devclass; DRIVER_MODULE(bcm_dma, simplebus, bcm_dma_driver, bcm_dma_devclass, 0, 0); MODULE_VERSION(bcm_dma, 1); Index: head/sys/arm/cavium/cns11xx/if_ece.c =================================================================== --- head/sys/arm/cavium/cns11xx/if_ece.c (revision 299068) +++ head/sys/arm/cavium/cns11xx/if_ece.c (revision 299069) @@ -1,1955 +1,1955 @@ /*- * Copyright (c) 2009 Yohanes Nugroho * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET #include #include #include #include #endif #include #include #include #include #include #include #include #include #include /* "device miibus" required. See GENERIC if you get errors here. */ #include "miibus_if.h" static uint8_t vlan0_mac[ETHER_ADDR_LEN] = {0x00, 0xaa, 0xbb, 0xcc, 0xdd, 0x19}; /* * Boot loader expects the hardware state to be the same when we * restart the device (warm boot), so we need to save the initial * config values. */ int initial_switch_config; int initial_cpu_config; int initial_port0_config; int initial_port1_config; static inline uint32_t read_4(struct ece_softc *sc, bus_size_t off) { return (bus_read_4(sc->mem_res, off)); } static inline void write_4(struct ece_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(sc->mem_res, off, val); } #define ECE_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define ECE_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define ECE_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \ MTX_NETWORK_LOCK, MTX_DEF) #define ECE_TXLOCK(_sc) mtx_lock(&(_sc)->sc_mtx_tx) #define ECE_TXUNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx_tx) #define ECE_TXLOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx_tx, device_get_nameunit(_sc->dev), \ "ECE TX Lock", MTX_DEF) #define ECE_CLEANUPLOCK(_sc) mtx_lock(&(_sc)->sc_mtx_cleanup) #define ECE_CLEANUPUNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx_cleanup) #define ECE_CLEANUPLOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx_cleanup, device_get_nameunit(_sc->dev), \ "ECE cleanup Lock", MTX_DEF) #define ECE_RXLOCK(_sc) mtx_lock(&(_sc)->sc_mtx_rx) #define ECE_RXUNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx_rx) #define ECE_RXLOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx_rx, device_get_nameunit(_sc->dev), \ "ECE RX Lock", MTX_DEF) #define ECE_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define ECE_TXLOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx_tx); #define ECE_RXLOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx_rx); #define ECE_CLEANUPLOCK_DESTROY(_sc) \ mtx_destroy(&_sc->sc_mtx_cleanup); #define ECE_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define ECE_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); static devclass_t ece_devclass; /* ifnet entry points */ static void eceinit_locked(void *); static void ecestart_locked(struct ifnet *); static void eceinit(void *); static void ecestart(struct ifnet *); static void ecestop(struct ece_softc *); static int eceioctl(struct ifnet * ifp, u_long, caddr_t); /* bus entry points */ static int ece_probe(device_t dev); static int ece_attach(device_t dev); static int ece_detach(device_t dev); static void ece_intr(void *); static void ece_intr_qf(void *); static void ece_intr_status(void *xsc); /* helper routines */ static int ece_activate(device_t dev); static void ece_deactivate(device_t dev); static int ece_ifmedia_upd(struct ifnet *ifp); static void ece_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr); static int ece_get_mac(struct ece_softc *sc, u_char *eaddr); static void ece_set_mac(struct ece_softc *sc, u_char *eaddr); static int configure_cpu_port(struct ece_softc *sc); static int configure_lan_port(struct ece_softc *sc, int phy_type); static void set_pvid(struct ece_softc *sc, int port0, int port1, int cpu); static void set_vlan_vid(struct ece_softc *sc, int vlan); static void set_vlan_member(struct ece_softc *sc, int vlan); static void set_vlan_tag(struct ece_softc *sc, int vlan); static int hardware_init(struct ece_softc *sc); static void ece_intr_rx_locked(struct ece_softc *sc, int count); static void ece_free_desc_dma_tx(struct ece_softc *sc); static void ece_free_desc_dma_rx(struct ece_softc *sc); static void ece_intr_task(void *arg, int pending __unused); static void ece_tx_task(void *arg, int pending __unused); static void ece_cleanup_task(void *arg, int pending __unused); static int ece_allocate_dma(struct ece_softc *sc); static void ece_intr_tx(void *xsc); static void clear_mac_entries(struct ece_softc *ec, int include_this_mac); static uint32_t read_mac_entry(struct ece_softc *ec, uint8_t *mac_result, int first); /*PHY related functions*/ static inline int phy_read(struct ece_softc *sc, int phy, int reg) { int val; int ii; int status; write_4(sc, PHY_CONTROL, PHY_RW_OK); write_4(sc, PHY_CONTROL, (PHY_ADDRESS(phy)|PHY_READ_COMMAND | PHY_REGISTER(reg))); for (ii = 0; ii < 0x1000; ii++) { status = read_4(sc, PHY_CONTROL); if (status & PHY_RW_OK) { /* Clear the rw_ok status, and clear other * bits value. */ write_4(sc, PHY_CONTROL, PHY_RW_OK); val = PHY_GET_DATA(status); return (val); } } return (0); } static inline void phy_write(struct ece_softc *sc, int phy, int reg, int data) { int ii; write_4(sc, PHY_CONTROL, PHY_RW_OK); write_4(sc, PHY_CONTROL, PHY_ADDRESS(phy) | PHY_REGISTER(reg) | PHY_WRITE_COMMAND | PHY_DATA(data)); for (ii = 0; ii < 0x1000; ii++) { if (read_4(sc, PHY_CONTROL) & PHY_RW_OK) { /* Clear the rw_ok status, and clear other * bits value. */ write_4(sc, PHY_CONTROL, PHY_RW_OK); return; } } } static int get_phy_type(struct ece_softc *sc) { uint16_t phy0_id = 0, phy1_id = 0; /* * Use SMI (MDC/MDIO) to read Link Partner's PHY Identifier * Register 1. */ phy0_id = phy_read(sc, 0, 0x2); phy1_id = phy_read(sc, 1, 0x2); if ((phy0_id == 0xFFFF) && (phy1_id == 0x000F)) return (ASIX_GIGA_PHY); else if ((phy0_id == 0x0243) && (phy1_id == 0x0243)) return (TWO_SINGLE_PHY); else if ((phy0_id == 0xFFFF) && (phy1_id == 0x0007)) return (VSC8601_GIGA_PHY); else if ((phy0_id == 0x0243) && (phy1_id == 0xFFFF)) return (IC_PLUS_PHY); return (NOT_FOUND_PHY); } static int ece_probe(device_t dev) { device_set_desc(dev, "Econa Ethernet Controller"); return (0); } static int ece_attach(device_t dev) { struct ece_softc *sc; struct ifnet *ifp = NULL; struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; u_char eaddr[ETHER_ADDR_LEN]; int err; int i, rid; uint32_t rnd; err = 0; sc = device_get_softc(dev); sc->dev = dev; rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) goto out; power_on_network_interface(); rid = 0; sc->irq_res_status = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res_status == NULL) goto out; rid = 1; /*TSTC: Fm-Switch-Tx-Complete*/ sc->irq_res_tx = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res_tx == NULL) goto out; rid = 2; /*FSRC: Fm-Switch-Rx-Complete*/ sc->irq_res_rec = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res_rec == NULL) goto out; rid = 4; /*FSQF: Fm-Switch-Queue-Full*/ sc->irq_res_qf = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res_qf == NULL) goto out; err = ece_activate(dev); if (err) goto out; /* Sysctls */ sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); ECE_LOCK_INIT(sc); callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0); if ((err = ece_get_mac(sc, eaddr)) != 0) { /* No MAC address configured. Generate the random one. */ if (bootverbose) device_printf(dev, "Generating random ethernet address.\n"); rnd = arc4random(); /*from if_ae.c/if_ate.c*/ /* * Set OUI to convenient locally assigned address. 'b' * is 0x62, which has the locally assigned bit set, and * the broadcast/multicast bit clear. */ eaddr[0] = 'b'; eaddr[1] = 's'; eaddr[2] = 'd'; eaddr[3] = (rnd >> 16) & 0xff; eaddr[4] = (rnd >> 8) & 0xff; eaddr[5] = rnd & 0xff; for (i = 0; i < ETHER_ADDR_LEN; i++) eaddr[i] = vlan0_mac[i]; } ece_set_mac(sc, eaddr); sc->ifp = ifp = if_alloc(IFT_ETHER); /* Only one PHY at address 0 in this device. */ err = mii_attach(dev, &sc->miibus, ifp, ece_ifmedia_upd, ece_ifmedia_sts, BMSR_DEFCAPMASK, 0, MII_OFFSET_ANY, 0); if (err != 0) { device_printf(dev, "attaching PHYs failed\n"); goto out; } ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_capabilities = IFCAP_HWCSUM; ifp->if_hwassist = (CSUM_IP | CSUM_TCP | CSUM_UDP); ifp->if_capenable = ifp->if_capabilities; ifp->if_start = ecestart; ifp->if_ioctl = eceioctl; ifp->if_init = eceinit; ifp->if_snd.ifq_drv_maxlen = ECE_MAX_TX_BUFFERS - 1; IFQ_SET_MAXLEN(&ifp->if_snd, ECE_MAX_TX_BUFFERS - 1); IFQ_SET_READY(&ifp->if_snd); /* Create local taskq. */ TASK_INIT(&sc->sc_intr_task, 0, ece_intr_task, sc); TASK_INIT(&sc->sc_tx_task, 1, ece_tx_task, ifp); TASK_INIT(&sc->sc_cleanup_task, 2, ece_cleanup_task, sc); sc->sc_tq = taskqueue_create_fast("ece_taskq", M_WAITOK, taskqueue_thread_enqueue, &sc->sc_tq); if (sc->sc_tq == NULL) { device_printf(sc->dev, "could not create taskqueue\n"); goto out; } ether_ifattach(ifp, eaddr); /* * Activate interrupts */ err = bus_setup_intr(dev, sc->irq_res_rec, INTR_TYPE_NET | INTR_MPSAFE, NULL, ece_intr, sc, &sc->intrhand); if (err) { ether_ifdetach(ifp); ECE_LOCK_DESTROY(sc); goto out; } err = bus_setup_intr(dev, sc->irq_res_status, INTR_TYPE_NET | INTR_MPSAFE, NULL, ece_intr_status, sc, &sc->intrhand_status); if (err) { ether_ifdetach(ifp); ECE_LOCK_DESTROY(sc); goto out; } err = bus_setup_intr(dev, sc->irq_res_qf, INTR_TYPE_NET | INTR_MPSAFE, NULL,ece_intr_qf, sc, &sc->intrhand_qf); if (err) { ether_ifdetach(ifp); ECE_LOCK_DESTROY(sc); goto out; } err = bus_setup_intr(dev, sc->irq_res_tx, INTR_TYPE_NET | INTR_MPSAFE, NULL, ece_intr_tx, sc, &sc->intrhand_tx); if (err) { ether_ifdetach(ifp); ECE_LOCK_DESTROY(sc); goto out; } ECE_TXLOCK_INIT(sc); ECE_RXLOCK_INIT(sc); ECE_CLEANUPLOCK_INIT(sc); /* Enable all interrupt sources. */ write_4(sc, INTERRUPT_MASK, 0x00000000); /* Enable port 0. */ write_4(sc, PORT_0_CONFIG, read_4(sc, PORT_0_CONFIG) & ~(PORT_DISABLE)); taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s taskq", device_get_nameunit(sc->dev)); out: if (err) ece_deactivate(dev); if (err && ifp) if_free(ifp); return (err); } static int ece_detach(device_t dev) { struct ece_softc *sc = device_get_softc(dev); struct ifnet *ifp = sc->ifp; ecestop(sc); if (ifp != NULL) { ether_ifdetach(ifp); if_free(ifp); } ece_deactivate(dev); return (0); } static void ece_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { u_int32_t *paddr; KASSERT(nsegs == 1, ("wrong number of segments, should be 1")); paddr = arg; *paddr = segs->ds_addr; } static int ece_alloc_desc_dma_tx(struct ece_softc *sc) { int i; int error; /* Allocate a busdma tag and DMA safe memory for TX/RX descriptors. */ error = bus_dma_tag_create(sc->sc_parent_tag, /* parent */ 16, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ sizeof(eth_tx_desc_t)*ECE_MAX_TX_BUFFERS, /* max size */ 1, /*nsegments */ sizeof(eth_tx_desc_t)*ECE_MAX_TX_BUFFERS, 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sc->dmatag_data_tx); /* dmat */ /* Allocate memory for TX ring. */ error = bus_dmamem_alloc(sc->dmatag_data_tx, (void**)&(sc->desc_tx), BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &(sc->dmamap_ring_tx)); if (error) { if_printf(sc->ifp, "failed to allocate DMA memory\n"); bus_dma_tag_destroy(sc->dmatag_data_tx); sc->dmatag_data_tx = 0; return (ENXIO); } /* Load Ring DMA. */ error = bus_dmamap_load(sc->dmatag_data_tx, sc->dmamap_ring_tx, sc->desc_tx, sizeof(eth_tx_desc_t)*ECE_MAX_TX_BUFFERS, ece_getaddr, &(sc->ring_paddr_tx), BUS_DMA_NOWAIT); if (error) { if_printf(sc->ifp, "can't load descriptor\n"); bus_dmamem_free(sc->dmatag_data_tx, sc->desc_tx, sc->dmamap_ring_tx); sc->desc_tx = NULL; bus_dma_tag_destroy(sc->dmatag_data_tx); sc->dmatag_data_tx = 0; return (ENXIO); } /* Allocate a busdma tag for mbufs. Alignment is 2 bytes */ error = bus_dma_tag_create(sc->sc_parent_tag, /* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ MCLBYTES*MAX_FRAGMENT, /* maxsize */ MAX_FRAGMENT, /* nsegments */ MCLBYTES, 0, /* maxsegsz, flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sc->dmatag_ring_tx); /* dmat */ if (error) { if_printf(sc->ifp, "failed to create busdma tag for mbufs\n"); return (ENXIO); } for (i = 0; i < ECE_MAX_TX_BUFFERS; i++) { /* Create dma map for each descriptor. */ error = bus_dmamap_create(sc->dmatag_ring_tx, 0, &(sc->tx_desc[i].dmamap)); if (error) { if_printf(sc->ifp, "failed to create map for mbuf\n"); return (ENXIO); } } return (0); } static void ece_free_desc_dma_tx(struct ece_softc *sc) { int i; for (i = 0; i < ECE_MAX_TX_BUFFERS; i++) { if (sc->tx_desc[i].buff) { m_freem(sc->tx_desc[i].buff); sc->tx_desc[i].buff= 0; } } if (sc->ring_paddr_tx) { bus_dmamap_unload(sc->dmatag_data_tx, sc->dmamap_ring_tx); sc->ring_paddr_tx = 0; } if (sc->desc_tx) { bus_dmamem_free(sc->dmatag_data_tx, sc->desc_tx, sc->dmamap_ring_tx); sc->desc_tx = NULL; } if (sc->dmatag_data_tx) { bus_dma_tag_destroy(sc->dmatag_data_tx); sc->dmatag_data_tx = 0; } if (sc->dmatag_ring_tx) { for (i = 0; idmatag_ring_tx, sc->tx_desc[i].dmamap); sc->tx_desc[i].dmamap = 0; } bus_dma_tag_destroy(sc->dmatag_ring_tx); sc->dmatag_ring_tx = 0; } } static int ece_alloc_desc_dma_rx(struct ece_softc *sc) { int error; int i; /* Allocate a busdma tag and DMA safe memory for RX descriptors. */ error = bus_dma_tag_create(sc->sc_parent_tag, /* parent */ 16, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ /* maxsize, nsegments */ sizeof(eth_rx_desc_t)*ECE_MAX_RX_BUFFERS, 1, /* maxsegsz, flags */ sizeof(eth_rx_desc_t)*ECE_MAX_RX_BUFFERS, 0, NULL, NULL, /* lockfunc, lockfuncarg */ &sc->dmatag_data_rx); /* dmat */ /* Allocate RX ring. */ error = bus_dmamem_alloc(sc->dmatag_data_rx, (void**)&(sc->desc_rx), BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &(sc->dmamap_ring_rx)); if (error) { if_printf(sc->ifp, "failed to allocate DMA memory\n"); return (ENXIO); } /* Load dmamap. */ error = bus_dmamap_load(sc->dmatag_data_rx, sc->dmamap_ring_rx, sc->desc_rx, sizeof(eth_rx_desc_t)*ECE_MAX_RX_BUFFERS, ece_getaddr, &(sc->ring_paddr_rx), BUS_DMA_NOWAIT); if (error) { if_printf(sc->ifp, "can't load descriptor\n"); bus_dmamem_free(sc->dmatag_data_rx, sc->desc_rx, sc->dmamap_ring_rx); bus_dma_tag_destroy(sc->dmatag_data_rx); sc->desc_rx = NULL; return (ENXIO); } /* Allocate a busdma tag for mbufs. */ error = bus_dma_tag_create(sc->sc_parent_tag,/* parent */ 16, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ MCLBYTES, 1, /* maxsize, nsegments */ MCLBYTES, 0, /* maxsegsz, flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sc->dmatag_ring_rx); /* dmat */ if (error) { if_printf(sc->ifp, "failed to create busdma tag for mbufs\n"); return (ENXIO); } for (i = 0; idmatag_ring_rx, 0, &sc->rx_desc[i].dmamap); if (error) { if_printf(sc->ifp, "failed to create map for mbuf\n"); return (ENXIO); } } error = bus_dmamap_create(sc->dmatag_ring_rx, 0, &sc->rx_sparemap); if (error) { if_printf(sc->ifp, "failed to create spare map\n"); return (ENXIO); } return (0); } static void ece_free_desc_dma_rx(struct ece_softc *sc) { int i; for (i = 0; i < ECE_MAX_RX_BUFFERS; i++) { if (sc->rx_desc[i].buff) { m_freem(sc->rx_desc[i].buff); sc->rx_desc[i].buff = NULL; } } if (sc->ring_paddr_rx) { bus_dmamap_unload(sc->dmatag_data_rx, sc->dmamap_ring_rx); sc->ring_paddr_rx = 0; } if (sc->desc_rx) { bus_dmamem_free(sc->dmatag_data_rx, sc->desc_rx, sc->dmamap_ring_rx); sc->desc_rx = NULL; } if (sc->dmatag_data_rx) { bus_dma_tag_destroy(sc->dmatag_data_rx); sc->dmatag_data_rx = NULL; } if (sc->dmatag_ring_rx) { for (i = 0; i < ECE_MAX_RX_BUFFERS; i++) bus_dmamap_destroy(sc->dmatag_ring_rx, sc->rx_desc[i].dmamap); bus_dmamap_destroy(sc->dmatag_ring_rx, sc->rx_sparemap); bus_dma_tag_destroy(sc->dmatag_ring_rx); sc->dmatag_ring_rx = NULL; } } static int ece_new_rxbuf(struct ece_softc *sc, struct rx_desc_info* descinfo) { struct mbuf *new_mbuf; bus_dma_segment_t seg[1]; bus_dmamap_t map; int error; int nsegs; bus_dma_tag_t tag; tag = sc->dmatag_ring_rx; new_mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); if (new_mbuf == NULL) return (ENOBUFS); new_mbuf->m_len = new_mbuf->m_pkthdr.len = MCLBYTES; error = bus_dmamap_load_mbuf_sg(tag, sc->rx_sparemap, new_mbuf, seg, &nsegs, BUS_DMA_NOWAIT); KASSERT(nsegs == 1, ("Too many segments returned!")); if (nsegs != 1 || error) { m_free(new_mbuf); return (ENOBUFS); } if (descinfo->buff != NULL) { bus_dmamap_sync(tag, descinfo->dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(tag, descinfo->dmamap); } map = descinfo->dmamap; descinfo->dmamap = sc->rx_sparemap; sc->rx_sparemap = map; bus_dmamap_sync(tag, descinfo->dmamap, BUS_DMASYNC_PREREAD); descinfo->buff = new_mbuf; descinfo->desc->data_ptr = seg->ds_addr; descinfo->desc->length = seg->ds_len - 2; return (0); } static int ece_allocate_dma(struct ece_softc *sc) { eth_tx_desc_t *desctx; eth_rx_desc_t *descrx; int i; int error; /* Create parent tag for tx and rx */ error = bus_dma_tag_create( bus_get_dma_tag(sc->dev),/* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT, 0,/* maxsize, nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->sc_parent_tag); ece_alloc_desc_dma_tx(sc); for (i = 0; i < ECE_MAX_TX_BUFFERS; i++) { desctx = (eth_tx_desc_t *)(&sc->desc_tx[i]); memset(desctx, 0, sizeof(eth_tx_desc_t)); desctx->length = MAX_PACKET_LEN; desctx->cown = 1; if (i == ECE_MAX_TX_BUFFERS - 1) desctx->eor = 1; } ece_alloc_desc_dma_rx(sc); for (i = 0; i < ECE_MAX_RX_BUFFERS; i++) { descrx = &(sc->desc_rx[i]); memset(descrx, 0, sizeof(eth_rx_desc_t)); sc->rx_desc[i].desc = descrx; sc->rx_desc[i].buff = 0; ece_new_rxbuf(sc, &(sc->rx_desc[i])); if (i == ECE_MAX_RX_BUFFERS - 1) descrx->eor = 1; } sc->tx_prod = 0; sc->tx_cons = 0; sc->last_rx = 0; sc->desc_curr_tx = 0; return (0); } static int ece_activate(device_t dev) { struct ece_softc *sc; int err; uint32_t mac_port_config; struct ifnet *ifp; sc = device_get_softc(dev); ifp = sc->ifp; initial_switch_config = read_4(sc, SWITCH_CONFIG); initial_cpu_config = read_4(sc, CPU_PORT_CONFIG); initial_port0_config = read_4(sc, MAC_PORT_0_CONFIG); initial_port1_config = read_4(sc, MAC_PORT_1_CONFIG); /* Disable Port 0 */ mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); mac_port_config |= (PORT_DISABLE); write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); /* Disable Port 1 */ mac_port_config = read_4(sc, MAC_PORT_1_CONFIG); mac_port_config |= (PORT_DISABLE); write_4(sc, MAC_PORT_1_CONFIG, mac_port_config); err = ece_allocate_dma(sc); if (err) { if_printf(sc->ifp, "failed allocating dma\n"); goto out; } write_4(sc, TS_DESCRIPTOR_POINTER, sc->ring_paddr_tx); write_4(sc, TS_DESCRIPTOR_BASE_ADDR, sc->ring_paddr_tx); write_4(sc, FS_DESCRIPTOR_POINTER, sc->ring_paddr_rx); write_4(sc, FS_DESCRIPTOR_BASE_ADDR, sc->ring_paddr_rx); write_4(sc, FS_DMA_CONTROL, 1); return (0); out: return (ENXIO); } static void ece_deactivate(device_t dev) { struct ece_softc *sc; sc = device_get_softc(dev); if (sc->intrhand) bus_teardown_intr(dev, sc->irq_res_rec, sc->intrhand); sc->intrhand = 0; if (sc->intrhand_qf) bus_teardown_intr(dev, sc->irq_res_qf, sc->intrhand_qf); sc->intrhand_qf = 0; bus_generic_detach(sc->dev); if (sc->miibus) device_delete_child(sc->dev, sc->miibus); if (sc->mem_res) bus_release_resource(dev, SYS_RES_IOPORT, rman_get_rid(sc->mem_res), sc->mem_res); sc->mem_res = 0; if (sc->irq_res_rec) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res_rec), sc->irq_res_rec); if (sc->irq_res_qf) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res_qf), sc->irq_res_qf); if (sc->irq_res_qf) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res_status), sc->irq_res_status); sc->irq_res_rec = 0; sc->irq_res_qf = 0; sc->irq_res_status = 0; ECE_TXLOCK_DESTROY(sc); ECE_RXLOCK_DESTROY(sc); ece_free_desc_dma_tx(sc); ece_free_desc_dma_rx(sc); return; } /* * Change media according to request. */ static int ece_ifmedia_upd(struct ifnet *ifp) { struct ece_softc *sc = ifp->if_softc; struct mii_data *mii; int error; mii = device_get_softc(sc->miibus); ECE_LOCK(sc); error = mii_mediachg(mii); ECE_UNLOCK(sc); return (error); } /* * Notify the world which media we're using. */ static void ece_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) { struct ece_softc *sc = ifp->if_softc; struct mii_data *mii; mii = device_get_softc(sc->miibus); ECE_LOCK(sc); mii_pollstat(mii); ifmr->ifm_active = mii->mii_media_active; ifmr->ifm_status = mii->mii_media_status; ECE_UNLOCK(sc); } static void ece_tick(void *xsc) { struct ece_softc *sc = xsc; struct mii_data *mii; int active; mii = device_get_softc(sc->miibus); active = mii->mii_media_active; mii_tick(mii); /* * Schedule another timeout one second from now. */ callout_reset(&sc->tick_ch, hz, ece_tick, sc); } static uint32_t read_mac_entry(struct ece_softc *ec, uint8_t *mac_result, int first) { uint32_t ii; struct arl_table_entry_t entry; uint32_t *entry_val; write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_1, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_2, 0); if (first) write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, 0x1); else write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, 0x2); for (ii = 0; ii < 0x1000; ii++) if (read_4(ec, ARL_TABLE_ACCESS_CONTROL_1) & (0x1)) break; entry_val = (uint32_t*) (&entry); entry_val[0] = read_4(ec, ARL_TABLE_ACCESS_CONTROL_1); entry_val[1] = read_4(ec, ARL_TABLE_ACCESS_CONTROL_2); if (mac_result) memcpy(mac_result, entry.mac_addr, ETHER_ADDR_LEN); return (entry.table_end); } static uint32_t write_arl_table_entry(struct ece_softc *ec, uint32_t filter, uint32_t vlan_mac, uint32_t vlan_gid, uint32_t age_field, uint32_t port_map, const uint8_t *mac_addr) { uint32_t ii; uint32_t *entry_val; struct arl_table_entry_t entry; memset(&entry, 0, sizeof(entry)); entry.filter = filter; entry.vlan_mac = vlan_mac; entry.vlan_gid = vlan_gid; entry.age_field = age_field; entry.port_map = port_map; memcpy(entry.mac_addr, mac_addr, ETHER_ADDR_LEN); entry_val = (uint32_t*) (&entry); write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_1, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_2, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_1, entry_val[0]); write_4(ec, ARL_TABLE_ACCESS_CONTROL_2, entry_val[1]); write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, ARL_WRITE_COMMAND); for (ii = 0; ii < 0x1000; ii++) if (read_4(ec, ARL_TABLE_ACCESS_CONTROL_1) & ARL_COMMAND_COMPLETE) return (1); /* Write OK. */ /* Write failed. */ return (0); } static void remove_mac_entry(struct ece_softc *sc, uint8_t *mac) { /* Invalid age_field mean erase this entry. */ write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, INVALID_ENTRY, VLAN0_GROUP, mac); } static void add_mac_entry(struct ece_softc *sc, uint8_t *mac) { write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, NEW_ENTRY, VLAN0_GROUP, mac); } /** * The behavior of ARL table reading and deletion is not well defined * in the documentation. To be safe, all mac addresses are put to a * list, then deleted. * */ static void clear_mac_entries(struct ece_softc *ec, int include_this_mac) { int table_end; struct mac_list * temp; struct mac_list * mac_list_header; struct mac_list * current; char mac[ETHER_ADDR_LEN]; current = NULL; mac_list_header = NULL; table_end = read_mac_entry(ec, mac, 1); while (!table_end) { if (!include_this_mac && memcmp(mac, vlan0_mac, ETHER_ADDR_LEN) == 0) { /* Read next entry. */ table_end = read_mac_entry(ec, mac, 0); continue; } temp = (struct mac_list*)malloc(sizeof(struct mac_list), M_DEVBUF, M_NOWAIT | M_ZERO); memcpy(temp->mac_addr, mac, ETHER_ADDR_LEN); temp->next = 0; if (mac_list_header) { current->next = temp; current = temp; } else { mac_list_header = temp; current = temp; } /* Read next Entry */ table_end = read_mac_entry(ec, mac, 0); } current = mac_list_header; while (current) { remove_mac_entry(ec, current->mac_addr); temp = current; current = current->next; free(temp, M_DEVBUF); } } static int configure_lan_port(struct ece_softc *sc, int phy_type) { uint32_t sw_config; uint32_t mac_port_config; /* * Configure switch */ sw_config = read_4(sc, SWITCH_CONFIG); /* Enable fast aging. */ sw_config |= FAST_AGING; /* Enable IVL learning. */ sw_config |= IVL_LEARNING; /* Disable hardware NAT. */ sw_config &= ~(HARDWARE_NAT); sw_config |= SKIP_L2_LOOKUP_PORT_0 | SKIP_L2_LOOKUP_PORT_1| NIC_MODE; write_4(sc, SWITCH_CONFIG, sw_config); sw_config = read_4(sc, SWITCH_CONFIG); mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); if (!(mac_port_config & 0x1) || (mac_port_config & 0x2)) if_printf(sc->ifp, "Link Down\n"); else write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); return (0); } static void set_pvid(struct ece_softc *sc, int port0, int port1, int cpu) { uint32_t val; val = read_4(sc, VLAN_PORT_PVID) & (~(0x7 << 0)); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) | ((port0) & 0x07); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) & (~(0x7 << 4)); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) | (((port1) & 0x07) << 4); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) & (~(0x7 << 8)); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) | (((cpu) & 0x07) << 8); write_4(sc, VLAN_PORT_PVID, val); } /* VLAN related functions */ static void set_vlan_vid(struct ece_softc *sc, int vlan) { const uint32_t regs[] = { VLAN_VID_0_1, VLAN_VID_0_1, VLAN_VID_2_3, VLAN_VID_2_3, VLAN_VID_4_5, VLAN_VID_4_5, VLAN_VID_6_7, VLAN_VID_6_7 }; const int vids[] = { VLAN0_VID, VLAN1_VID, VLAN2_VID, VLAN3_VID, VLAN4_VID, VLAN5_VID, VLAN6_VID, VLAN7_VID }; uint32_t val; uint32_t reg; int vid; reg = regs[vlan]; vid = vids[vlan]; if (vlan & 1) { val = read_4(sc, reg); write_4(sc, reg, val & (~(0xFFF << 0))); val = read_4(sc, reg); write_4(sc, reg, val|((vid & 0xFFF) << 0)); } else { val = read_4(sc, reg); write_4(sc, reg, val & (~(0xFFF << 12))); val = read_4(sc, reg); write_4(sc, reg, val|((vid & 0xFFF) << 12)); } } static void set_vlan_member(struct ece_softc *sc, int vlan) { unsigned char shift; uint32_t val; int group; const int groups[] = { VLAN0_GROUP, VLAN1_GROUP, VLAN2_GROUP, VLAN3_GROUP, VLAN4_GROUP, VLAN5_GROUP, VLAN6_GROUP, VLAN7_GROUP }; group = groups[vlan]; shift = vlan*3; val = read_4(sc, VLAN_MEMBER_PORT_MAP) & (~(0x7 << shift)); write_4(sc, VLAN_MEMBER_PORT_MAP, val); val = read_4(sc, VLAN_MEMBER_PORT_MAP); write_4(sc, VLAN_MEMBER_PORT_MAP, val | ((group & 0x7) << shift)); } static void set_vlan_tag(struct ece_softc *sc, int vlan) { unsigned char shift; uint32_t val; int tag = 0; shift = vlan*3; val = read_4(sc, VLAN_TAG_PORT_MAP) & (~(0x7 << shift)); write_4(sc, VLAN_TAG_PORT_MAP, val); val = read_4(sc, VLAN_TAG_PORT_MAP); write_4(sc, VLAN_TAG_PORT_MAP, val | ((tag & 0x7) << shift)); } static int configure_cpu_port(struct ece_softc *sc) { uint32_t cpu_port_config; int i; cpu_port_config = read_4(sc, CPU_PORT_CONFIG); /* SA learning Disable */ cpu_port_config |= (SA_LEARNING_DISABLE); /* set data offset + 2 */ cpu_port_config &= ~(1U << 31); write_4(sc, CPU_PORT_CONFIG, cpu_port_config); if (!write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, STATIC_ENTRY, VLAN0_GROUP, vlan0_mac)) return (1); set_pvid(sc, PORT0_PVID, PORT1_PVID, CPU_PORT_PVID); for (i = 0; i < 8; i++) { set_vlan_vid(sc, i); set_vlan_member(sc, i); set_vlan_tag(sc, i); } /* disable all interrupt status sources */ write_4(sc, INTERRUPT_MASK, 0xffff1fff); /* clear previous interrupt sources */ write_4(sc, INTERRUPT_STATUS, 0x00001FFF); write_4(sc, TS_DMA_CONTROL, 0); write_4(sc, FS_DMA_CONTROL, 0); return (0); } static int hardware_init(struct ece_softc *sc) { int status = 0; static int gw_phy_type; gw_phy_type = get_phy_type(sc); /* Currently only ic_plus phy is supported. */ if (gw_phy_type != IC_PLUS_PHY) { device_printf(sc->dev, "PHY type is not supported (%d)\n", gw_phy_type); return (-1); } status = configure_lan_port(sc, gw_phy_type); configure_cpu_port(sc); return (0); } static void set_mac_address(struct ece_softc *sc, const char *mac, int mac_len) { /* Invalid age_field mean erase this entry. */ write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, INVALID_ENTRY, VLAN0_GROUP, mac); memcpy(vlan0_mac, mac, ETHER_ADDR_LEN); write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, STATIC_ENTRY, VLAN0_GROUP, mac); } static void ece_set_mac(struct ece_softc *sc, u_char *eaddr) { memcpy(vlan0_mac, eaddr, ETHER_ADDR_LEN); set_mac_address(sc, eaddr, ETHER_ADDR_LEN); } /* * TODO: the device doesn't have MAC stored, we should read the * configuration stored in FLASH, but the format depends on the * bootloader used.* */ static int ece_get_mac(struct ece_softc *sc, u_char *eaddr) { return (ENXIO); } static void ece_intr_rx_locked(struct ece_softc *sc, int count) { struct ifnet *ifp = sc->ifp; struct mbuf *mb; struct rx_desc_info *rxdesc; eth_rx_desc_t *desc; int fssd_curr; int fssd; int i; int idx; int rxcount; uint32_t status; fssd_curr = read_4(sc, FS_DESCRIPTOR_POINTER); fssd = (fssd_curr - (uint32_t)sc->ring_paddr_rx)>>4; desc = sc->rx_desc[sc->last_rx].desc; /* Prepare to read the data in the ring. */ bus_dmamap_sync(sc->dmatag_ring_rx, sc->dmamap_ring_rx, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); if (fssd > sc->last_rx) rxcount = fssd - sc->last_rx; else if (fssd < sc->last_rx) rxcount = (ECE_MAX_RX_BUFFERS - sc->last_rx) + fssd; else { if (desc->cown == 0) return; else rxcount = ECE_MAX_RX_BUFFERS; } for (i= 0; i < rxcount; i++) { status = desc->cown; if (!status) break; idx = sc->last_rx; rxdesc = &sc->rx_desc[idx]; mb = rxdesc->buff; if (desc->length < ETHER_MIN_LEN - ETHER_CRC_LEN || desc->length > ETHER_MAX_LEN - ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); desc->cown = 0; desc->length = MCLBYTES - 2; /* Invalid packet, skip and process next * packet. */ continue; } if (ece_new_rxbuf(sc, rxdesc) != 0) { if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); desc->cown = 0; desc->length = MCLBYTES - 2; break; } /** * The device will write to addrress + 2 So we need to adjust * the address after the packet is received. */ mb->m_data += 2; mb->m_len = mb->m_pkthdr.len = desc->length; mb->m_flags |= M_PKTHDR; mb->m_pkthdr.rcvif = ifp; if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { /*check for valid checksum*/ if ( (!desc->l4f) && (desc->prot != 3)) { mb->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; mb->m_pkthdr.csum_flags |= CSUM_IP_VALID; mb->m_pkthdr.csum_data = 0xffff; } } ECE_RXUNLOCK(sc); (*ifp->if_input)(ifp, mb); ECE_RXLOCK(sc); desc->cown = 0; desc->length = MCLBYTES - 2; bus_dmamap_sync(sc->dmatag_ring_rx, sc->dmamap_ring_rx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); if (sc->last_rx == ECE_MAX_RX_BUFFERS - 1) sc->last_rx = 0; else sc->last_rx++; desc = sc->rx_desc[sc->last_rx].desc; } /* Sync updated flags. */ bus_dmamap_sync(sc->dmatag_ring_rx, sc->dmamap_ring_rx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); return; } static void ece_intr_task(void *arg, int pending __unused) { struct ece_softc *sc = arg; ECE_RXLOCK(sc); ece_intr_rx_locked(sc, -1); ECE_RXUNLOCK(sc); } static void ece_intr(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { write_4(sc, FS_DMA_CONTROL, 0); return; } taskqueue_enqueue(sc->sc_tq, &sc->sc_intr_task); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) taskqueue_enqueue(sc->sc_tq, &sc->sc_tx_task); } static void ece_intr_status(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; int stat; stat = read_4(sc, INTERRUPT_STATUS); write_4(sc, INTERRUPT_STATUS, stat); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { if ((stat & ERROR_MASK) != 0) if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); } } static void ece_cleanup_locked(struct ece_softc *sc) { eth_tx_desc_t *desc; if (sc->tx_cons == sc->tx_prod) return; /* Prepare to read the ring (owner bit). */ bus_dmamap_sync(sc->dmatag_ring_tx, sc->dmamap_ring_tx, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); while (sc->tx_cons != sc->tx_prod) { desc = sc->tx_desc[sc->tx_cons].desc; if (desc->cown != 0) { struct tx_desc_info *td = &(sc->tx_desc[sc->tx_cons]); /* We are finished with this descriptor ... */ bus_dmamap_sync(sc->dmatag_data_tx, td->dmamap, BUS_DMASYNC_POSTWRITE); /* ... and unload, so we can reuse. */ bus_dmamap_unload(sc->dmatag_data_tx, td->dmamap); m_freem(td->buff); td->buff = 0; sc->tx_cons = (sc->tx_cons + 1) % ECE_MAX_TX_BUFFERS; } else { break; } } } static void ece_cleanup_task(void *arg, int pending __unused) { struct ece_softc *sc = arg; ECE_CLEANUPLOCK(sc); ece_cleanup_locked(sc); ECE_CLEANUPUNLOCK(sc); } static void ece_intr_tx(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { /* This should not happen, stop DMA. */ write_4(sc, FS_DMA_CONTROL, 0); return; } taskqueue_enqueue(sc->sc_tq, &sc->sc_cleanup_task); } static void ece_intr_qf(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { /* This should not happen, stop DMA. */ write_4(sc, FS_DMA_CONTROL, 0); return; } taskqueue_enqueue(sc->sc_tq, &sc->sc_intr_task); write_4(sc, FS_DMA_CONTROL, 1); } /* * Reset and initialize the chip */ static void eceinit_locked(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; struct mii_data *mii; uint32_t cfg_reg; uint32_t cpu_port_config; uint32_t mac_port_config; while (1) { cfg_reg = read_4(sc, BIST_RESULT_TEST_0); if ((cfg_reg & (1<<17))) break; DELAY(100); } /* Set to default values. */ write_4(sc, SWITCH_CONFIG, 0x007AA7A1); write_4(sc, MAC_PORT_0_CONFIG, 0x00423D00); write_4(sc, MAC_PORT_1_CONFIG, 0x00423D80); write_4(sc, CPU_PORT_CONFIG, 0x004C0000); hardware_init(sc); mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); /* Enable Port 0 */ mac_port_config &= (~(PORT_DISABLE)); write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); cpu_port_config = read_4(sc, CPU_PORT_CONFIG); /* Enable CPU. */ cpu_port_config &= ~(PORT_DISABLE); write_4(sc, CPU_PORT_CONFIG, cpu_port_config); /* * Set 'running' flag, and clear output active flag * and attempt to start the output */ ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; mii = device_get_softc(sc->miibus); mii_pollstat(mii); /* Enable DMA. */ write_4(sc, FS_DMA_CONTROL, 1); callout_reset(&sc->tick_ch, hz, ece_tick, sc); } static inline int ece_encap(struct ece_softc *sc, struct mbuf *m0) { struct ifnet *ifp; bus_dma_segment_t segs[MAX_FRAGMENT]; bus_dmamap_t mapp; eth_tx_desc_t *desc = NULL; int csum_flags; int desc_no; int error; int nsegs; int seg; ifp = sc->ifp; /* Fetch unused map */ mapp = sc->tx_desc[sc->tx_prod].dmamap; error = bus_dmamap_load_mbuf_sg(sc->dmatag_ring_tx, mapp, m0, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { bus_dmamap_unload(sc->dmatag_ring_tx, mapp); return ((error != 0) ? error : -1); } desc = &(sc->desc_tx[sc->desc_curr_tx]); sc->tx_desc[sc->tx_prod].desc = desc; sc->tx_desc[sc->tx_prod].buff = m0; desc_no = sc->desc_curr_tx; for (seg = 0; seg < nsegs; seg++) { if (desc->cown == 0 ) { if_printf(ifp, "ERROR: descriptor is still used\n"); return (-1); } desc->length = segs[seg].ds_len; desc->data_ptr = segs[seg].ds_addr; if (seg == 0) { desc->fs = 1; } else { desc->fs = 0; } if (seg == nsegs - 1) { desc->ls = 1; } else { desc->ls = 0; } csum_flags = m0->m_pkthdr.csum_flags; desc->fr = 1; desc->pmap = 1; desc->insv = 0; desc->ico = 0; desc->tco = 0; desc->uco = 0; desc->interrupt = 1; if (csum_flags & CSUM_IP) { desc->ico = 1; if (csum_flags & CSUM_TCP) desc->tco = 1; if (csum_flags & CSUM_UDP) desc->uco = 1; } desc++; sc->desc_curr_tx = (sc->desc_curr_tx + 1) % ECE_MAX_TX_BUFFERS; if (sc->desc_curr_tx == 0) { desc = (eth_tx_desc_t *)&(sc->desc_tx[0]); } } desc = sc->tx_desc[sc->tx_prod].desc; sc->tx_prod = (sc->tx_prod + 1) % ECE_MAX_TX_BUFFERS; /* * After all descriptors are set, we set the flags to start the - * sending proces. + * sending process. */ for (seg = 0; seg < nsegs; seg++) { desc->cown = 0; desc++; desc_no = (desc_no + 1) % ECE_MAX_TX_BUFFERS; if (desc_no == 0) desc = (eth_tx_desc_t *)&(sc->desc_tx[0]); } bus_dmamap_sync(sc->dmatag_data_tx, mapp, BUS_DMASYNC_PREWRITE); return (0); } /* * dequeu packets and transmit */ static void ecestart_locked(struct ifnet *ifp) { struct ece_softc *sc; struct mbuf *m0; uint32_t queued = 0; sc = ifp->if_softc; if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; bus_dmamap_sync(sc->dmatag_ring_tx, sc->dmamap_ring_tx, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); for (;;) { /* Get packet from the queue */ IF_DEQUEUE(&ifp->if_snd, m0); if (m0 == NULL) break; if (ece_encap(sc, m0)) { IF_PREPEND(&ifp->if_snd, m0); ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } queued++; BPF_MTAP(ifp, m0); } if (queued) { bus_dmamap_sync(sc->dmatag_ring_tx, sc->dmamap_ring_tx, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); write_4(sc, TS_DMA_CONTROL, 1); } } static void eceinit(void *xsc) { struct ece_softc *sc = xsc; ECE_LOCK(sc); eceinit_locked(sc); ECE_UNLOCK(sc); } static void ece_tx_task(void *arg, int pending __unused) { struct ifnet *ifp; ifp = (struct ifnet *)arg; ecestart(ifp); } static void ecestart(struct ifnet *ifp) { struct ece_softc *sc = ifp->if_softc; ECE_TXLOCK(sc); ecestart_locked(ifp); ECE_TXUNLOCK(sc); } /* * Turn off interrupts, and stop the nic. Can be called with sc->ifp * NULL so be careful. */ static void ecestop(struct ece_softc *sc) { struct ifnet *ifp = sc->ifp; uint32_t mac_port_config; write_4(sc, TS_DMA_CONTROL, 0); write_4(sc, FS_DMA_CONTROL, 0); if (ifp) ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); callout_stop(&sc->tick_ch); /*Disable Port 0 */ mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); mac_port_config |= (PORT_DISABLE); write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); /*Disable Port 1 */ mac_port_config = read_4(sc, MAC_PORT_1_CONFIG); mac_port_config |= (PORT_DISABLE); write_4(sc, MAC_PORT_1_CONFIG, mac_port_config); /* Disable all interrupt status sources. */ write_4(sc, INTERRUPT_MASK, 0x00001FFF); /* Clear previous interrupt sources. */ write_4(sc, INTERRUPT_STATUS, 0x00001FFF); write_4(sc, SWITCH_CONFIG, initial_switch_config); write_4(sc, CPU_PORT_CONFIG, initial_cpu_config); write_4(sc, MAC_PORT_0_CONFIG, initial_port0_config); write_4(sc, MAC_PORT_1_CONFIG, initial_port1_config); clear_mac_entries(sc, 1); } static void ece_restart(struct ece_softc *sc) { struct ifnet *ifp = sc->ifp; ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; /* Enable port 0. */ write_4(sc, PORT_0_CONFIG, read_4(sc, PORT_0_CONFIG) & ~(PORT_DISABLE)); write_4(sc, INTERRUPT_MASK, 0x00000000); write_4(sc, FS_DMA_CONTROL, 1); callout_reset(&sc->tick_ch, hz, ece_tick, sc); } static void set_filter(struct ece_softc *sc) { struct ifnet *ifp; struct ifmultiaddr *ifma; uint32_t mac_port_config; ifp = sc->ifp; clear_mac_entries(sc, 0); if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) { mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); mac_port_config &= ~(DISABLE_BROADCAST_PACKET); mac_port_config &= ~(DISABLE_MULTICAST_PACKET); write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); return; } if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; add_mac_entry(sc, LLADDR((struct sockaddr_dl *)ifma->ifma_addr)); } if_maddr_runlock(ifp); } static int eceioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct ece_softc *sc = ifp->if_softc; struct mii_data *mii; struct ifreq *ifr = (struct ifreq *)data; int mask, error = 0; switch (cmd) { case SIOCSIFFLAGS: ECE_LOCK(sc); if ((ifp->if_flags & IFF_UP) == 0 && ifp->if_drv_flags & IFF_DRV_RUNNING) { ifp->if_drv_flags &= ~IFF_DRV_RUNNING; ecestop(sc); } else { /* Reinitialize card on any parameter change. */ if ((ifp->if_flags & IFF_UP) && !(ifp->if_drv_flags & IFF_DRV_RUNNING)) ece_restart(sc); } ECE_UNLOCK(sc); break; case SIOCADDMULTI: case SIOCDELMULTI: ECE_LOCK(sc); set_filter(sc); ECE_UNLOCK(sc); break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: mii = device_get_softc(sc->miibus); error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); break; case SIOCSIFCAP: mask = ifp->if_capenable ^ ifr->ifr_reqcap; if (mask & IFCAP_VLAN_MTU) { ECE_LOCK(sc); ECE_UNLOCK(sc); } default: error = ether_ioctl(ifp, cmd, data); break; } return (error); } static void ece_child_detached(device_t dev, device_t child) { struct ece_softc *sc; sc = device_get_softc(dev); if (child == sc->miibus) sc->miibus = NULL; } /* * MII bus support routines. */ static int ece_miibus_readreg(device_t dev, int phy, int reg) { struct ece_softc *sc; sc = device_get_softc(dev); return (phy_read(sc, phy, reg)); } static int ece_miibus_writereg(device_t dev, int phy, int reg, int data) { struct ece_softc *sc; sc = device_get_softc(dev); phy_write(sc, phy, reg, data); return (0); } static device_method_t ece_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ece_probe), DEVMETHOD(device_attach, ece_attach), DEVMETHOD(device_detach, ece_detach), /* Bus interface */ DEVMETHOD(bus_child_detached, ece_child_detached), /* MII interface */ DEVMETHOD(miibus_readreg, ece_miibus_readreg), DEVMETHOD(miibus_writereg, ece_miibus_writereg), { 0, 0 } }; static driver_t ece_driver = { "ece", ece_methods, sizeof(struct ece_softc), }; DRIVER_MODULE(ece, econaarm, ece_driver, ece_devclass, 0, 0); DRIVER_MODULE(miibus, ece, miibus_driver, miibus_devclass, 0, 0); MODULE_DEPEND(ece, miibus, 1, 1, 1); MODULE_DEPEND(ece, ether, 1, 1, 1); Index: head/sys/arm/freescale/imx/imx6_ipu.c =================================================================== --- head/sys/arm/freescale/imx/imx6_ipu.c (revision 299068) +++ head/sys/arm/freescale/imx/imx6_ipu.c (revision 299069) @@ -1,1203 +1,1203 @@ /*- * Copyright 2015 Oleksandr Tymoshenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fb_if.h" #include "hdmi_if.h" #define EDID_DEBUG_not static int have_ipu = 0; #define LDB_CLOCK_RATE 280000000 #define MODE_HBP(mode) ((mode)->htotal - (mode)->hsync_end) #define MODE_HFP(mode) ((mode)->hsync_start - (mode)->hdisplay) #define MODE_HSW(mode) ((mode)->hsync_end - (mode)->hsync_start) #define MODE_VBP(mode) ((mode)->vtotal - (mode)->vsync_end) #define MODE_VFP(mode) ((mode)->vsync_start - (mode)->vdisplay) #define MODE_VSW(mode) ((mode)->vsync_end - (mode)->vsync_start) #define MODE_BPP 16 #define MODE_PIXEL_CLOCK_INVERT 1 #define M(nm,hr,vr,clk,hs,he,ht,vs,ve,vt,f) \ { clk, hr, hs, he, ht, vr, vs, ve, vt, f, nm } static struct videomode mode1024x768 = M("1024x768x60",1024,768,65000,1048,1184,1344,771,777,806,VID_NHSYNC|VID_PHSYNC); #define DMA_CHANNEL 23 #define DC_CHAN5 5 #define DI_PORT 0 #define IPU_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define IPU_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define IPU_LOCK_INIT(_sc) mtx_init(&(_sc)->sc_mtx, \ device_get_nameunit(_sc->sc_dev), "ipu", MTX_DEF) #define IPU_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx) #define IPU_READ4(_sc, reg) bus_read_4((_sc)->sc_mem_res, (reg)) #define IPU_WRITE4(_sc, reg, value) \ bus_write_4((_sc)->sc_mem_res, (reg), (value)) #define CPMEM_BASE 0x300000 #define DC_TEMPL_BASE 0x380000 /* Microcode */ /* Word 1 */ #define TEMPLATE_SYNC(v) ((v) << 0) #define TEMPLATE_GLUELOGIC(v) ((v) << 4) #define TEMPLATE_MAPPING(v) ((v) << 15) #define TEMPLATE_WAVEFORM(v) ((v) << 11) #define GLUELOGIC_KEEP_ASSERTED (1 << 3) #define GLUELOGIC_KEEP_NEGATED (1 << 2) /* Word 2 */ #define TEMPLATE_OPCODE(v) ((v) << 4) #define OPCODE_WROD 0x18 #define TEMPLATE_STOP (1 << 9) #define IPU_CONF 0x200000 #define IPU_CONF_DMFC_EN (1 << 10) #define IPU_CONF_DC_EN (1 << 9) #define IPU_CONF_DI1_EN (1 << 7) #define IPU_CONF_DI0_EN (1 << 6) #define IPU_CONF_DP_EN (1 << 5) #define IPU_DISP_GEN 0x2000C4 #define DISP_GEN_DI1_CNTR_RELEASE (1 << 25) #define DISP_GEN_DI0_CNTR_RELEASE (1 << 24) #define DISP_GEN_MCU_MAX_BURST_STOP (1 << 22) #define DISP_GEN_MCU_T_SHIFT 18 #define IPU_MEM_RST 0x2000DC #define IPU_MEM_RST_START (1 << 31) #define IPU_MEM_RST_ALL 0x807FFFFF #define IPU_CH_DB_MODE_SEL_0 0x200150 #define IPU_CH_DB_MODE_SEL_1 0x200154 #define IPU_CUR_BUF_0 0x20023C #define IPU_CUR_BUF_1 0x200240 #define IPU_IDMAC_CH_EN_1 0x208004 #define IPU_IDMAC_CH_EN_2 0x208008 #define IPU_IDMAC_CH_PRI_1 0x208014 #define IPU_IDMAC_CH_PRI_2 0x208018 #define IPU_DI0_GENERAL 0x240000 #define DI_CLOCK_EXTERNAL (1 << 20) #define DI_GENERAL_POL_CLK (1 << 17) #define DI_GENERAL_POLARITY_3 (1 << 2) #define DI_GENERAL_POLARITY_2 (1 << 1) #define IPU_DI0_BS_CLKGEN0 0x240004 #define DI_BS_CLKGEN0(_int, _frac) (((_int) << 4) | (_frac)) #define IPU_DI0_BS_CLKGEN1 0x240008 #define DI_BS_CLKGEN1_DOWN(_int, _frac) ((((_int) << 1) | (_frac)) << 16) #define IPU_DI0_SW_GEN0_1 0x24000C #define DI_RUN_VALUE_M1(v) ((v) << 19) #define DI_RUN_RESOLUTION(v) ((v) << 16) #define DI_OFFSET_VALUE(v) ((v) << 3) #define IPU_DI0_SW_GEN1_1 0x240030 #define DI_CNT_POLARITY_GEN_EN(v) ((v) << 29) #define DI_CNT_AUTO_RELOAD (1 << 28) #define DI_CNT_CLR_SEL(v) ((v) << 25) #define DI_CNT_DOWN(v) ((v) << 16) #define DI_CNT_POLARITY_TRIGGER_SEL(v) ((v) << 12) #define DI_CNT_POLARITY_CLR_SEL(v) ((v) << 9) #define IPU_DI0_SYNC_AS_GEN 0x240054 #define SYNC_AS_GEN_VSYNC_SEL(v) ((v) << 13) #define SYNC_AS_GEN_SYNC_START(v) ((v) << 0) #define IPU_DI0_DW_GEN_0 0x240058 #define DW_GEN_DI_ACCESS_SIZE(v) ((v) << 24) #define DW_GEN_DI_COMPONENT_SIZE(v) ((v) << 16) #define DW_GEN_DI_SET_MASK 3 #define DW_GEN_DI_PIN_15_SET(v) ((v) << 8) #define IPU_DI0_DW_SET3_0 0x240118 #define DW_SET_DATA_CNT_DOWN(v) ((v) << 16) #define DW_SET_DATA_CNT_UP(v) ((v) << 0) #define IPU_DI0_STP_REP 0x240148 #define IPU_DI0_POL 0x240164 #define DI_POL_DRDY_POLARITY_15 (1 << 4) #define IPU_DI0_SCR_CONF 0x240170 #define IPU_DI1_GENERAL 0x248000 #define IPU_DI1_BS_CLKGEN0 0x248004 #define IPU_DI1_BS_CLKGEN1 0x248008 #define IPU_DI1_SW_GEN0_1 0x24800C #define IPU_DI1_SW_GEN1_1 0x248030 #define IPU_DI1_SYNC_AS_GEN 0x248054 #define IPU_DI1_DW_GEN_0 0x248058 #define IPU_DI1_POL 0x248164 #define IPU_DI1_DW_SET3_0 0x248118 #define IPU_DI1_STP_REP 0x248148 #define IPU_DI1_SCR_CONF 0x248170 #define DMFC_RD_CHAN 0x260000 #define DMFC_WR_CHAN 0x260004 #define DMFC_WR_CHAN_BURST_SIZE_32 (0 << 6) #define DMFC_WR_CHAN_BURST_SIZE_16 (1 << 6) #define DMFC_WR_CHAN_BURST_SIZE_8 (2 << 6) #define DMFC_WR_CHAN_BURST_SIZE_4 (3 << 6) #define DMFC_WR_CHAN_BURST_SIZE_4 (3 << 6) #define DMFC_WR_CHAN_FIFO_SIZE_128 (2 << 3) #define DMFC_WR_CHAN_DEF 0x260008 #define DMFC_WR_CHAN_DEF_WM_CLR_2C(v) ((v) << 29) #define DMFC_WR_CHAN_DEF_WM_CLR_1C(v) ((v) << 21) #define DMFC_WR_CHAN_DEF_WM_CLR_2(v) ((v) << 13) #define DMFC_WR_CHAN_DEF_WM_CLR_1(v) ((v) << 5) #define DMFC_WR_CHAN_DEF_WM_SET_1(v) ((v) << 2) #define DMFC_WR_CHAN_DEF_WM_EN_1 (1 << 1) #define DMFC_DP_CHAN 0x26000C #define DMFC_DP_CHAN_BURST_SIZE_8 2 #define DMFC_DP_CHAN_FIFO_SIZE_256 1 #define DMFC_DP_CHAN_FIFO_SIZE_128 2 #define DMFC_DP_CHAN_BURST_SIZE_5F(v) ((v) << 14) #define DMFC_DP_CHAN_FIFO_SIZE_5F(v) ((v) << 11) #define DMFC_DP_CHAN_ST_ADDR_SIZE_5F(v) ((v) << 8) #define DMFC_DP_CHAN_BURST_SIZE_5B(v) ((v) << 6) #define DMFC_DP_CHAN_FIFO_SIZE_5B(v) ((v) << 3) #define DMFC_DP_CHAN_ST_ADDR_SIZE_5B(v) ((v) << 0) #define DMFC_DP_CHAN_DEF 0x260010 #define DMFC_DP_CHAN_DEF_WM_CLR_6F(v) ((v) << 29) #define DMFC_DP_CHAN_DEF_WM_CLR_6B(v) ((v) << 21) #define DMFC_DP_CHAN_DEF_WM_CLR_5F(v) ((v) << 13) #define DMFC_DP_CHAN_DEF_WM_SET_5F(v) ((v) << 10) #define DMFC_DP_CHAN_DEF_WM_EN_5F (1 << 9) #define DMFC_DP_CHAN_DEF_WM_CLR_5B(v) ((v) << 5) #define DMFC_DP_CHAN_DEF_WM_SET_5B(v) ((v) << 2) #define DMFC_DP_CHAN_DEF_WM_EN_5B (1 << 1) #define DMFC_GENERAL_1 0x260014 #define DMFC_GENERAL_1_WAIT4EOT_5B (1 << 20) #define DMFC_IC_CTRL 0x26001C #define DMFC_IC_CTRL_DISABLED 0x2 #define DC_WRITE_CH_CONF_1 0x0025801C #define WRITE_CH_CONF_PROG_CHAN_TYP_MASK (7 << 5) #define WRITE_CH_CONF_PROG_CHAN_NORMAL (4 << 5) #define DC_WRITE_CH_ADDR_1 0x00258020 #define DC_WRITE_CH_CONF_5 0x0025805C #define WRITE_CH_CONF_PROG_DISP_ID(v) ((v) << 3) #define WRITE_CH_CONF_PROG_DI_ID(v) ((v) << 2) #define WRITE_CH_CONF_PROG_W_SIZE(v) (v) #define DC_WRITE_CH_ADDR_5 0x00258060 #define DC_RL0_CH_5 0x00258064 #define DC_GEN 0x002580D4 #define DC_GEN_SYNC_PRIORITY (1 << 7) #define DC_GEN_ASYNC (0 << 1) #define DC_GEN_SYNC (2 << 1) #define DC_DISP_CONF2(di) (0x002580E8 + (di) * 4) #define DC_MAP_CONF_0 0x00258108 #define DC_MAP_CONF_15 0x00258144 #define DC_MAP_CONF_VAL(map) (DC_MAP_CONF_15 + ((map) / 2) * sizeof(uint32_t)) #define MAP_CONF_VAL_MASK 0xffff #define DC_MAP_CONF_PTR(ptr) (DC_MAP_CONF_0 + ((ptr) / 2) * sizeof(uint32_t)) #define MAP_CONF_PTR_MASK 0x1f #define DI_COUNTER_INT_HSYNC 1 #define DI_COUNTER_HSYNC 2 #define DI_COUNTER_VSYNC 3 #define DI_COUNTER_AD_0 4 #define DI_COUNTER_AD_1 5 #define DI_SYNC_NONE 0 #define DI_SYNC_CLK 1 #define DI_SYNC_COUNTER(c) ((c) + 1) struct ipu_cpmem_word { uint32_t data[5]; uint32_t padding[3]; }; struct ipu_cpmem_ch_param { struct ipu_cpmem_word word[2]; }; #define CH_PARAM_RESET(param) memset(param, 0, sizeof(*param)) #define IPU_READ_CH_PARAM(_sc, ch, param) bus_read_region_4( \ (_sc)->sc_mem_res, CPMEM_BASE + ch * (sizeof(*param)),\ (uint32_t*)param, sizeof(*param) / 4) #define IPU_WRITE_CH_PARAM(_sc, ch, param) bus_write_region_4( \ (_sc)->sc_mem_res, CPMEM_BASE + ch * (sizeof(*param)),\ (uint32_t*)param, sizeof(*param) / 4) #define CH_PARAM_SET_FW(param, v) ipu_ch_param_set_value((param), \ 0, 125, 13, (v)) #define CH_PARAM_SET_FH(param, v) ipu_ch_param_set_value((param), \ 0, 138, 12, (v)) #define CH_PARAM_SET_SLY(param, v) ipu_ch_param_set_value((param), \ 1, 102, 14, (v)) #define CH_PARAM_SET_EBA0(param, v) ipu_ch_param_set_value((param), \ 1, 0, 29, (v)) #define CH_PARAM_SET_EBA1(param, v) ipu_ch_param_set_value((param), \ 1, 29, 29, (v)) #define CH_PARAM_SET_BPP(param, v) ipu_ch_param_set_value((param), \ 0, 107, 3, (v)) #define CH_PARAM_SET_PFS(param, v) ipu_ch_param_set_value((param), \ 1, 85, 4, (v)) #define CH_PARAM_SET_NPB(param, v) ipu_ch_param_set_value((param), \ 1, 78, 7, (v)) #define CH_PARAM_SET_UBO(param, v) ipu_ch_param_set_value((param), \ 0, 46, 22, (v)) #define CH_PARAM_SET_VBO(param, v) ipu_ch_param_set_value((param), \ 0, 68, 22, (v)) #define CH_PARAM_SET_RED_WIDTH(param, v) ipu_ch_param_set_value((param), \ 1, 116, 3, (v)) #define CH_PARAM_SET_RED_OFFSET(param, v) ipu_ch_param_set_value((param), \ 1, 128, 5, (v)) #define CH_PARAM_SET_GREEN_WIDTH(param, v) ipu_ch_param_set_value((param), \ 1, 119, 3, (v)) #define CH_PARAM_SET_GREEN_OFFSET(param, v) ipu_ch_param_set_value((param), \ 1, 133, 5, (v)) #define CH_PARAM_SET_BLUE_WIDTH(param, v) ipu_ch_param_set_value((param), \ 1, 122, 3, (v)) #define CH_PARAM_SET_BLUE_OFFSET(param, v) ipu_ch_param_set_value((param), \ 1, 138, 5, (v)) #define CH_PARAM_SET_ALPHA_WIDTH(param, v) ipu_ch_param_set_value((param), \ 1, 125, 3, (v)) #define CH_PARAM_SET_ALPHA_OFFSET(param, v) ipu_ch_param_set_value((param), \ 1, 143, 5, (v)) #define CH_PARAM_GET_FW(param) ipu_ch_param_get_value((param), \ 0, 125, 13) #define CH_PARAM_GET_FH(param) ipu_ch_param_get_value((param), \ 0, 138, 12) #define CH_PARAM_GET_SLY(param) ipu_ch_param_get_value((param), \ 1, 102, 14) #define CH_PARAM_GET_EBA0(param) ipu_ch_param_get_value((param), \ 1, 0, 29) #define CH_PARAM_GET_EBA1(param) ipu_ch_param_get_value((param), \ 1, 29, 29) #define CH_PARAM_GET_BPP(param) ipu_ch_param_get_value((param), \ 0, 107, 3) #define CH_PARAM_GET_PFS(param) ipu_ch_param_get_value((param), \ 1, 85, 4) #define CH_PARAM_GET_NPB(param) ipu_ch_param_get_value((param), \ 1, 78, 7) #define CH_PARAM_GET_UBO(param) ipu_ch_param_get_value((param), \ 0, 46, 22) #define CH_PARAM_GET_VBO(param) ipu_ch_param_get_value((param), \ 0, 68, 22) #define CH_PARAM_GET_RED_WIDTH(param) ipu_ch_param_get_value((param), \ 1, 116, 3) #define CH_PARAM_GET_RED_OFFSET(param) ipu_ch_param_get_value((param), \ 1, 128, 5) #define CH_PARAM_GET_GREEN_WIDTH(param) ipu_ch_param_get_value((param), \ 1, 119, 3) #define CH_PARAM_GET_GREEN_OFFSET(param) ipu_ch_param_get_value((param), \ 1, 133, 5) #define CH_PARAM_GET_BLUE_WIDTH(param) ipu_ch_param_get_value((param), \ 1, 122, 3) #define CH_PARAM_GET_BLUE_OFFSET(param) ipu_ch_param_get_value((param), \ 1, 138, 5) #define CH_PARAM_GET_ALPHA_WIDTH(param) ipu_ch_param_get_value((param), \ 1, 125, 3) #define CH_PARAM_GET_ALPHA_OFFSET(param) ipu_ch_param_get_value((param), \ 1, 143, 5) #define IPU_PIX_FORMAT_BPP_32 0 #define IPU_PIX_FORMAT_BPP_24 1 #define IPU_PIX_FORMAT_BPP_18 2 #define IPU_PIX_FORMAT_BPP_16 3 #define IPU_PIX_FORMAT_BPP_12 4 #define IPU_PIX_FORMAT_BPP_8 5 #define IPU_PIX_FORMAT_BPP_ #define IPU_PIX_FORMAT_RGB 7 enum dc_event_t { DC_EVENT_NF = 0, DC_EVENT_NL, DC_EVENT_EOF, DC_EVENT_NFIELD, DC_EVENT_EOL, DC_EVENT_EOFIELD, DC_EVENT_NEW_ADDR, DC_EVENT_NEW_CHAN, DC_EVENT_NEW_DATA }; struct ipu_softc { device_t sc_dev; struct resource *sc_mem_res; int sc_mem_rid; struct resource *sc_irq_res; int sc_irq_rid; void *sc_intr_hl; struct mtx sc_mtx; struct fb_info sc_fb_info; struct videomode *sc_mode; /* Framebuffer */ bus_dma_tag_t sc_dma_tag; bus_dmamap_t sc_dma_map; size_t sc_fb_size; bus_addr_t sc_fb_phys; uint8_t *sc_fb_base; /* HDMI */ eventhandler_tag sc_hdmi_evh; }; static void ipu_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) { bus_addr_t *addr; if (err) return; addr = (bus_addr_t*)arg; *addr = segs[0].ds_addr; } static void ipu_ch_param_set_value(struct ipu_cpmem_ch_param *param, int word, unsigned int offset, int len, uint32_t value) { uint32_t datapos, bitpos, mask; uint32_t data, data2; KASSERT((len <= 32), ("%s: field len is more than 32", __func__)); datapos = offset / 32; bitpos = offset % 32; mask = (1 << len) - 1; data = param->word[word].data[datapos]; data &= ~(mask << bitpos); data |= (value << bitpos); param->word[word].data[datapos] = data; if ((bitpos + len) > 32) { len = bitpos + len - 32; mask = (1UL << len) - 1; data2 = param->word[word].data[datapos + 1]; data2 &= mask; data2 |= (value >> (32 - bitpos)); param->word[word].data[datapos + 1] = data2; } } #ifdef DEBUG static uint32_t ipu_ch_param_get_value(struct ipu_cpmem_ch_param *param, int word, unsigned int offset, int len) { uint32_t datapos, bitpos, mask; uint32_t data, data2; KASSERT((len <= 32), ("%s: field len is more than 32", __func__)); datapos = offset / 32; bitpos = offset % 32; mask = (1UL << len) - 1; data = param->word[word].data[datapos]; data = data >> bitpos; data &= mask; if ((bitpos + len) > 32) { len = bitpos + len - 32; mask = (1UL << len) - 1; data2 = param->word[word].data[datapos + 1]; data2 &= mask; data |= (data2 << (32 - bitpos)); } return (data); } static void ipu_print_channel(struct ipu_cpmem_ch_param *param) { int offset0[] = {0, 10, 19, 32, 44, 45, 46, 68, 90, 94, 95, 113, 114, 117, 119, 120, 121, 122, 123, 124, 125, 138, 150, 151, -1}; int offset1[] = {0, 29, 58, 78, 85, 89, 90, 93, 95, 102, 116, 119, 122, 125, 128, 133, 138, 143, 148, 149, 150, -1}; printf("WORD0: %08x %08x %08x %08x %08x\n", param->word[0].data[0], param->word[0].data[1], param->word[0].data[2], param->word[0].data[3], param->word[0].data[4]); printf("WORD1: %08x %08x %08x %08x %08x\n", param->word[1].data[0], param->word[1].data[1], param->word[1].data[2], param->word[1].data[3], param->word[1].data[4]); for (int i = 0; offset0[i + 1] != -1; i++) { int len = offset0[i + 1] - offset0[i]; printf("W0[%d:%d] = %d\n", offset0[i], offset0[i] + len - 1, ipu_ch_param_get_value(param, 0, offset0[i], len) ); } for (int i = 0; offset1[i + 1] != -1; i++) { int len = offset1[i + 1] - offset1[i]; printf("W1[%d:%d] = %d\n", offset1[i], offset1[i] + len - 1, ipu_ch_param_get_value(param, 1, offset1[i], len) ); } printf("FW: %d\n", CH_PARAM_GET_FW(param)); printf("FH: %d\n", CH_PARAM_GET_FH(param)); printf("SLY: %d\n", CH_PARAM_GET_SLY(param)); printf("EBA0: 0x%08x\n", CH_PARAM_GET_EBA0(param)); printf("EBA1: 0x%08x\n", CH_PARAM_GET_EBA1(param)); printf("BPP: %d\n", CH_PARAM_GET_BPP(param)); printf("PFS: %d\n", CH_PARAM_GET_PFS(param)); printf("NPB: %d\n", CH_PARAM_GET_NPB(param)); printf("UBO: %d\n", CH_PARAM_GET_UBO(param)); printf("VBO: %d\n", CH_PARAM_GET_VBO(param)); printf("RED: %d bits @%d\n", CH_PARAM_GET_RED_WIDTH(param) + 1, CH_PARAM_GET_RED_OFFSET(param)); printf("GREEN: %d bits @%d\n", CH_PARAM_GET_GREEN_WIDTH(param) + 1, CH_PARAM_GET_GREEN_OFFSET(param)); printf("BLUE: %d bits @%d\n", CH_PARAM_GET_BLUE_WIDTH(param) + 1, CH_PARAM_GET_BLUE_OFFSET(param)); printf("ALPHA: %d bits @%d\n", CH_PARAM_GET_ALPHA_WIDTH(param) + 1, CH_PARAM_GET_ALPHA_OFFSET(param)); } #endif static void ipu_di_enable(struct ipu_softc *sc, int di) { uint32_t flag, reg; flag = di ? DISP_GEN_DI1_CNTR_RELEASE : DISP_GEN_DI0_CNTR_RELEASE; reg = IPU_READ4(sc, IPU_DISP_GEN); reg |= flag; IPU_WRITE4(sc, IPU_DISP_GEN, reg); } static void ipu_config_wave_gen_0(struct ipu_softc *sc, int di, int wave_gen, int run_value, int run_res, int offset_value, int offset_res) { uint32_t addr, reg; addr = (di ? IPU_DI1_SW_GEN0_1 : IPU_DI0_SW_GEN0_1) + (wave_gen - 1) * sizeof(uint32_t); reg = DI_RUN_VALUE_M1(run_value) | DI_RUN_RESOLUTION(run_res) | DI_OFFSET_VALUE(offset_value) | offset_res; IPU_WRITE4(sc, addr, reg); } static void ipu_config_wave_gen_1(struct ipu_softc *sc, int di, int wave_gen, int repeat_count, int cnt_clr_src, int cnt_polarity_gen_en, int cnt_polarity_clr_src, int cnt_polarity_trigger_src, int cnt_up, int cnt_down) { uint32_t addr, reg; addr = (di ? IPU_DI1_SW_GEN1_1 : IPU_DI0_SW_GEN1_1) + (wave_gen - 1) * sizeof(uint32_t); reg = DI_CNT_POLARITY_GEN_EN(cnt_polarity_gen_en) | DI_CNT_CLR_SEL(cnt_clr_src) | DI_CNT_POLARITY_TRIGGER_SEL(cnt_polarity_trigger_src) | DI_CNT_POLARITY_CLR_SEL(cnt_polarity_clr_src); reg |= DI_CNT_DOWN(cnt_down) | cnt_up; if (repeat_count == 0) reg |= DI_CNT_AUTO_RELOAD; IPU_WRITE4(sc, addr, reg); addr = (di ? IPU_DI1_STP_REP : IPU_DI0_STP_REP) + (wave_gen - 1) / 2 * sizeof(uint32_t); reg = IPU_READ4(sc, addr); if (wave_gen % 2) { reg &= ~(0xffff); reg |= repeat_count; } else { reg &= ~(0xffff << 16); reg |= (repeat_count << 16); } IPU_WRITE4(sc, addr, reg); } static void ipu_reset_wave_gen(struct ipu_softc *sc, int di, int wave_gen) { uint32_t addr, reg; addr = (di ? IPU_DI1_SW_GEN0_1 : IPU_DI0_SW_GEN0_1) + (wave_gen - 1) * sizeof(uint32_t); IPU_WRITE4(sc, addr, 0); addr = (di ? IPU_DI1_SW_GEN1_1 : IPU_DI0_SW_GEN1_1) + (wave_gen - 1) * sizeof(uint32_t); IPU_WRITE4(sc, addr, 0); addr = (di ? IPU_DI1_STP_REP : IPU_DI0_STP_REP) + (wave_gen - 1) / 2 * sizeof(uint32_t); reg = IPU_READ4(sc, addr); if (wave_gen % 2) reg &= ~(0xffff); else reg &= ~(0xffff << 16); IPU_WRITE4(sc, addr, reg); } static void ipu_init_microcode_template(struct ipu_softc *sc, int di, int map) { uint32_t addr; uint32_t w1, w2; int i, word; int glue; word = di ? 2 : 5; for (i = 0; i < 3; i++) { if (i == 0) glue = GLUELOGIC_KEEP_ASSERTED; else if (i == 1) glue = GLUELOGIC_KEEP_NEGATED; else if (i == 2) glue = 0; w1 = TEMPLATE_SYNC(5) | TEMPLATE_GLUELOGIC(glue) | TEMPLATE_WAVEFORM(1) | /* wave unit 0 */ TEMPLATE_MAPPING(map + 1); /* operand is zero */ /* Write data to DI and Hold data in register */ w2 = TEMPLATE_OPCODE(OPCODE_WROD) | TEMPLATE_STOP; addr = DC_TEMPL_BASE + (word + i) * 2 * sizeof(uint32_t); IPU_WRITE4(sc, addr, w1); IPU_WRITE4(sc, addr + sizeof(uint32_t), w2); } } static void ipu_config_timing(struct ipu_softc *sc, int di) { int div; uint32_t di_scr_conf; uint32_t gen_offset, gen; uint32_t as_gen_offset, as_gen; uint32_t dw_gen_offset, dw_gen; uint32_t dw_set_offset, dw_set; uint32_t bs_clkgen_offset; int map; /* TODO: check mode restrictions / fixup */ /* TODO: enable timers, get divisors */ div = 1; map = 0; bs_clkgen_offset = di ? IPU_DI1_BS_CLKGEN0 : IPU_DI0_BS_CLKGEN0; IPU_WRITE4(sc, bs_clkgen_offset, DI_BS_CLKGEN0(div, 0)); /* half of the divider */ IPU_WRITE4(sc, bs_clkgen_offset + 4, DI_BS_CLKGEN1_DOWN(div / 2, div % 2)); /* * TODO: Configure LLDB clock by changing following fields * in CCM fields: * CS2CDR_LDB_DI0_CLK_SEL * CSCMR2_LDB_DI0_IPU_DIV * CBCDR_MMDC_CH1_AXI_PODF */ /* Setup wave generator */ dw_gen_offset = di ? IPU_DI1_DW_GEN_0 : IPU_DI0_DW_GEN_0; dw_gen = DW_GEN_DI_ACCESS_SIZE(div - 1) | DW_GEN_DI_COMPONENT_SIZE(div - 1); dw_gen &= ~DW_GEN_DI_PIN_15_SET(DW_GEN_DI_SET_MASK); dw_gen |= DW_GEN_DI_PIN_15_SET(3); /* set 3*/ IPU_WRITE4(sc, dw_gen_offset, dw_gen); dw_set_offset = di ? IPU_DI1_DW_SET3_0 : IPU_DI0_DW_SET3_0; dw_set = DW_SET_DATA_CNT_DOWN(div * 2) | DW_SET_DATA_CNT_UP(0); IPU_WRITE4(sc, dw_set_offset, dw_set); /* DI_COUNTER_INT_HSYNC */ ipu_config_wave_gen_0(sc, di, DI_COUNTER_INT_HSYNC, sc->sc_mode->htotal - 1, DI_SYNC_CLK, 0, DI_SYNC_NONE); ipu_config_wave_gen_1(sc, di, DI_COUNTER_INT_HSYNC, 0, DI_SYNC_NONE, 0, DI_SYNC_NONE, DI_SYNC_NONE, 0, 0); /* DI_COUNTER_HSYNC */ ipu_config_wave_gen_0(sc, di, DI_COUNTER_HSYNC, sc->sc_mode->htotal - 1, DI_SYNC_CLK, 0, DI_SYNC_CLK); ipu_config_wave_gen_1(sc, di, DI_COUNTER_HSYNC, 0, DI_SYNC_NONE, 1, DI_SYNC_NONE, DI_SYNC_CLK, 0, MODE_HSW(sc->sc_mode) * 2); /* DI_COUNTER_VSYNC */ ipu_config_wave_gen_0(sc, di, DI_COUNTER_VSYNC, sc->sc_mode->vtotal - 1, DI_SYNC_COUNTER(DI_COUNTER_INT_HSYNC), 0, DI_SYNC_NONE); ipu_config_wave_gen_1(sc, di, DI_COUNTER_VSYNC, 0, DI_SYNC_NONE, 1, DI_SYNC_NONE, DI_SYNC_COUNTER(DI_COUNTER_INT_HSYNC), 0, MODE_VSW(sc->sc_mode) * 2); di_scr_conf = di ? IPU_DI1_SCR_CONF : IPU_DI0_SCR_CONF; IPU_WRITE4(sc, di_scr_conf, sc->sc_mode->vtotal - 1); /* TODO: update DI_SCR_CONF */ /* Active Data 0 */ ipu_config_wave_gen_0(sc, di, DI_COUNTER_AD_0, 0, DI_SYNC_COUNTER(DI_COUNTER_HSYNC), MODE_VSW(sc->sc_mode) + MODE_VFP(sc->sc_mode), DI_SYNC_COUNTER(DI_COUNTER_HSYNC)); ipu_config_wave_gen_1(sc, di, DI_COUNTER_AD_0, sc->sc_mode->vdisplay, DI_SYNC_COUNTER(DI_COUNTER_VSYNC), 0, DI_SYNC_NONE, DI_SYNC_NONE, 0, 0); ipu_config_wave_gen_0(sc, di, DI_COUNTER_AD_1, 0, DI_SYNC_CLK, MODE_HSW(sc->sc_mode) + MODE_HFP(sc->sc_mode), DI_SYNC_CLK); ipu_config_wave_gen_1(sc, di, DI_COUNTER_AD_1, sc->sc_mode->hdisplay, DI_SYNC_COUNTER(DI_COUNTER_AD_0), 0, DI_SYNC_NONE, DI_SYNC_NONE, 0, 0); ipu_reset_wave_gen(sc, di, 6); ipu_reset_wave_gen(sc, di, 7); ipu_reset_wave_gen(sc, di, 8); ipu_reset_wave_gen(sc, di, 9); ipu_init_microcode_template(sc, di, map); gen_offset = di ? IPU_DI1_GENERAL : IPU_DI0_GENERAL; gen = IPU_READ4(sc, gen_offset); if (sc->sc_mode->flags & VID_NHSYNC) gen &= ~DI_GENERAL_POLARITY_2; else /* active high */ gen |= DI_GENERAL_POLARITY_2; if (sc->sc_mode->flags & VID_NVSYNC) gen &= ~DI_GENERAL_POLARITY_3; else /* active high */ gen |= DI_GENERAL_POLARITY_3; if (MODE_PIXEL_CLOCK_INVERT) gen &= ~DI_GENERAL_POL_CLK; else gen |= DI_GENERAL_POL_CLK; /* Use LDB clock to drive pixel clock */ gen |= DI_CLOCK_EXTERNAL; IPU_WRITE4(sc, gen_offset, gen); as_gen_offset = di ? IPU_DI1_SYNC_AS_GEN : IPU_DI0_SYNC_AS_GEN; as_gen = SYNC_AS_GEN_VSYNC_SEL(DI_COUNTER_VSYNC - 1) | SYNC_AS_GEN_SYNC_START(2); IPU_WRITE4(sc, as_gen_offset, as_gen); IPU_WRITE4(sc, (di ? IPU_DI1_POL : IPU_DI0_POL), DI_POL_DRDY_POLARITY_15); IPU_WRITE4(sc, DC_DISP_CONF2(di), sc->sc_mode->hdisplay); } static void ipu_dc_enable(struct ipu_softc *sc) { uint32_t conf; /* channel 1 uses DI1 */ IPU_WRITE4(sc, DC_WRITE_CH_CONF_1, WRITE_CH_CONF_PROG_DI_ID(1)); conf = IPU_READ4(sc, DC_WRITE_CH_CONF_5); conf &= ~WRITE_CH_CONF_PROG_CHAN_TYP_MASK; conf |= WRITE_CH_CONF_PROG_CHAN_NORMAL; IPU_WRITE4(sc, DC_WRITE_CH_CONF_5, conf); /* TODO: enable clock */ } static void ipu_dc_link_event(struct ipu_softc *sc, int event, int addr, int priority) { uint32_t reg; int offset; int shift; if (event % 2) shift = 16; else shift = 0; offset = DC_RL0_CH_5 + (event / 2) * sizeof(uint32_t); reg = IPU_READ4(sc, offset); reg &= ~(0xFFFF << shift); reg |= ((addr << 8) | priority) << shift; IPU_WRITE4(sc, offset, reg); } static void ipu_dc_setup_map(struct ipu_softc *sc, int map, int byte, int offset, int mask) { uint32_t reg, shift, ptr; ptr = map * 3 + byte; reg = IPU_READ4(sc, DC_MAP_CONF_VAL(ptr)); if (ptr & 1) shift = 16; else shift = 0; reg &= ~(0xffff << shift); reg |= ((offset << 8) | mask) << shift; IPU_WRITE4(sc, DC_MAP_CONF_VAL(ptr), reg); reg = IPU_READ4(sc, DC_MAP_CONF_PTR(map)); if (map & 1) shift = 16 + 5 * byte; else shift = 5 * byte; reg &= ~(MAP_CONF_PTR_MASK << shift); reg |= (ptr) << shift; IPU_WRITE4(sc, DC_MAP_CONF_PTR(map), reg); } static void ipu_dc_reset_map(struct ipu_softc *sc, int map) { uint32_t reg, shift; reg = IPU_READ4(sc, DC_MAP_CONF_VAL(map)); if (map & 1) shift = 16; else shift = 0; reg &= ~(MAP_CONF_VAL_MASK << shift); IPU_WRITE4(sc, DC_MAP_CONF_VAL(map), reg); } static void ipu_dc_init(struct ipu_softc *sc, int di_port) { int addr; uint32_t conf; if (di_port) addr = 2; else addr = 5; ipu_dc_link_event(sc, DC_EVENT_NL, addr, 3); ipu_dc_link_event(sc, DC_EVENT_EOL, addr + 1, 2); ipu_dc_link_event(sc, DC_EVENT_NEW_DATA, addr + 2, 1); ipu_dc_link_event(sc, DC_EVENT_NF, 0, 0); ipu_dc_link_event(sc, DC_EVENT_NFIELD, 0, 0); ipu_dc_link_event(sc, DC_EVENT_EOF, 0, 0); ipu_dc_link_event(sc, DC_EVENT_EOFIELD, 0, 0); ipu_dc_link_event(sc, DC_EVENT_NEW_CHAN, 0, 0); ipu_dc_link_event(sc, DC_EVENT_NEW_ADDR, 0, 0); conf = WRITE_CH_CONF_PROG_W_SIZE(0x02) | WRITE_CH_CONF_PROG_DISP_ID(DI_PORT) | WRITE_CH_CONF_PROG_DI_ID(DI_PORT); IPU_WRITE4(sc, DC_WRITE_CH_CONF_5, conf); IPU_WRITE4(sc, DC_WRITE_CH_ADDR_5, 0x00000000); IPU_WRITE4(sc, DC_GEN, DC_GEN_SYNC_PRIORITY | DC_GEN_SYNC); /* High priority, sync */ } static void ipu_init_buffer(struct ipu_softc *sc) { struct ipu_cpmem_ch_param param; uint32_t stride; uint32_t reg, db_mode_sel, cur_buf; stride = sc->sc_mode->hdisplay * MODE_BPP / 8; - /* init channel paramters */ + /* init channel parameters */ CH_PARAM_RESET(¶m); /* XXX: interlaced modes are not supported yet */ CH_PARAM_SET_FW(¶m, sc->sc_mode->hdisplay - 1); CH_PARAM_SET_FH(¶m, sc->sc_mode->vdisplay - 1); CH_PARAM_SET_SLY(¶m, stride - 1); CH_PARAM_SET_EBA0(¶m, (sc->sc_fb_phys >> 3)); CH_PARAM_SET_EBA1(¶m, (sc->sc_fb_phys >> 3)); CH_PARAM_SET_BPP(¶m, IPU_PIX_FORMAT_BPP_16); CH_PARAM_SET_PFS(¶m, IPU_PIX_FORMAT_RGB); /* 16 pixels per burst access */ CH_PARAM_SET_NPB(¶m, 16 - 1); CH_PARAM_SET_RED_OFFSET(¶m, 0); CH_PARAM_SET_RED_WIDTH(¶m, 5 - 1); CH_PARAM_SET_GREEN_OFFSET(¶m, 5); CH_PARAM_SET_GREEN_WIDTH(¶m, 6 - 1); CH_PARAM_SET_BLUE_OFFSET(¶m, 11); CH_PARAM_SET_BLUE_WIDTH(¶m, 5 - 1); CH_PARAM_SET_ALPHA_OFFSET(¶m, 16); CH_PARAM_SET_ALPHA_WIDTH(¶m, 8 - 1); CH_PARAM_SET_UBO(¶m, 0); CH_PARAM_SET_VBO(¶m, 0); IPU_WRITE_CH_PARAM(sc, DMA_CHANNEL, ¶m); #ifdef DEBUG ipu_print_channel(¶m); #endif /* init DMFC */ IPU_WRITE4(sc, DMFC_IC_CTRL, DMFC_IC_CTRL_DISABLED); /* High resolution DP */ IPU_WRITE4(sc, DMFC_WR_CHAN, DMFC_WR_CHAN_BURST_SIZE_8 | DMFC_WR_CHAN_FIFO_SIZE_128); IPU_WRITE4(sc, DMFC_WR_CHAN_DEF, DMFC_WR_CHAN_DEF_WM_CLR_2C(1) | DMFC_WR_CHAN_DEF_WM_CLR_1C(1) | DMFC_WR_CHAN_DEF_WM_CLR_2(1) | DMFC_WR_CHAN_DEF_WM_CLR_1(7) | DMFC_WR_CHAN_DEF_WM_SET_1(5) | DMFC_WR_CHAN_DEF_WM_EN_1); IPU_WRITE4(sc, DMFC_DP_CHAN, DMFC_DP_CHAN_BURST_SIZE_5F(DMFC_DP_CHAN_BURST_SIZE_8) | DMFC_DP_CHAN_FIFO_SIZE_5F(DMFC_DP_CHAN_FIFO_SIZE_128) | DMFC_DP_CHAN_ST_ADDR_SIZE_5F(6) /* segment 6 */ | DMFC_DP_CHAN_BURST_SIZE_5B(DMFC_DP_CHAN_BURST_SIZE_8) | DMFC_DP_CHAN_FIFO_SIZE_5B(DMFC_DP_CHAN_FIFO_SIZE_256) | DMFC_DP_CHAN_ST_ADDR_SIZE_5B(2) /* segment 2 */); IPU_WRITE4(sc, DMFC_DP_CHAN_DEF, DMFC_DP_CHAN_DEF_WM_CLR_6F(1) | DMFC_DP_CHAN_DEF_WM_CLR_6B(1) | DMFC_DP_CHAN_DEF_WM_CLR_5F(7) | DMFC_DP_CHAN_DEF_WM_SET_5F(5) | DMFC_DP_CHAN_DEF_WM_EN_5F | DMFC_DP_CHAN_DEF_WM_CLR_5B(7) | DMFC_DP_CHAN_DEF_WM_SET_5B(5) | DMFC_DP_CHAN_DEF_WM_EN_5B); reg = IPU_READ4(sc, DMFC_GENERAL_1); reg &= ~(DMFC_GENERAL_1_WAIT4EOT_5B); IPU_WRITE4(sc, DMFC_GENERAL_1, reg); /* XXX: set priority? */ /* Set single buffer mode */ if (DMA_CHANNEL < 32) { db_mode_sel = IPU_CH_DB_MODE_SEL_0; cur_buf = IPU_CUR_BUF_0; } else { db_mode_sel = IPU_CH_DB_MODE_SEL_1; cur_buf = IPU_CUR_BUF_1; } reg = IPU_READ4(sc, db_mode_sel); reg |= (1UL << (DMA_CHANNEL & 0x1f)); IPU_WRITE4(sc, db_mode_sel, reg); IPU_WRITE4(sc, cur_buf, (1UL << (DMA_CHANNEL & 0x1f))); } static int ipu_init(struct ipu_softc *sc) { uint32_t reg, off; int i, err; size_t dma_size; IPU_WRITE4(sc, IPU_CONF, DI_PORT ? IPU_CONF_DI1_EN : IPU_CONF_DI0_EN); IPU_WRITE4(sc, IPU_MEM_RST, IPU_MEM_RST_ALL); i = 1000; while (i-- > 0) { if (!(IPU_READ4(sc, IPU_MEM_RST) & IPU_MEM_RST_START)) break; DELAY(1); } if (i <= 0) { err = ETIMEDOUT; device_printf(sc->sc_dev, "timeout while resetting memory\n"); goto fail; } ipu_dc_reset_map(sc, 0); ipu_dc_setup_map(sc, 0, 0, 7, 0xff); ipu_dc_setup_map(sc, 0, 1, 15, 0xff); ipu_dc_setup_map(sc, 0, 2, 23, 0xff); dma_size = round_page(sc->sc_mode->hdisplay * sc->sc_mode->vdisplay * (MODE_BPP / 8)); /* * Now allocate framebuffer memory */ err = bus_dma_tag_create( bus_get_dma_tag(sc->sc_dev), 4, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ dma_size, 1, /* maxsize, nsegments */ dma_size, 0, /* maxsegsize, flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->sc_dma_tag); if (err) goto fail; err = bus_dmamem_alloc(sc->sc_dma_tag, (void **)&sc->sc_fb_base, BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->sc_dma_map); if (err) { device_printf(sc->sc_dev, "cannot allocate framebuffer\n"); goto fail; } err = bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, sc->sc_fb_base, dma_size, ipu_dmamap_cb, &sc->sc_fb_phys, BUS_DMA_NOWAIT); if (err) { device_printf(sc->sc_dev, "cannot load DMA map\n"); goto fail; } /* Calculate actual FB Size */ sc->sc_fb_size = sc->sc_mode->hdisplay * sc->sc_mode->vdisplay * MODE_BPP / 8; ipu_dc_init(sc, DI_PORT); reg = IPU_READ4(sc, IPU_CONF); reg |= IPU_CONF_DMFC_EN | IPU_CONF_DC_EN | IPU_CONF_DP_EN; IPU_WRITE4(sc, IPU_CONF, reg); ipu_config_timing(sc, DI_PORT); ipu_init_buffer(sc); ipu_di_enable(sc, DI_PORT); /* Enable DMA channel */ off = (DMA_CHANNEL > 31) ? IPU_IDMAC_CH_EN_2 : IPU_IDMAC_CH_EN_1; reg = IPU_READ4(sc, off); reg |= (1 << (DMA_CHANNEL & 0x1f)); IPU_WRITE4(sc, off, reg); ipu_dc_enable(sc); sc->sc_fb_info.fb_name = device_get_nameunit(sc->sc_dev); sc->sc_fb_info.fb_vbase = (intptr_t)sc->sc_fb_base; sc->sc_fb_info.fb_pbase = sc->sc_fb_phys; sc->sc_fb_info.fb_size = sc->sc_fb_size; sc->sc_fb_info.fb_bpp = sc->sc_fb_info.fb_depth = MODE_BPP; sc->sc_fb_info.fb_stride = sc->sc_mode->hdisplay * MODE_BPP / 8; sc->sc_fb_info.fb_width = sc->sc_mode->hdisplay; sc->sc_fb_info.fb_height = sc->sc_mode->vdisplay; device_t fbd = device_add_child(sc->sc_dev, "fbd", device_get_unit(sc->sc_dev)); if (fbd == NULL) { device_printf(sc->sc_dev, "Failed to add fbd child\n"); goto fail; } if (device_probe_and_attach(fbd) != 0) { device_printf(sc->sc_dev, "Failed to attach fbd device\n"); goto fail; } return (0); fail: return (err); } static void ipu_hdmi_event(void *arg, device_t hdmi_dev) { struct ipu_softc *sc; uint8_t *edid; uint32_t edid_len; #ifdef EDID_DEBUG struct edid_info ei; #endif const struct videomode *videomode; sc = arg; edid = NULL; edid_len = 0; if (HDMI_GET_EDID(hdmi_dev, &edid, &edid_len) != 0) { device_printf(sc->sc_dev, "failed to get EDID info from HDMI framer\n"); } videomode = NULL; #ifdef EDID_DEBUG if ( edid && (edid_parse(edid, &ei) == 0)) { edid_print(&ei); } else device_printf(sc->sc_dev, "failed to parse EDID\n"); #endif sc->sc_mode = &mode1024x768; ipu_init(sc); HDMI_SET_VIDEOMODE(hdmi_dev, sc->sc_mode); } static int ipu_probe(device_t dev) { if (have_ipu) return (ENXIO); if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "fsl,imx6q-ipu")) return (ENXIO); device_set_desc(dev, "Freescale IPU"); return (BUS_PROBE_DEFAULT); } static int ipu_attach(device_t dev) { struct ipu_softc *sc; if (have_ipu) return (ENXIO); sc = device_get_softc(dev); sc->sc_dev = dev; sc->sc_mem_rid = 0; sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_mem_rid, RF_ACTIVE); if (!sc->sc_mem_res) { device_printf(dev, "cannot allocate memory window\n"); return (ENXIO); } sc->sc_irq_rid = 0; sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irq_rid, RF_ACTIVE); if (!sc->sc_irq_res) { bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_mem_rid, sc->sc_mem_res); device_printf(dev, "cannot allocate interrupt\n"); return (ENXIO); } /* Enable IPU1 */ imx_ccm_ipu_enable(1); if (src_reset_ipu() != 0) { device_printf(dev, "failed to reset IPU\n"); return (ENXIO); } IPU_LOCK_INIT(sc); sc->sc_hdmi_evh = EVENTHANDLER_REGISTER(hdmi_event, ipu_hdmi_event, sc, 0); have_ipu = 1; return (0); } static int ipu_detach(device_t dev) { /* Do not let unload driver */ return (EBUSY); } static struct fb_info * ipu_fb_getinfo(device_t dev) { struct ipu_softc *sc; sc = device_get_softc(dev); return (&sc->sc_fb_info); } static device_method_t ipu_methods[] = { DEVMETHOD(device_probe, ipu_probe), DEVMETHOD(device_attach, ipu_attach), DEVMETHOD(device_detach, ipu_detach), /* Framebuffer service methods */ DEVMETHOD(fb_getinfo, ipu_fb_getinfo), DEVMETHOD_END }; static driver_t ipu_driver = { "fb", ipu_methods, sizeof(struct ipu_softc), }; static devclass_t ipu_devclass; DRIVER_MODULE(ipu, simplebus, ipu_driver, ipu_devclass, 0, 0); MODULE_VERSION(ipu, 1); MODULE_DEPEND(ipu, simplebus, 1, 1, 1); Index: head/sys/arm/freescale/imx/imx6_ssi.c =================================================================== --- head/sys/arm/freescale/imx/imx6_ssi.c (revision 299068) +++ head/sys/arm/freescale/imx/imx6_ssi.c (revision 299069) @@ -1,854 +1,854 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * i.MX6 Synchronous Serial Interface (SSI) * * Chapter 61, i.MX 6Dual/6Quad Applications Processor Reference Manual, * Rev. 1, 04/2013 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define READ4(_sc, _reg) \ bus_space_read_4(_sc->bst, _sc->bsh, _reg) #define WRITE4(_sc, _reg, _val) \ bus_space_write_4(_sc->bst, _sc->bsh, _reg, _val) #define SSI_NCHANNELS 1 /* i.MX6 SSI registers */ #define SSI_STX0 0x00 /* Transmit Data Register n */ #define SSI_STX1 0x04 /* Transmit Data Register n */ #define SSI_SRX0 0x08 /* Receive Data Register n */ #define SSI_SRX1 0x0C /* Receive Data Register n */ #define SSI_SCR 0x10 /* Control Register */ #define SCR_I2S_MODE_S 5 /* I2S Mode Select. */ #define SCR_I2S_MODE_M 0x3 #define SCR_SYN (1 << 4) #define SCR_NET (1 << 3) /* Network mode */ #define SCR_RE (1 << 2) /* Receive Enable. */ #define SCR_TE (1 << 1) /* Transmit Enable. */ #define SCR_SSIEN (1 << 0) /* SSI Enable */ #define SSI_SISR 0x14 /* Interrupt Status Register */ #define SSI_SIER 0x18 /* Interrupt Enable Register */ #define SIER_RDMAE (1 << 22) /* Receive DMA Enable. */ #define SIER_RIE (1 << 21) /* Receive Interrupt Enable. */ #define SIER_TDMAE (1 << 20) /* Transmit DMA Enable. */ #define SIER_TIE (1 << 19) /* Transmit Interrupt Enable. */ #define SIER_TDE0IE (1 << 12) /* Transmit Data Register Empty 0. */ #define SIER_TUE0IE (1 << 8) /* Transmitter Underrun Error 0. */ #define SIER_TFE0IE (1 << 0) /* Transmit FIFO Empty 0 IE. */ #define SSI_STCR 0x1C /* Transmit Configuration Register */ #define STCR_TXBIT0 (1 << 9) /* Transmit Bit 0 shift MSB/LSB */ #define STCR_TFEN1 (1 << 8) /* Transmit FIFO Enable 1. */ #define STCR_TFEN0 (1 << 7) /* Transmit FIFO Enable 0. */ #define STCR_TFDIR (1 << 6) /* Transmit Frame Direction. */ #define STCR_TXDIR (1 << 5) /* Transmit Clock Direction. */ #define STCR_TSHFD (1 << 4) /* Transmit Shift Direction. */ #define STCR_TSCKP (1 << 3) /* Transmit Clock Polarity. */ #define STCR_TFSI (1 << 2) /* Transmit Frame Sync Invert. */ #define STCR_TFSL (1 << 1) /* Transmit Frame Sync Length. */ #define STCR_TEFS (1 << 0) /* Transmit Early Frame Sync. */ #define SSI_SRCR 0x20 /* Receive Configuration Register */ #define SSI_STCCR 0x24 /* Transmit Clock Control Register */ #define STCCR_DIV2 (1 << 18) /* Divide By 2. */ #define STCCR_PSR (1 << 17) /* Divide clock by 8. */ #define WL3_WL0_S 13 #define WL3_WL0_M 0xf #define DC4_DC0_S 8 #define DC4_DC0_M 0x1f #define PM7_PM0_S 0 #define PM7_PM0_M 0xff #define SSI_SRCCR 0x28 /* Receive Clock Control Register */ #define SSI_SFCSR 0x2C /* FIFO Control/Status Register */ #define SFCSR_RFWM1_S 20 /* Receive FIFO Empty WaterMark 1 */ #define SFCSR_RFWM1_M 0xf #define SFCSR_TFWM1_S 16 /* Transmit FIFO Empty WaterMark 1 */ #define SFCSR_TFWM1_M 0xf #define SFCSR_RFWM0_S 4 /* Receive FIFO Empty WaterMark 0 */ #define SFCSR_RFWM0_M 0xf #define SFCSR_TFWM0_S 0 /* Transmit FIFO Empty WaterMark 0 */ #define SFCSR_TFWM0_M 0xf #define SSI_SACNT 0x38 /* AC97 Control Register */ #define SSI_SACADD 0x3C /* AC97 Command Address Register */ #define SSI_SACDAT 0x40 /* AC97 Command Data Register */ #define SSI_SATAG 0x44 /* AC97 Tag Register */ #define SSI_STMSK 0x48 /* Transmit Time Slot Mask Register */ #define SSI_SRMSK 0x4C /* Receive Time Slot Mask Register */ #define SSI_SACCST 0x50 /* AC97 Channel Status Register */ #define SSI_SACCEN 0x54 /* AC97 Channel Enable Register */ #define SSI_SACCDIS 0x58 /* AC97 Channel Disable Register */ static MALLOC_DEFINE(M_SSI, "ssi", "ssi audio"); uint32_t ssi_dma_intr(void *arg, int chn); struct ssi_rate { uint32_t speed; uint32_t mfi; /* PLL4 Multiplication Factor Integer */ uint32_t mfn; /* PLL4 Multiplication Factor Numerator */ uint32_t mfd; /* PLL4 Multiplication Factor Denominator */ /* More dividers to configure can be added here */ }; static struct ssi_rate rate_map[] = { { 192000, 49, 152, 1000 }, /* PLL4 49.152 Mhz */ /* TODO: add more frequences */ { 0, 0 }, }; /* * i.MX6 example bit clock formula * * BCLK = 2 channels * 192000 hz * 24 bit = 9216000 hz = * (24000000 * (49 + 152/1000.0) / 4 / 4 / 2 / 2 / 2 / 1 / 1) * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ * | | | | | | | | | | | * Fref ------/ | | | | | | | | | | * PLL4 div select -/ | | | | | | | | | * PLL4 num --------------/ | | | | | | | | * PLL4 denom -------------------/ | | | | | | | * PLL4 post div ---------------------/ | | | | | | * CCM ssi pre div (CCM_CS1CDR) ----------/ | | | | | * CCM ssi post div (CCM_CS1CDR) -------------/ | | | | * SSI PM7_PM0_S ---------------------------------/ | | | * SSI Fixed divider ---------------------------------/ | | * SSI DIV2 ----------------------------------------------/ | * SSI PSR (prescaler /1 or /8) ------------------------------/ * * MCLK (Master clock) depends on DAC, usually BCLK * 4 */ struct sc_info { struct resource *res[2]; bus_space_tag_t bst; bus_space_handle_t bsh; device_t dev; struct mtx *lock; void *ih; int pos; int dma_size; bus_dma_tag_t dma_tag; bus_dmamap_t dma_map; bus_addr_t buf_base_phys; uint32_t *buf_base; struct sdma_conf *conf; struct ssi_rate *sr; struct sdma_softc *sdma_sc; int sdma_ev_rx; int sdma_ev_tx; int sdma_channel; }; /* Channel registers */ struct sc_chinfo { struct snd_dbuf *buffer; struct pcm_channel *channel; struct sc_pcminfo *parent; /* Channel information */ uint32_t dir; uint32_t format; /* Flags */ uint32_t run; }; /* PCM device private data */ struct sc_pcminfo { device_t dev; uint32_t (*ih)(struct sc_pcminfo *scp); uint32_t chnum; struct sc_chinfo chan[SSI_NCHANNELS]; struct sc_info *sc; }; static struct resource_spec ssi_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { -1, 0 } }; static int setup_dma(struct sc_pcminfo *scp); static void setup_ssi(struct sc_info *); static void ssi_configure_clock(struct sc_info *); /* * Mixer interface. */ static int ssimixer_init(struct snd_mixer *m) { struct sc_pcminfo *scp; struct sc_info *sc; int mask; scp = mix_getdevinfo(m); sc = scp->sc; if (sc == NULL) return -1; mask = SOUND_MASK_PCM; mask |= SOUND_MASK_VOLUME; snd_mtxlock(sc->lock); pcm_setflags(scp->dev, pcm_getflags(scp->dev) | SD_F_SOFTPCMVOL); mix_setdevs(m, mask); snd_mtxunlock(sc->lock); return (0); } static int ssimixer_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct sc_pcminfo *scp; scp = mix_getdevinfo(m); /* Here we can configure hardware volume on our DAC */ #if 1 device_printf(scp->dev, "ssimixer_set() %d %d\n", left, right); #endif return (0); } static kobj_method_t ssimixer_methods[] = { KOBJMETHOD(mixer_init, ssimixer_init), KOBJMETHOD(mixer_set, ssimixer_set), KOBJMETHOD_END }; MIXER_DECLARE(ssimixer); /* * Channel interface. */ static void * ssichan_init(kobj_t obj, void *devinfo, struct snd_dbuf *b, struct pcm_channel *c, int dir) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct sc_info *sc; scp = (struct sc_pcminfo *)devinfo; sc = scp->sc; snd_mtxlock(sc->lock); ch = &scp->chan[0]; ch->dir = dir; ch->run = 0; ch->buffer = b; ch->channel = c; ch->parent = scp; snd_mtxunlock(sc->lock); if (sndbuf_setup(ch->buffer, sc->buf_base, sc->dma_size) != 0) { device_printf(scp->dev, "Can't setup sndbuf.\n"); return NULL; } return ch; } static int ssichan_free(kobj_t obj, void *data) { struct sc_chinfo *ch = data; struct sc_pcminfo *scp = ch->parent; struct sc_info *sc = scp->sc; #if 0 device_printf(scp->dev, "ssichan_free()\n"); #endif snd_mtxlock(sc->lock); /* TODO: free channel buffer */ snd_mtxunlock(sc->lock); return (0); } static int ssichan_setformat(kobj_t obj, void *data, uint32_t format) { struct sc_chinfo *ch = data; ch->format = format; return (0); } static uint32_t ssichan_setspeed(kobj_t obj, void *data, uint32_t speed) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct ssi_rate *sr; struct sc_info *sc; int threshold; int i; ch = data; scp = ch->parent; sc = scp->sc; sr = NULL; /* First look for equal frequency. */ for (i = 0; rate_map[i].speed != 0; i++) { if (rate_map[i].speed == speed) sr = &rate_map[i]; } /* If no match, just find nearest. */ if (sr == NULL) { for (i = 0; rate_map[i].speed != 0; i++) { sr = &rate_map[i]; threshold = sr->speed + ((rate_map[i + 1].speed != 0) ? ((rate_map[i + 1].speed - sr->speed) >> 1) : 0); if (speed < threshold) break; } } sc->sr = sr; ssi_configure_clock(sc); return (sr->speed); } static void ssi_configure_clock(struct sc_info *sc) { struct ssi_rate *sr; sr = sc->sr; pll4_configure_output(sr->mfi, sr->mfn, sr->mfd); /* Configure other dividers here, if any */ } static uint32_t ssichan_setblocksize(kobj_t obj, void *data, uint32_t blocksize) { struct sc_chinfo *ch = data; struct sc_pcminfo *scp = ch->parent; struct sc_info *sc = scp->sc; sndbuf_resize(ch->buffer, sc->dma_size / blocksize, blocksize); setup_dma(scp); return (sndbuf_getblksz(ch->buffer)); } uint32_t ssi_dma_intr(void *arg, int chn) { struct sc_pcminfo *scp; struct sdma_conf *conf; struct sc_chinfo *ch; struct sc_info *sc; int bufsize; scp = arg; ch = &scp->chan[0]; sc = scp->sc; conf = sc->conf; bufsize = sndbuf_getsize(ch->buffer); sc->pos += conf->period; if (sc->pos >= bufsize) sc->pos -= bufsize; if (ch->run) chn_intr(ch->channel); return (0); } static int find_sdma_controller(struct sc_info *sc) { struct sdma_softc *sdma_sc; phandle_t node, sdma_node; device_t sdma_dev; int dts_value[8]; int len; if ((node = ofw_bus_get_node(sc->dev)) == -1) return (ENXIO); if ((len = OF_getproplen(node, "dmas")) <= 0) return (ENXIO); OF_getprop(node, "dmas", &dts_value, len); sc->sdma_ev_rx = fdt32_to_cpu(dts_value[1]); sc->sdma_ev_tx = fdt32_to_cpu(dts_value[5]); sdma_node = OF_node_from_xref(fdt32_to_cpu(dts_value[0])); sdma_sc = NULL; sdma_dev = devclass_get_device(devclass_find("sdma"), 0); if (sdma_dev) sdma_sc = device_get_softc(sdma_dev); if (sdma_sc == NULL) { device_printf(sc->dev, "No sDMA found. Can't operate\n"); return (ENXIO); } sc->sdma_sc = sdma_sc; return (0); }; static int setup_dma(struct sc_pcminfo *scp) { struct sdma_conf *conf; struct sc_chinfo *ch; struct sc_info *sc; int fmt; ch = &scp->chan[0]; sc = scp->sc; conf = sc->conf; conf->ih = ssi_dma_intr; conf->ih_user = scp; conf->saddr = sc->buf_base_phys; conf->daddr = rman_get_start(sc->res[0]) + SSI_STX0; conf->event = sc->sdma_ev_tx; /* SDMA TX event */ conf->period = sndbuf_getblksz(ch->buffer); conf->num_bd = sndbuf_getblkcnt(ch->buffer); /* * Word Length * Can be 32, 24, 16 or 8 for sDMA. * * SSI supports 24 at max. */ fmt = sndbuf_getfmt(ch->buffer); if (fmt & AFMT_16BIT) { conf->word_length = 16; conf->command = CMD_2BYTES; } else if (fmt & AFMT_24BIT) { conf->word_length = 24; conf->command = CMD_3BYTES; } else { device_printf(sc->dev, "Unknown format\n"); return (-1); } return (0); } static int ssi_start(struct sc_pcminfo *scp) { struct sc_info *sc; int reg; sc = scp->sc; if (sdma_configure(sc->sdma_channel, sc->conf) != 0) { device_printf(sc->dev, "Can't configure sDMA\n"); return (-1); } /* Enable DMA interrupt */ reg = (SIER_TDMAE); WRITE4(sc, SSI_SIER, reg); sdma_start(sc->sdma_channel); return (0); } static int ssi_stop(struct sc_pcminfo *scp) { struct sc_info *sc; int reg; sc = scp->sc; reg = READ4(sc, SSI_SIER); reg &= ~(SIER_TDMAE); WRITE4(sc, SSI_SIER, reg); sdma_stop(sc->sdma_channel); bzero(sc->buf_base, sc->dma_size); return (0); } static int ssichan_trigger(kobj_t obj, void *data, int go) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct sc_info *sc; ch = data; scp = ch->parent; sc = scp->sc; snd_mtxlock(sc->lock); switch (go) { case PCMTRIG_START: #if 0 device_printf(scp->dev, "trigger start\n"); #endif ch->run = 1; ssi_start(scp); break; case PCMTRIG_STOP: case PCMTRIG_ABORT: #if 0 device_printf(scp->dev, "trigger stop or abort\n"); #endif ch->run = 0; ssi_stop(scp); break; } snd_mtxunlock(sc->lock); return (0); } static uint32_t ssichan_getptr(kobj_t obj, void *data) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct sc_info *sc; ch = data; scp = ch->parent; sc = scp->sc; return (sc->pos); } static uint32_t ssi_pfmt[] = { SND_FORMAT(AFMT_S24_LE, 2, 0), 0 }; static struct pcmchan_caps ssi_pcaps = {44100, 192000, ssi_pfmt, 0}; static struct pcmchan_caps * ssichan_getcaps(kobj_t obj, void *data) { return (&ssi_pcaps); } static kobj_method_t ssichan_methods[] = { KOBJMETHOD(channel_init, ssichan_init), KOBJMETHOD(channel_free, ssichan_free), KOBJMETHOD(channel_setformat, ssichan_setformat), KOBJMETHOD(channel_setspeed, ssichan_setspeed), KOBJMETHOD(channel_setblocksize, ssichan_setblocksize), KOBJMETHOD(channel_trigger, ssichan_trigger), KOBJMETHOD(channel_getptr, ssichan_getptr), KOBJMETHOD(channel_getcaps, ssichan_getcaps), KOBJMETHOD_END }; CHANNEL_DECLARE(ssichan); static int ssi_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "fsl,imx6q-ssi")) return (ENXIO); device_set_desc(dev, "i.MX6 Synchronous Serial Interface (SSI)"); return (BUS_PROBE_DEFAULT); } static void ssi_intr(void *arg) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct sc_info *sc; scp = arg; sc = scp->sc; ch = &scp->chan[0]; /* We don't use SSI interrupt */ #if 0 device_printf(sc->dev, "SSI Intr 0x%08x\n", READ4(sc, SSI_SISR)); #endif } static void setup_ssi(struct sc_info *sc) { int reg; reg = READ4(sc, SSI_STCCR); reg &= ~(WL3_WL0_M << WL3_WL0_S); reg |= (0xb << WL3_WL0_S); /* 24 bit */ reg &= ~(DC4_DC0_M << DC4_DC0_S); reg |= (1 << DC4_DC0_S); /* 2 words per frame */ reg &= ~(STCCR_DIV2); /* Divide by 1 */ reg &= ~(STCCR_PSR); /* Divide by 1 */ reg &= ~(PM7_PM0_M << PM7_PM0_S); reg |= (1 << PM7_PM0_S); /* Divide by 2 */ WRITE4(sc, SSI_STCCR, reg); reg = READ4(sc, SSI_SFCSR); reg &= ~(SFCSR_TFWM0_M << SFCSR_TFWM0_S); reg |= (8 << SFCSR_TFWM0_S); /* empty slots */ WRITE4(sc, SSI_SFCSR, reg); reg = READ4(sc, SSI_STCR); reg |= (STCR_TFEN0); reg &= ~(STCR_TFEN1); reg &= ~(STCR_TSHFD); /* MSB */ reg |= (STCR_TXBIT0); reg |= (STCR_TXDIR | STCR_TFDIR); reg |= (STCR_TSCKP); /* falling edge */ reg |= (STCR_TFSI); reg &= ~(STCR_TFSI); /* active high frame sync */ reg &= ~(STCR_TFSL); reg |= STCR_TEFS; WRITE4(sc, SSI_STCR, reg); reg = READ4(sc, SSI_SCR); reg &= ~(SCR_I2S_MODE_M << SCR_I2S_MODE_S); /* Not master */ reg |= (SCR_SSIEN | SCR_TE); reg |= (SCR_NET); reg |= (SCR_SYN); WRITE4(sc, SSI_SCR, reg); } static void ssi_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) { bus_addr_t *addr; if (err) return; addr = (bus_addr_t*)arg; *addr = segs[0].ds_addr; } static int ssi_attach(device_t dev) { char status[SND_STATUSLEN]; struct sc_pcminfo *scp; struct sc_info *sc; int err; sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK | M_ZERO); sc->dev = dev; sc->sr = &rate_map[0]; sc->pos = 0; sc->conf = malloc(sizeof(struct sdma_conf), M_DEVBUF, M_WAITOK | M_ZERO); sc->lock = snd_mtxcreate(device_get_nameunit(dev), "ssi softc"); if (sc->lock == NULL) { - device_printf(dev, "Cant create mtx\n"); + device_printf(dev, "Can't create mtx\n"); return (ENXIO); } if (bus_alloc_resources(dev, ssi_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* Memory interface */ sc->bst = rman_get_bustag(sc->res[0]); sc->bsh = rman_get_bushandle(sc->res[0]); /* SDMA */ if (find_sdma_controller(sc)) { device_printf(dev, "could not find active SDMA\n"); return (ENXIO); } /* Setup PCM */ scp = malloc(sizeof(struct sc_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO); scp->sc = sc; scp->dev = dev; /* * Maximum possible DMA buffer. - * Will be used partialy to match 24 bit word. + * Will be used partially to match 24 bit word. */ sc->dma_size = 131072; /* * Must use dma_size boundary as modulo feature required. * Modulo feature allows setup circular buffer. */ err = bus_dma_tag_create( bus_get_dma_tag(sc->dev), 4, sc->dma_size, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ sc->dma_size, 1, /* maxsize, nsegments */ sc->dma_size, 0, /* maxsegsize, flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->dma_tag); err = bus_dmamem_alloc(sc->dma_tag, (void **)&sc->buf_base, BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->dma_map); if (err) { device_printf(dev, "cannot allocate framebuffer\n"); return (ENXIO); } err = bus_dmamap_load(sc->dma_tag, sc->dma_map, sc->buf_base, sc->dma_size, ssi_dmamap_cb, &sc->buf_base_phys, BUS_DMA_NOWAIT); if (err) { device_printf(dev, "cannot load DMA map\n"); return (ENXIO); } bzero(sc->buf_base, sc->dma_size); /* Setup interrupt handler */ err = bus_setup_intr(dev, sc->res[1], INTR_MPSAFE | INTR_TYPE_AV, NULL, ssi_intr, scp, &sc->ih); if (err) { device_printf(dev, "Unable to alloc interrupt resource.\n"); return (ENXIO); } pcm_setflags(dev, pcm_getflags(dev) | SD_F_MPSAFE); err = pcm_register(dev, scp, 1, 0); if (err) { device_printf(dev, "Can't register pcm.\n"); return (ENXIO); } scp->chnum = 0; pcm_addchan(dev, PCMDIR_PLAY, &ssichan_class, scp); scp->chnum++; snprintf(status, SND_STATUSLEN, "at simplebus"); pcm_setstatus(dev, status); mixer_init(dev, &ssimixer_class, scp); setup_ssi(sc); imx_ccm_ssi_configure(dev); sc->sdma_channel = sdma_alloc(); if (sc->sdma_channel < 0) { device_printf(sc->dev, "Can't get sDMA channel\n"); return (1); } return (0); } static device_method_t ssi_pcm_methods[] = { DEVMETHOD(device_probe, ssi_probe), DEVMETHOD(device_attach, ssi_attach), { 0, 0 } }; static driver_t ssi_pcm_driver = { "pcm", ssi_pcm_methods, PCM_SOFTC_SIZE, }; DRIVER_MODULE(ssi, simplebus, ssi_pcm_driver, pcm_devclass, 0, 0); MODULE_DEPEND(ssi, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER); MODULE_VERSION(ssi, 1); Index: head/sys/arm/freescale/vybrid/vf_uart.c =================================================================== --- head/sys/arm/freescale/vybrid/vf_uart.c (revision 299068) +++ head/sys/arm/freescale/vybrid/vf_uart.c (revision 299069) @@ -1,516 +1,516 @@ /*- * Copyright (c) 2013 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Vybrid Family Universal Asynchronous Receiver/Transmitter * Chapter 49, Vybrid Reference Manual, Rev. 5, 07/2013 */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include "uart_if.h" #define UART_BDH 0x00 /* Baud Rate Registers: High */ #define UART_BDL 0x01 /* Baud Rate Registers: Low */ #define UART_C1 0x02 /* Control Register 1 */ #define UART_C2 0x03 /* Control Register 2 */ #define UART_S1 0x04 /* Status Register 1 */ #define UART_S2 0x05 /* Status Register 2 */ #define UART_C3 0x06 /* Control Register 3 */ #define UART_D 0x07 /* Data Register */ #define UART_MA1 0x08 /* Match Address Registers 1 */ #define UART_MA2 0x09 /* Match Address Registers 2 */ #define UART_C4 0x0A /* Control Register 4 */ #define UART_C5 0x0B /* Control Register 5 */ #define UART_ED 0x0C /* Extended Data Register */ #define UART_MODEM 0x0D /* Modem Register */ #define UART_IR 0x0E /* Infrared Register */ #define UART_PFIFO 0x10 /* FIFO Parameters */ #define UART_CFIFO 0x11 /* FIFO Control Register */ #define UART_SFIFO 0x12 /* FIFO Status Register */ #define UART_TWFIFO 0x13 /* FIFO Transmit Watermark */ #define UART_TCFIFO 0x14 /* FIFO Transmit Count */ #define UART_RWFIFO 0x15 /* FIFO Receive Watermark */ #define UART_RCFIFO 0x16 /* FIFO Receive Count */ #define UART_C7816 0x18 /* 7816 Control Register */ #define UART_IE7816 0x19 /* 7816 Interrupt Enable Register */ #define UART_IS7816 0x1A /* 7816 Interrupt Status Register */ #define UART_WP7816T0 0x1B /* 7816 Wait Parameter Register */ #define UART_WP7816T1 0x1B /* 7816 Wait Parameter Register */ #define UART_WN7816 0x1C /* 7816 Wait N Register */ #define UART_WF7816 0x1D /* 7816 Wait FD Register */ #define UART_ET7816 0x1E /* 7816 Error Threshold Register */ #define UART_TL7816 0x1F /* 7816 Transmit Length Register */ #define UART_C6 0x21 /* CEA709.1-B Control Register 6 */ #define UART_PCTH 0x22 /* CEA709.1-B Packet Cycle Time Counter High */ #define UART_PCTL 0x23 /* CEA709.1-B Packet Cycle Time Counter Low */ #define UART_B1T 0x24 /* CEA709.1-B Beta1 Timer */ #define UART_SDTH 0x25 /* CEA709.1-B Secondary Delay Timer High */ #define UART_SDTL 0x26 /* CEA709.1-B Secondary Delay Timer Low */ #define UART_PRE 0x27 /* CEA709.1-B Preamble */ #define UART_TPL 0x28 /* CEA709.1-B Transmit Packet Length */ #define UART_IE 0x29 /* CEA709.1-B Interrupt Enable Register */ #define UART_WB 0x2A /* CEA709.1-B WBASE */ #define UART_S3 0x2B /* CEA709.1-B Status Register */ #define UART_S4 0x2C /* CEA709.1-B Status Register */ #define UART_RPL 0x2D /* CEA709.1-B Received Packet Length */ #define UART_RPREL 0x2E /* CEA709.1-B Received Preamble Length */ #define UART_CPW 0x2F /* CEA709.1-B Collision Pulse Width */ #define UART_RIDT 0x30 /* CEA709.1-B Receive Indeterminate Time */ #define UART_TIDT 0x31 /* CEA709.1-B Transmit Indeterminate Time */ #define UART_C2_TE (1 << 3) /* Transmitter Enable */ #define UART_C2_TIE (1 << 7) /* Transmitter Interrupt Enable */ #define UART_C2_RE (1 << 2) /* Receiver Enable */ #define UART_C2_RIE (1 << 5) /* Receiver Interrupt Enable */ #define UART_S1_TDRE (1 << 7) /* Transmit Data Register Empty Flag */ #define UART_S1_RDRF (1 << 5) /* Receive Data Register Full Flag */ #define UART_S2_LBKDIF (1 << 7) /* LIN Break Detect Interrupt Flag */ #define UART_C4_BRFA 0x1f /* Baud Rate Fine Adjust */ #define UART_BDH_SBR 0x1f /* UART Baud Rate Bits */ /* * Low-level UART interface. */ static int vf_uart_probe(struct uart_bas *bas); static void vf_uart_init(struct uart_bas *bas, int, int, int, int); static void vf_uart_term(struct uart_bas *bas); static void vf_uart_putc(struct uart_bas *bas, int); static int vf_uart_rxready(struct uart_bas *bas); static int vf_uart_getc(struct uart_bas *bas, struct mtx *); void uart_reinit(struct uart_softc *,int,int); static struct uart_ops uart_vybrid_ops = { .probe = vf_uart_probe, .init = vf_uart_init, .term = vf_uart_term, .putc = vf_uart_putc, .rxready = vf_uart_rxready, .getc = vf_uart_getc, }; static int vf_uart_probe(struct uart_bas *bas) { return (0); } static void vf_uart_init(struct uart_bas *bas, int baudrate, int databits, int stopbits, int parity) { } static void vf_uart_term(struct uart_bas *bas) { } static void vf_uart_putc(struct uart_bas *bas, int c) { while (!(uart_getreg(bas, UART_S1) & UART_S1_TDRE)) ; uart_setreg(bas, UART_D, c); } static int vf_uart_rxready(struct uart_bas *bas) { int usr1; usr1 = uart_getreg(bas, UART_S1); if (usr1 & UART_S1_RDRF) { return (1); } return (0); } static int vf_uart_getc(struct uart_bas *bas, struct mtx *hwmtx) { int c; uart_lock(hwmtx); while (!(uart_getreg(bas, UART_S1) & UART_S1_RDRF)) ; c = uart_getreg(bas, UART_D); uart_unlock(hwmtx); return (c & 0xff); } /* * High-level UART interface. */ struct vf_uart_softc { struct uart_softc base; }; void uart_reinit(struct uart_softc *sc, int clkspeed, int baud) { struct uart_bas *bas; int sbr; int brfa; int reg; bas = &sc->sc_bas; if (!bas) { - printf("Error: cant reconfigure bas\n"); + printf("Error: can't reconfigure bas\n"); return; } uart_setreg(bas, UART_MODEM, 0x00); /* * Disable transmitter and receiver * for a while. */ reg = uart_getreg(bas, UART_C2); reg &= ~(UART_C2_RE | UART_C2_TE); uart_setreg(bas, UART_C2, 0x00); uart_setreg(bas, UART_C1, 0x00); sbr = (uint16_t) (clkspeed / (baud * 16)); brfa = (clkspeed / baud) - (sbr * 16); reg = uart_getreg(bas, UART_BDH); reg &= ~UART_BDH_SBR; reg |= ((sbr & 0x1f00) >> 8); uart_setreg(bas, UART_BDH, reg); reg = sbr & 0x00ff; uart_setreg(bas, UART_BDL, reg); reg = uart_getreg(bas, UART_C4); reg &= ~UART_C4_BRFA; reg |= (brfa & UART_C4_BRFA); uart_setreg(bas, UART_C4, reg); reg = uart_getreg(bas, UART_C2); reg |= (UART_C2_RE | UART_C2_TE); uart_setreg(bas, UART_C2, reg); } static int vf_uart_bus_attach(struct uart_softc *); static int vf_uart_bus_detach(struct uart_softc *); static int vf_uart_bus_flush(struct uart_softc *, int); static int vf_uart_bus_getsig(struct uart_softc *); static int vf_uart_bus_ioctl(struct uart_softc *, int, intptr_t); static int vf_uart_bus_ipend(struct uart_softc *); static int vf_uart_bus_param(struct uart_softc *, int, int, int, int); static int vf_uart_bus_probe(struct uart_softc *); static int vf_uart_bus_receive(struct uart_softc *); static int vf_uart_bus_setsig(struct uart_softc *, int); static int vf_uart_bus_transmit(struct uart_softc *); static kobj_method_t vf_uart_methods[] = { KOBJMETHOD(uart_attach, vf_uart_bus_attach), KOBJMETHOD(uart_detach, vf_uart_bus_detach), KOBJMETHOD(uart_flush, vf_uart_bus_flush), KOBJMETHOD(uart_getsig, vf_uart_bus_getsig), KOBJMETHOD(uart_ioctl, vf_uart_bus_ioctl), KOBJMETHOD(uart_ipend, vf_uart_bus_ipend), KOBJMETHOD(uart_param, vf_uart_bus_param), KOBJMETHOD(uart_probe, vf_uart_bus_probe), KOBJMETHOD(uart_receive, vf_uart_bus_receive), KOBJMETHOD(uart_setsig, vf_uart_bus_setsig), KOBJMETHOD(uart_transmit, vf_uart_bus_transmit), { 0, 0 } }; static struct uart_class uart_vybrid_class = { "vybrid", vf_uart_methods, sizeof(struct vf_uart_softc), .uc_ops = &uart_vybrid_ops, .uc_range = 0x100, .uc_rclk = 24000000, /* TODO: get value from CCM */ .uc_rshift = 0 }; static struct ofw_compat_data compat_data[] = { {"fsl,mvf600-uart", (uintptr_t)&uart_vybrid_class}, {NULL, (uintptr_t)NULL}, }; UART_FDT_CLASS_AND_DEVICE(compat_data); static int vf_uart_bus_attach(struct uart_softc *sc) { struct uart_bas *bas; int reg; bas = &sc->sc_bas; sc->sc_hwiflow = 0; sc->sc_hwoflow = 0; uart_reinit(sc, 66000000, 115200); reg = uart_getreg(bas, UART_C2); if (sc->sc_sysdev != NULL && sc->sc_sysdev->type == UART_DEV_CONSOLE) { reg &= ~UART_C2_RIE; } else { reg |= UART_C2_RIE; } uart_setreg(bas, UART_C2, reg); return (0); } static int vf_uart_bus_detach(struct uart_softc *sc) { /* TODO */ return (0); } static int vf_uart_bus_flush(struct uart_softc *sc, int what) { /* TODO */ return (0); } static int vf_uart_bus_getsig(struct uart_softc *sc) { /* TODO */ return (0); } static int vf_uart_bus_ioctl(struct uart_softc *sc, int request, intptr_t data) { struct uart_bas *bas; int error; bas = &sc->sc_bas; error = 0; uart_lock(sc->sc_hwmtx); switch (request) { case UART_IOCTL_BREAK: /* TODO */ break; case UART_IOCTL_BAUD: /* TODO */ *(int*)data = 115200; break; default: error = EINVAL; break; } uart_unlock(sc->sc_hwmtx); return (error); } static int vf_uart_bus_ipend(struct uart_softc *sc) { struct uart_bas *bas; int ipend; uint32_t usr1, usr2; int reg; int sfifo; bas = &sc->sc_bas; ipend = 0; uart_lock(sc->sc_hwmtx); usr1 = uart_getreg(bas, UART_S1); usr2 = uart_getreg(bas, UART_S2); sfifo = uart_getreg(bas, UART_SFIFO); /* ack usr2 */ uart_setreg(bas, UART_S2, usr2); if (usr1 & UART_S1_TDRE) { reg = uart_getreg(bas, UART_C2); reg &= ~(UART_C2_TIE); uart_setreg(bas, UART_C2, reg); if (sc->sc_txbusy != 0) { ipend |= SER_INT_TXIDLE; } } if (usr1 & UART_S1_RDRF) { reg = uart_getreg(bas, UART_C2); reg &= ~(UART_C2_RIE); uart_setreg(bas, UART_C2, reg); ipend |= SER_INT_RXREADY; } if (usr2 & UART_S2_LBKDIF) { ipend |= SER_INT_BREAK; } uart_unlock(sc->sc_hwmtx); return (ipend); } static int vf_uart_bus_param(struct uart_softc *sc, int baudrate, int databits, int stopbits, int parity) { uart_lock(sc->sc_hwmtx); vf_uart_init(&sc->sc_bas, baudrate, databits, stopbits, parity); uart_unlock(sc->sc_hwmtx); return (0); } static int vf_uart_bus_probe(struct uart_softc *sc) { int error; error = vf_uart_probe(&sc->sc_bas); if (error) return (error); sc->sc_rxfifosz = 1; sc->sc_txfifosz = 1; device_set_desc(sc->sc_dev, "Vybrid Family UART"); return (0); } static int vf_uart_bus_receive(struct uart_softc *sc) { struct uart_bas *bas; int reg; int c; bas = &sc->sc_bas; uart_lock(sc->sc_hwmtx); /* Read FIFO */ while (uart_getreg(bas, UART_S1) & UART_S1_RDRF) { if (uart_rx_full(sc)) { /* No space left in input buffer */ sc->sc_rxbuf[sc->sc_rxput] = UART_STAT_OVERRUN; break; } c = uart_getreg(bas, UART_D); uart_rx_put(sc, c); } /* Reenable Data Ready interrupt */ reg = uart_getreg(bas, UART_C2); reg |= (UART_C2_RIE); uart_setreg(bas, UART_C2, reg); uart_unlock(sc->sc_hwmtx); return (0); } static int vf_uart_bus_setsig(struct uart_softc *sc, int sig) { struct uart_bas *bas; int reg; /* TODO: implement (?) */ /* XXX workaround to have working console on mount prompt */ /* Enable RX interrupt */ bas = &sc->sc_bas; if (sc->sc_sysdev != NULL && sc->sc_sysdev->type == UART_DEV_CONSOLE) { reg = uart_getreg(bas, UART_C2); reg |= (UART_C2_RIE); uart_setreg(bas, UART_C2, reg); } return (0); } static int vf_uart_bus_transmit(struct uart_softc *sc) { struct uart_bas *bas = &sc->sc_bas; int i; int reg; bas = &sc->sc_bas; uart_lock(sc->sc_hwmtx); /* Fill TX FIFO */ for (i = 0; i < sc->sc_txdatasz; i++) { uart_setreg(bas, UART_D, sc->sc_txbuf[i] & 0xff); uart_barrier(&sc->sc_bas); } sc->sc_txbusy = 1; /* Call me when ready */ reg = uart_getreg(bas, UART_C2); reg |= (UART_C2_TIE); uart_setreg(bas, UART_C2, reg); uart_unlock(sc->sc_hwmtx); return (0); } Index: head/sys/arm/mv/mpic.c =================================================================== --- head/sys/arm/mv/mpic.c (revision 299068) +++ head/sys/arm/mv/mpic.c (revision 299069) @@ -1,631 +1,631 @@ /*- * Copyright (c) 2006 Benno Rice. * Copyright (C) 2007-2011 MARVELL INTERNATIONAL LTD. * Copyright (c) 2012 Semihalf. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * from: FreeBSD: //depot/projects/arm/src/sys/arm/xscale/pxa2x0/pxa2x0_icu.c, rev 1 * from: FreeBSD: src/sys/arm/mv/ic.c,v 1.5 2011/02/08 01:49:30 */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INTRNG #include "pic_if.h" #endif #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif #define MPIC_INT_ERR 4 #define MPIC_INT_MSI 96 #define MPIC_IRQ_MASK 0x3ff #define MPIC_CTRL 0x0 #define MPIC_SOFT_INT 0x4 #define MPIC_SOFT_INT_DRBL1 (1 << 5) #define MPIC_ERR_CAUSE 0x20 #define MPIC_ISE 0x30 #define MPIC_ICE 0x34 #define MPIC_INT_CTL(irq) (0x100 + (irq)*4) #define MPIC_INT_IRQ_FIQ_MASK(cpuid) (0x101 << (cpuid)) #define MPIC_CTRL_NIRQS(ctrl) (((ctrl) >> 2) & 0x3ff) #define MPIC_IN_DRBL 0x08 #define MPIC_IN_DRBL_MASK 0x0c #define MPIC_PPI_CAUSE 0x10 #define MPIC_CTP 0x40 #define MPIC_IIACK 0x44 #define MPIC_ISM 0x48 #define MPIC_ICM 0x4c #define MPIC_ERR_MASK 0xe50 #define MPIC_PPI 32 #ifdef INTRNG struct mv_mpic_irqsrc { struct intr_irqsrc mmi_isrc; u_int mmi_irq; }; #endif struct mv_mpic_softc { device_t sc_dev; struct resource * mpic_res[4]; bus_space_tag_t mpic_bst; bus_space_handle_t mpic_bsh; bus_space_tag_t cpu_bst; bus_space_handle_t cpu_bsh; bus_space_tag_t drbl_bst; bus_space_handle_t drbl_bsh; struct mtx mtx; #ifdef INTRNG struct mv_mpic_irqsrc * mpic_isrcs; #endif int nirqs; void * intr_hand; }; static struct resource_spec mv_mpic_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_MEMORY, 1, RF_ACTIVE }, { SYS_RES_MEMORY, 2, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, { -1, 0 } }; static struct ofw_compat_data compat_data[] = { {"mrvl,mpic", true}, {"marvell,mpic", true}, {NULL, false} }; static struct mv_mpic_softc *mv_mpic_sc = NULL; void mpic_send_ipi(int cpus, u_int ipi); static int mv_mpic_probe(device_t); static int mv_mpic_attach(device_t); uint32_t mv_mpic_get_cause(void); uint32_t mv_mpic_get_cause_err(void); uint32_t mv_mpic_get_msi(void); static void mpic_unmask_irq(uintptr_t nb); static void mpic_mask_irq(uintptr_t nb); static void mpic_mask_irq_err(uintptr_t nb); static void mpic_unmask_irq_err(uintptr_t nb); static int mpic_intr(void *arg); static void mpic_unmask_msi(void); #ifndef INTRNG static void arm_mask_irq_err(uintptr_t); static void arm_unmask_irq_err(uintptr_t); #endif #define MPIC_WRITE(softc, reg, val) \ bus_space_write_4((softc)->mpic_bst, (softc)->mpic_bsh, (reg), (val)) #define MPIC_READ(softc, reg) \ bus_space_read_4((softc)->mpic_bst, (softc)->mpic_bsh, (reg)) #define MPIC_CPU_WRITE(softc, reg, val) \ bus_space_write_4((softc)->cpu_bst, (softc)->cpu_bsh, (reg), (val)) #define MPIC_CPU_READ(softc, reg) \ bus_space_read_4((softc)->cpu_bst, (softc)->cpu_bsh, (reg)) #define MPIC_DRBL_WRITE(softc, reg, val) \ bus_space_write_4((softc)->drbl_bst, (softc)->drbl_bsh, (reg), (val)) #define MPIC_DRBL_READ(softc, reg) \ bus_space_read_4((softc)->drbl_bst, (softc)->drbl_bsh, (reg)) static int mv_mpic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "Marvell Integrated Interrupt Controller"); return (0); } #ifdef INTRNG static int mv_mpic_register_isrcs(struct mv_mpic_softc *sc) { int error; uint32_t irq; struct intr_irqsrc *isrc; const char *name; sc->mpic_isrcs = malloc(sc->nirqs * sizeof (*sc->mpic_isrcs), M_DEVBUF, M_WAITOK | M_ZERO); name = device_get_nameunit(sc->sc_dev); for (irq = 0; irq < sc->nirqs; irq++) { sc->mpic_isrcs[irq].mmi_irq = irq; isrc = &sc->mpic_isrcs[irq].mmi_isrc; if (irq < MPIC_PPI) { error = intr_isrc_register(isrc, sc->sc_dev, INTR_ISRCF_PPI, "%s", name); } else { error = intr_isrc_register(isrc, sc->sc_dev, 0, "%s", name); } if (error != 0) { /* XXX call intr_isrc_deregister() */ device_printf(sc->sc_dev, "%s failed", __func__); return (error); } } return (0); } #endif static int mv_mpic_attach(device_t dev) { struct mv_mpic_softc *sc; int error; uint32_t val; sc = (struct mv_mpic_softc *)device_get_softc(dev); if (mv_mpic_sc != NULL) return (ENXIO); mv_mpic_sc = sc; sc->sc_dev = dev; mtx_init(&sc->mtx, "MPIC lock", NULL, MTX_SPIN); error = bus_alloc_resources(dev, mv_mpic_spec, sc->mpic_res); if (error) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } #ifdef INTRNG if (sc->mpic_res[3] == NULL) device_printf(dev, "No interrupt to use.\n"); else bus_setup_intr(dev, sc->mpic_res[3], INTR_TYPE_CLK, mpic_intr, NULL, sc, &sc->intr_hand); #endif sc->mpic_bst = rman_get_bustag(sc->mpic_res[0]); sc->mpic_bsh = rman_get_bushandle(sc->mpic_res[0]); sc->cpu_bst = rman_get_bustag(sc->mpic_res[1]); sc->cpu_bsh = rman_get_bushandle(sc->mpic_res[1]); if (sc->mpic_res[2] != NULL) { /* This is required only if MSIs are used. */ sc->drbl_bst = rman_get_bustag(sc->mpic_res[2]); sc->drbl_bsh = rman_get_bushandle(sc->mpic_res[2]); } bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_CTRL, 1); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_CTP, 0); val = MPIC_READ(mv_mpic_sc, MPIC_CTRL); sc->nirqs = MPIC_CTRL_NIRQS(val); #ifdef INTRNG if (mv_mpic_register_isrcs(sc) != 0) { device_printf(dev, "could not register PIC ISRCs\n"); bus_release_resources(dev, mv_mpic_spec, sc->mpic_res); return (ENXIO); } if (intr_pic_register(dev, OF_xref_from_device(dev)) != 0) { device_printf(dev, "could not register PIC\n"); bus_release_resources(dev, mv_mpic_spec, sc->mpic_res); return (ENXIO); } #endif mpic_unmask_msi(); return (0); } #ifdef INTRNG static int mpic_intr(void *arg) { struct mv_mpic_softc *sc; uint32_t cause, irqsrc; unsigned int irq; u_int cpuid; sc = arg; cpuid = PCPU_GET(cpuid); irq = 0; for (cause = MPIC_CPU_READ(sc, MPIC_PPI_CAUSE); cause > 0; cause >>= 1, irq++) { if (cause & 1) { irqsrc = MPIC_READ(sc, MPIC_INT_CTL(irq)); if ((irqsrc & MPIC_INT_IRQ_FIQ_MASK(cpuid)) == 0) continue; if (intr_isrc_dispatch(&sc->mpic_isrcs[irq].mmi_isrc, curthread->td_intr_frame) != 0) { mpic_mask_irq(irq); device_printf(sc->sc_dev, "Stray irq %u " "disabled\n", irq); } } } return (FILTER_HANDLED); } static void mpic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { u_int irq; irq = ((struct mv_mpic_irqsrc *)isrc)->mmi_irq; mpic_mask_irq(irq); } static void mpic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { u_int irq; irq = ((struct mv_mpic_irqsrc *)isrc)->mmi_irq; mpic_unmask_irq(irq); } static int mpic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct mv_mpic_softc *sc; sc = device_get_softc(dev); if (data->type != INTR_MAP_DATA_FDT || data->fdt.ncells !=1 || data->fdt.cells[0] >= sc->nirqs) return (EINVAL); *isrcp = &sc->mpic_isrcs[data->fdt.cells[0]].mmi_isrc; return (0); } static void mpic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { mpic_disable_intr(dev, isrc); } static void mpic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { mpic_enable_intr(dev, isrc); } #endif static device_method_t mv_mpic_methods[] = { DEVMETHOD(device_probe, mv_mpic_probe), DEVMETHOD(device_attach, mv_mpic_attach), #ifdef INTRNG DEVMETHOD(pic_disable_intr, mpic_disable_intr), DEVMETHOD(pic_enable_intr, mpic_enable_intr), DEVMETHOD(pic_map_intr, mpic_map_intr), DEVMETHOD(pic_post_ithread, mpic_post_ithread), DEVMETHOD(pic_pre_ithread, mpic_pre_ithread), #endif { 0, 0 } }; static driver_t mv_mpic_driver = { "mpic", mv_mpic_methods, sizeof(struct mv_mpic_softc), }; static devclass_t mv_mpic_devclass; EARLY_DRIVER_MODULE(mpic, simplebus, mv_mpic_driver, mv_mpic_devclass, 0, 0, BUS_PASS_INTERRUPT); #ifndef INTRNG int arm_get_next_irq(int last) { u_int irq, next = -1; irq = mv_mpic_get_cause() & MPIC_IRQ_MASK; CTR2(KTR_INTR, "%s: irq:%#x", __func__, irq); if (irq != MPIC_IRQ_MASK) { if (irq == MPIC_INT_ERR) irq = mv_mpic_get_cause_err(); if (irq == MPIC_INT_MSI) irq = mv_mpic_get_msi(); next = irq; } CTR3(KTR_INTR, "%s: last=%d, next=%d", __func__, last, next); return (next); } /* * XXX We can make arm_enable_irq to operate on ICE and then mask/unmask only * by ISM/ICM and remove access to ICE in masking operation */ void arm_mask_irq(uintptr_t nb) { mpic_mask_irq(nb); } static void arm_mask_irq_err(uintptr_t nb) { mpic_mask_irq_err(nb); } void arm_unmask_irq(uintptr_t nb) { mpic_unmask_irq(nb); } void arm_unmask_irq_err(uintptr_t nb) { mpic_unmask_irq_err(nb); } #endif static void mpic_unmask_msi(void) { mpic_unmask_irq(MPIC_INT_MSI); } static void mpic_unmask_irq_err(uintptr_t nb) { uint32_t mask; uint8_t bit_off; bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_ISE, MPIC_INT_ERR); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ICM, MPIC_INT_ERR); bit_off = nb - ERR_IRQ; mask = MPIC_CPU_READ(mv_mpic_sc, MPIC_ERR_MASK); mask |= (1 << bit_off); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ERR_MASK, mask); } static void mpic_mask_irq_err(uintptr_t nb) { uint32_t mask; uint8_t bit_off; bit_off = nb - ERR_IRQ; mask = MPIC_CPU_READ(mv_mpic_sc, MPIC_ERR_MASK); mask &= ~(1 << bit_off); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ERR_MASK, mask); } static void mpic_unmask_irq(uintptr_t nb) { if (nb < ERR_IRQ) { bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_ISE, nb); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ICM, nb); } else if (nb < MSI_IRQ) mpic_unmask_irq_err(nb); if (nb == 0) MPIC_CPU_WRITE(mv_mpic_sc, MPIC_IN_DRBL_MASK, 0xffffffff); } static void mpic_mask_irq(uintptr_t nb) { if (nb < ERR_IRQ) { bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_ICE, nb); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ISM, nb); } else if (nb < MSI_IRQ) mpic_mask_irq_err(nb); } uint32_t mv_mpic_get_cause(void) { return (MPIC_CPU_READ(mv_mpic_sc, MPIC_IIACK)); } uint32_t mv_mpic_get_cause_err(void) { uint32_t err_cause; uint8_t bit_off; err_cause = bus_space_read_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_ERR_CAUSE); if (err_cause) bit_off = ffs(err_cause) - 1; else return (-1); debugf("%s: irq:%x cause:%x\n", __func__, bit_off, err_cause); return (ERR_IRQ + bit_off); } uint32_t mv_mpic_get_msi(void) { uint32_t cause; uint8_t bit_off; KASSERT(mv_mpic_sc->drbl_bst != NULL, ("No doorbell in mv_mpic_get_msi")); cause = MPIC_DRBL_READ(mv_mpic_sc, 0); if (cause) bit_off = ffs(cause) - 1; else return (-1); debugf("%s: irq:%x cause:%x\n", __func__, bit_off, cause); cause &= ~(1 << bit_off); MPIC_DRBL_WRITE(mv_mpic_sc, 0, cause); return (MSI_IRQ + bit_off); } int mv_msi_data(int irq, uint64_t *addr, uint32_t *data) { u_long phys, base, size; phandle_t node; int error; node = ofw_bus_get_node(mv_mpic_sc->sc_dev); - /* Get physical addres of register space */ + /* Get physical address of register space */ error = fdt_get_range(OF_parent(node), 0, &phys, &size); if (error) { printf("%s: Cannot get register physical address, err:%d", __func__, error); return (error); } /* Get offset of MPIC register space */ error = fdt_regsize(node, &base, &size); if (error) { printf("%s: Cannot get MPIC register offset, err:%d", __func__, error); return (error); } *addr = phys + base + MPIC_SOFT_INT; *data = MPIC_SOFT_INT_DRBL1 | irq; return (0); } #if defined(SMP) && defined(SOC_MV_ARMADAXP) void intr_pic_init_secondary(void) { } void pic_ipi_send(cpuset_t cpus, u_int ipi) { uint32_t val, i; val = 0x00000000; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= (1 << (8 + i)); val |= ipi; bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_SOFT_INT, val); } int pic_ipi_read(int i __unused) { uint32_t val; int ipi; val = MPIC_CPU_READ(mv_mpic_sc, MPIC_IN_DRBL); if (val) { ipi = ffs(val) - 1; MPIC_CPU_WRITE(mv_mpic_sc, MPIC_IN_DRBL, ~(1 << ipi)); return (ipi); } return (0x3ff); } void pic_ipi_clear(int ipi) { } #endif Index: head/sys/arm/mv/mv_common.c =================================================================== --- head/sys/arm/mv/mv_common.c (revision 299068) +++ head/sys/arm/mv/mv_common.c (revision 299069) @@ -1,2334 +1,2334 @@ /*- * Copyright (C) 2008-2011 MARVELL INTERNATIONAL LTD. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of MARVELL nor the names of contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MALLOC_DEFINE(M_IDMA, "idma", "idma dma test memory"); #define IDMA_DEBUG #undef IDMA_DEBUG #define MAX_CPU_WIN 5 #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif #ifdef DEBUG #define MV_DUMP_WIN 1 #else #define MV_DUMP_WIN 0 #endif static int win_eth_can_remap(int i); #ifndef SOC_MV_FREY static int decode_win_cpu_valid(void); #endif static int decode_win_usb_valid(void); static int decode_win_usb3_valid(void); static int decode_win_eth_valid(void); static int decode_win_pcie_valid(void); static int decode_win_sata_valid(void); static int decode_win_idma_valid(void); static int decode_win_xor_valid(void); #ifndef SOC_MV_FREY static void decode_win_cpu_setup(void); #endif #ifdef SOC_MV_ARMADAXP static int decode_win_sdram_fixup(void); #endif static void decode_win_usb_setup(u_long); static void decode_win_usb3_setup(u_long); static void decode_win_eth_setup(u_long); static void decode_win_sata_setup(u_long); static void decode_win_idma_setup(u_long); static void decode_win_xor_setup(u_long); static void decode_win_usb_dump(u_long); static void decode_win_usb3_dump(u_long); static void decode_win_eth_dump(u_long base); static void decode_win_idma_dump(u_long base); static void decode_win_xor_dump(u_long base); static int fdt_get_ranges(const char *, void *, int, int *, int *); #ifdef SOC_MV_ARMADA38X int gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, int *trig, int *pol); #endif static int win_cpu_from_dt(void); static int fdt_win_setup(void); static uint32_t dev_mask = 0; static int cpu_wins_no = 0; static int eth_port = 0; static int usb_port = 0; static struct decode_win cpu_win_tbl[MAX_CPU_WIN]; const struct decode_win *cpu_wins = cpu_win_tbl; typedef void (*decode_win_setup_t)(u_long); typedef void (*dump_win_t)(u_long); struct soc_node_spec { const char *compat; decode_win_setup_t decode_handler; dump_win_t dump_handler; }; static struct soc_node_spec soc_nodes[] = { { "mrvl,ge", &decode_win_eth_setup, &decode_win_eth_dump }, { "mrvl,usb-ehci", &decode_win_usb_setup, &decode_win_usb_dump }, { "marvell,armada-380-xhci", &decode_win_usb3_setup, &decode_win_usb3_dump }, { "mrvl,sata", &decode_win_sata_setup, NULL }, { "mrvl,xor", &decode_win_xor_setup, &decode_win_xor_dump }, { "mrvl,idma", &decode_win_idma_setup, &decode_win_idma_dump }, { "mrvl,pcie", &decode_win_pcie_setup, NULL }, { NULL, NULL, NULL }, }; struct fdt_pm_mask_entry fdt_pm_mask_table[] = { { "mrvl,ge", CPU_PM_CTRL_GE(0) }, { "mrvl,ge", CPU_PM_CTRL_GE(1) }, { "mrvl,usb-ehci", CPU_PM_CTRL_USB(0) }, { "mrvl,usb-ehci", CPU_PM_CTRL_USB(1) }, { "mrvl,usb-ehci", CPU_PM_CTRL_USB(2) }, { "mrvl,xor", CPU_PM_CTRL_XOR }, { "mrvl,sata", CPU_PM_CTRL_SATA }, { NULL, 0 } }; static __inline int pm_is_disabled(uint32_t mask) { #if defined(SOC_MV_KIRKWOOD) return (soc_power_ctrl_get(mask) == mask); #else return (soc_power_ctrl_get(mask) == mask ? 0 : 1); #endif } /* * Disable device using power management register. * 1 - Device Power On * 0 - Device Power Off * Mask can be set in loader. * EXAMPLE: * loader> set hw.pm-disable-mask=0x2 * * Common mask: * |-------------------------------| * | Device | Kirkwood | Discovery | * |-------------------------------| * | USB0 | 0x00008 | 0x020000 | * |-------------------------------| * | USB1 | - | 0x040000 | * |-------------------------------| * | USB2 | - | 0x080000 | * |-------------------------------| * | GE0 | 0x00001 | 0x000002 | * |-------------------------------| * | GE1 | - | 0x000004 | * |-------------------------------| * | IDMA | - | 0x100000 | * |-------------------------------| * | XOR | 0x10000 | 0x200000 | * |-------------------------------| * | CESA | 0x20000 | 0x400000 | * |-------------------------------| * | SATA | 0x04000 | 0x004000 | * --------------------------------| * This feature can be used only on Kirkwood and Discovery * machines. */ static __inline void pm_disable_device(int mask) { #ifdef DIAGNOSTIC uint32_t reg; reg = soc_power_ctrl_get(CPU_PM_CTRL_ALL); printf("Power Management Register: 0%x\n", reg); reg &= ~mask; soc_power_ctrl_set(reg); printf("Device %x is disabled\n", mask); reg = soc_power_ctrl_get(CPU_PM_CTRL_ALL); printf("Power Management Register: 0%x\n", reg); #endif } int fdt_pm(phandle_t node) { uint32_t cpu_pm_ctrl; int i, ena, compat; ena = 1; cpu_pm_ctrl = read_cpu_ctrl(CPU_PM_CTRL); for (i = 0; fdt_pm_mask_table[i].compat != NULL; i++) { if (dev_mask & (1 << i)) continue; compat = fdt_is_compatible(node, fdt_pm_mask_table[i].compat); #if defined(SOC_MV_KIRKWOOD) if (compat && (cpu_pm_ctrl & fdt_pm_mask_table[i].mask)) { dev_mask |= (1 << i); ena = 0; break; } else if (compat) { dev_mask |= (1 << i); break; } #else if (compat && (~cpu_pm_ctrl & fdt_pm_mask_table[i].mask)) { dev_mask |= (1 << i); ena = 0; break; } else if (compat) { dev_mask |= (1 << i); break; } #endif } return (ena); } uint32_t read_cpu_ctrl(uint32_t reg) { return (bus_space_read_4(fdtbus_bs_tag, MV_CPU_CONTROL_BASE, reg)); } void write_cpu_ctrl(uint32_t reg, uint32_t val) { bus_space_write_4(fdtbus_bs_tag, MV_CPU_CONTROL_BASE, reg, val); } #if defined(SOC_MV_ARMADAXP) || defined(SOC_MV_ARMADA38X) uint32_t read_cpu_mp_clocks(uint32_t reg) { return (bus_space_read_4(fdtbus_bs_tag, MV_MP_CLOCKS_BASE, reg)); } void write_cpu_mp_clocks(uint32_t reg, uint32_t val) { bus_space_write_4(fdtbus_bs_tag, MV_MP_CLOCKS_BASE, reg, val); } uint32_t read_cpu_misc(uint32_t reg) { return (bus_space_read_4(fdtbus_bs_tag, MV_MISC_BASE, reg)); } void write_cpu_misc(uint32_t reg, uint32_t val) { bus_space_write_4(fdtbus_bs_tag, MV_MISC_BASE, reg, val); } #endif void cpu_reset(void) { #if defined(SOC_MV_ARMADAXP) || defined (SOC_MV_ARMADA38X) write_cpu_misc(RSTOUTn_MASK, SOFT_RST_OUT_EN); write_cpu_misc(SYSTEM_SOFT_RESET, SYS_SOFT_RST); #else write_cpu_ctrl(RSTOUTn_MASK, SOFT_RST_OUT_EN); write_cpu_ctrl(SYSTEM_SOFT_RESET, SYS_SOFT_RST); #endif while (1); } uint32_t cpu_extra_feat(void) { uint32_t dev, rev; uint32_t ef = 0; soc_id(&dev, &rev); switch (dev) { case MV_DEV_88F6281: case MV_DEV_88F6282: case MV_DEV_88RC8180: case MV_DEV_MV78100_Z0: case MV_DEV_MV78100: __asm __volatile("mrc p15, 1, %0, c15, c1, 0" : "=r" (ef)); break; case MV_DEV_88F5182: case MV_DEV_88F5281: __asm __volatile("mrc p15, 0, %0, c14, c0, 0" : "=r" (ef)); break; default: if (bootverbose) printf("This ARM Core does not support any extra features\n"); } return (ef); } /* * Get the power status of device. This feature is only supported on * Kirkwood and Discovery SoCs. */ uint32_t soc_power_ctrl_get(uint32_t mask) { #if !defined(SOC_MV_ORION) && !defined(SOC_MV_LOKIPLUS) && !defined(SOC_MV_FREY) if (mask != CPU_PM_CTRL_NONE) mask &= read_cpu_ctrl(CPU_PM_CTRL); return (mask); #else return (mask); #endif } /* * Set the power status of device. This feature is only supported on * Kirkwood and Discovery SoCs. */ void soc_power_ctrl_set(uint32_t mask) { #if !defined(SOC_MV_ORION) && !defined(SOC_MV_LOKIPLUS) if (mask != CPU_PM_CTRL_NONE) write_cpu_ctrl(CPU_PM_CTRL, mask); #endif } void soc_id(uint32_t *dev, uint32_t *rev) { /* * Notice: system identifiers are available in the registers range of * PCIE controller, so using this function is only allowed (and * possible) after the internal registers range has been mapped in via * devmap_bootstrap(). */ *dev = bus_space_read_4(fdtbus_bs_tag, MV_PCIE_BASE, 0) >> 16; *rev = bus_space_read_4(fdtbus_bs_tag, MV_PCIE_BASE, 8) & 0xff; } static void soc_identify(void) { uint32_t d, r, size, mode; const char *dev; const char *rev; soc_id(&d, &r); printf("SOC: "); if (bootverbose) printf("(0x%4x:0x%02x) ", d, r); rev = ""; switch (d) { case MV_DEV_88F5181: dev = "Marvell 88F5181"; if (r == 3) rev = "B1"; break; case MV_DEV_88F5182: dev = "Marvell 88F5182"; if (r == 2) rev = "A2"; break; case MV_DEV_88F5281: dev = "Marvell 88F5281"; if (r == 4) rev = "D0"; else if (r == 5) rev = "D1"; else if (r == 6) rev = "D2"; break; case MV_DEV_88F6281: dev = "Marvell 88F6281"; if (r == 0) rev = "Z0"; else if (r == 2) rev = "A0"; else if (r == 3) rev = "A1"; break; case MV_DEV_88RC8180: dev = "Marvell 88RC8180"; break; case MV_DEV_88RC9480: dev = "Marvell 88RC9480"; break; case MV_DEV_88RC9580: dev = "Marvell 88RC9580"; break; case MV_DEV_88F6781: dev = "Marvell 88F6781"; if (r == 2) rev = "Y0"; break; case MV_DEV_88F6282: dev = "Marvell 88F6282"; if (r == 0) rev = "A0"; else if (r == 1) rev = "A1"; break; case MV_DEV_88F6828: dev = "Marvell 88F6828"; break; case MV_DEV_88F6820: dev = "Marvell 88F6820"; break; case MV_DEV_88F6810: dev = "Marvell 88F6810"; break; case MV_DEV_MV78100_Z0: dev = "Marvell MV78100 Z0"; break; case MV_DEV_MV78100: dev = "Marvell MV78100"; break; case MV_DEV_MV78160: dev = "Marvell MV78160"; break; case MV_DEV_MV78260: dev = "Marvell MV78260"; break; case MV_DEV_MV78460: dev = "Marvell MV78460"; break; default: dev = "UNKNOWN"; break; } printf("%s", dev); if (*rev != '\0') printf(" rev %s", rev); printf(", TClock %dMHz\n", get_tclk() / 1000 / 1000); mode = read_cpu_ctrl(CPU_CONFIG); printf(" Instruction cache prefetch %s, data cache prefetch %s\n", (mode & CPU_CONFIG_IC_PREF) ? "enabled" : "disabled", (mode & CPU_CONFIG_DC_PREF) ? "enabled" : "disabled"); switch (d) { case MV_DEV_88F6281: case MV_DEV_88F6282: mode = read_cpu_ctrl(CPU_L2_CONFIG) & CPU_L2_CONFIG_MODE; printf(" 256KB 4-way set-associative %s unified L2 cache\n", mode ? "write-through" : "write-back"); break; case MV_DEV_MV78100: mode = read_cpu_ctrl(CPU_CONTROL); size = mode & CPU_CONTROL_L2_SIZE; mode = mode & CPU_CONTROL_L2_MODE; printf(" %s set-associative %s unified L2 cache\n", size ? "256KB 4-way" : "512KB 8-way", mode ? "write-through" : "write-back"); break; default: break; } } static void platform_identify(void *dummy) { soc_identify(); /* * XXX Board identification e.g. read out from FPGA or similar should * go here */ } SYSINIT(platform_identify, SI_SUB_CPU, SI_ORDER_SECOND, platform_identify, NULL); #ifdef KDB static void mv_enter_debugger(void *dummy) { if (boothowto & RB_KDB) kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); } SYSINIT(mv_enter_debugger, SI_SUB_CPU, SI_ORDER_ANY, mv_enter_debugger, NULL); #endif int soc_decode_win(void) { uint32_t dev, rev; int mask, err; mask = 0; TUNABLE_INT_FETCH("hw.pm-disable-mask", &mask); if (mask != 0) pm_disable_device(mask); /* Retrieve data about physical addresses from device tree. */ if ((err = win_cpu_from_dt()) != 0) return (err); /* Retrieve our ID: some windows facilities vary between SoC models */ soc_id(&dev, &rev); #ifdef SOC_MV_ARMADAXP if ((err = decode_win_sdram_fixup()) != 0) return(err); #endif #ifndef SOC_MV_FREY if (!decode_win_cpu_valid() || !decode_win_usb_valid() || !decode_win_eth_valid() || !decode_win_idma_valid() || !decode_win_pcie_valid() || !decode_win_sata_valid() || !decode_win_xor_valid() || !decode_win_usb3_valid()) return (EINVAL); decode_win_cpu_setup(); #else if (!decode_win_usb_valid() || !decode_win_eth_valid() || !decode_win_idma_valid() || !decode_win_pcie_valid() || !decode_win_sata_valid() || !decode_win_xor_valid() || !decode_win_usb3_valid()) return (EINVAL); #endif if (MV_DUMP_WIN) soc_dump_decode_win(); eth_port = 0; usb_port = 0; if ((err = fdt_win_setup()) != 0) return (err); return (0); } /************************************************************************** * Decode windows registers accessors **************************************************************************/ #if !defined(SOC_MV_FREY) WIN_REG_IDX_RD(win_cpu, cr, MV_WIN_CPU_CTRL, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_RD(win_cpu, br, MV_WIN_CPU_BASE, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_RD(win_cpu, remap_l, MV_WIN_CPU_REMAP_LO, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_RD(win_cpu, remap_h, MV_WIN_CPU_REMAP_HI, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_WR(win_cpu, cr, MV_WIN_CPU_CTRL, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_WR(win_cpu, br, MV_WIN_CPU_BASE, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_WR(win_cpu, remap_l, MV_WIN_CPU_REMAP_LO, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_WR(win_cpu, remap_h, MV_WIN_CPU_REMAP_HI, MV_MBUS_BRIDGE_BASE) #endif WIN_REG_BASE_IDX_RD(win_usb, cr, MV_WIN_USB_CTRL) WIN_REG_BASE_IDX_RD(win_usb, br, MV_WIN_USB_BASE) WIN_REG_BASE_IDX_WR(win_usb, cr, MV_WIN_USB_CTRL) WIN_REG_BASE_IDX_WR(win_usb, br, MV_WIN_USB_BASE) #ifdef SOC_MV_ARMADA38X WIN_REG_BASE_IDX_RD(win_usb3, cr, MV_WIN_USB3_CTRL) WIN_REG_BASE_IDX_RD(win_usb3, br, MV_WIN_USB3_BASE) WIN_REG_BASE_IDX_WR(win_usb3, cr, MV_WIN_USB3_CTRL) WIN_REG_BASE_IDX_WR(win_usb3, br, MV_WIN_USB3_BASE) #endif WIN_REG_BASE_IDX_RD(win_eth, br, MV_WIN_ETH_BASE) WIN_REG_BASE_IDX_RD(win_eth, sz, MV_WIN_ETH_SIZE) WIN_REG_BASE_IDX_RD(win_eth, har, MV_WIN_ETH_REMAP) WIN_REG_BASE_IDX_WR(win_eth, br, MV_WIN_ETH_BASE) WIN_REG_BASE_IDX_WR(win_eth, sz, MV_WIN_ETH_SIZE) WIN_REG_BASE_IDX_WR(win_eth, har, MV_WIN_ETH_REMAP) WIN_REG_BASE_IDX_RD2(win_xor, br, MV_WIN_XOR_BASE) WIN_REG_BASE_IDX_RD2(win_xor, sz, MV_WIN_XOR_SIZE) WIN_REG_BASE_IDX_RD2(win_xor, har, MV_WIN_XOR_REMAP) WIN_REG_BASE_IDX_RD2(win_xor, ctrl, MV_WIN_XOR_CTRL) WIN_REG_BASE_IDX_WR2(win_xor, br, MV_WIN_XOR_BASE) WIN_REG_BASE_IDX_WR2(win_xor, sz, MV_WIN_XOR_SIZE) WIN_REG_BASE_IDX_WR2(win_xor, har, MV_WIN_XOR_REMAP) WIN_REG_BASE_IDX_WR2(win_xor, ctrl, MV_WIN_XOR_CTRL) WIN_REG_BASE_RD(win_eth, bare, 0x290) WIN_REG_BASE_RD(win_eth, epap, 0x294) WIN_REG_BASE_WR(win_eth, bare, 0x290) WIN_REG_BASE_WR(win_eth, epap, 0x294) WIN_REG_BASE_IDX_RD(win_pcie, cr, MV_WIN_PCIE_CTRL); WIN_REG_BASE_IDX_RD(win_pcie, br, MV_WIN_PCIE_BASE); WIN_REG_BASE_IDX_RD(win_pcie, remap, MV_WIN_PCIE_REMAP); WIN_REG_BASE_IDX_WR(win_pcie, cr, MV_WIN_PCIE_CTRL); WIN_REG_BASE_IDX_WR(win_pcie, br, MV_WIN_PCIE_BASE); WIN_REG_BASE_IDX_WR(win_pcie, remap, MV_WIN_PCIE_REMAP); WIN_REG_BASE_IDX_RD(pcie_bar, br, MV_PCIE_BAR_BASE); WIN_REG_BASE_IDX_WR(pcie_bar, br, MV_PCIE_BAR_BASE); WIN_REG_BASE_IDX_WR(pcie_bar, brh, MV_PCIE_BAR_BASE_H); WIN_REG_BASE_IDX_WR(pcie_bar, cr, MV_PCIE_BAR_CTRL); WIN_REG_BASE_IDX_RD(win_idma, br, MV_WIN_IDMA_BASE) WIN_REG_BASE_IDX_RD(win_idma, sz, MV_WIN_IDMA_SIZE) WIN_REG_BASE_IDX_RD(win_idma, har, MV_WIN_IDMA_REMAP) WIN_REG_BASE_IDX_RD(win_idma, cap, MV_WIN_IDMA_CAP) WIN_REG_BASE_IDX_WR(win_idma, br, MV_WIN_IDMA_BASE) WIN_REG_BASE_IDX_WR(win_idma, sz, MV_WIN_IDMA_SIZE) WIN_REG_BASE_IDX_WR(win_idma, har, MV_WIN_IDMA_REMAP) WIN_REG_BASE_IDX_WR(win_idma, cap, MV_WIN_IDMA_CAP) WIN_REG_BASE_RD(win_idma, bare, 0xa80) WIN_REG_BASE_WR(win_idma, bare, 0xa80) WIN_REG_BASE_IDX_RD(win_sata, cr, MV_WIN_SATA_CTRL); WIN_REG_BASE_IDX_RD(win_sata, br, MV_WIN_SATA_BASE); WIN_REG_BASE_IDX_WR(win_sata, cr, MV_WIN_SATA_CTRL); WIN_REG_BASE_IDX_WR(win_sata, br, MV_WIN_SATA_BASE); #ifndef SOC_MV_DOVE WIN_REG_IDX_RD(ddr, br, MV_WIN_DDR_BASE, MV_DDR_CADR_BASE) WIN_REG_IDX_RD(ddr, sz, MV_WIN_DDR_SIZE, MV_DDR_CADR_BASE) WIN_REG_IDX_WR(ddr, br, MV_WIN_DDR_BASE, MV_DDR_CADR_BASE) WIN_REG_IDX_WR(ddr, sz, MV_WIN_DDR_SIZE, MV_DDR_CADR_BASE) #else /* * On 88F6781 (Dove) SoC DDR Controller is accessed through * single MBUS <-> AXI bridge. In this case we provide emulated * ddr_br_read() and ddr_sz_read() functions to keep compatibility * with common decoding windows setup code. */ static inline uint32_t ddr_br_read(int i) { uint32_t mmap; /* Read Memory Address Map Register for CS i */ mmap = bus_space_read_4(fdtbus_bs_tag, MV_DDR_CADR_BASE + (i * 0x10), 0); /* Return CS i base address */ return (mmap & 0xFF000000); } static inline uint32_t ddr_sz_read(int i) { uint32_t mmap, size; /* Read Memory Address Map Register for CS i */ mmap = bus_space_read_4(fdtbus_bs_tag, MV_DDR_CADR_BASE + (i * 0x10), 0); /* Extract size of CS space in 64kB units */ size = (1 << ((mmap >> 16) & 0x0F)); /* Return CS size and enable/disable status */ return (((size - 1) << 16) | (mmap & 0x01)); } #endif #if !defined(SOC_MV_FREY) /************************************************************************** * Decode windows helper routines **************************************************************************/ void soc_dump_decode_win(void) { uint32_t dev, rev; int i; soc_id(&dev, &rev); for (i = 0; i < MV_WIN_CPU_MAX; i++) { printf("CPU window#%d: c 0x%08x, b 0x%08x", i, win_cpu_cr_read(i), win_cpu_br_read(i)); if (win_cpu_can_remap(i)) printf(", rl 0x%08x, rh 0x%08x", win_cpu_remap_l_read(i), win_cpu_remap_h_read(i)); printf("\n"); } printf("Internal regs base: 0x%08x\n", bus_space_read_4(fdtbus_bs_tag, MV_INTREGS_BASE, 0)); for (i = 0; i < MV_WIN_DDR_MAX; i++) printf("DDR CS#%d: b 0x%08x, s 0x%08x\n", i, ddr_br_read(i), ddr_sz_read(i)); } /************************************************************************** * CPU windows routines **************************************************************************/ int win_cpu_can_remap(int i) { uint32_t dev, rev; soc_id(&dev, &rev); /* Depending on the SoC certain windows have remap capability */ if ((dev == MV_DEV_88F5182 && i < 2) || (dev == MV_DEV_88F5281 && i < 4) || (dev == MV_DEV_88F6281 && i < 4) || (dev == MV_DEV_88F6282 && i < 4) || (dev == MV_DEV_88F6828 && i < 20) || (dev == MV_DEV_88F6820 && i < 20) || (dev == MV_DEV_88F6810 && i < 20) || (dev == MV_DEV_88RC8180 && i < 2) || (dev == MV_DEV_88F6781 && i < 4) || (dev == MV_DEV_MV78100_Z0 && i < 8) || ((dev & MV_DEV_FAMILY_MASK) == MV_DEV_DISCOVERY && i < 8)) return (1); return (0); } /* XXX This should check for overlapping remap fields too.. */ int decode_win_overlap(int win, int win_no, const struct decode_win *wintab) { const struct decode_win *tab; int i; tab = wintab; for (i = 0; i < win_no; i++, tab++) { if (i == win) /* Skip self */ continue; if ((tab->base + tab->size - 1) < (wintab + win)->base) continue; else if (((wintab + win)->base + (wintab + win)->size - 1) < tab->base) continue; else return (i); } return (-1); } static int decode_win_cpu_valid(void) { int i, j, rv; uint32_t b, e, s; if (cpu_wins_no > MV_WIN_CPU_MAX) { printf("CPU windows: too many entries: %d\n", cpu_wins_no); return (0); } rv = 1; for (i = 0; i < cpu_wins_no; i++) { if (cpu_wins[i].target == 0) { printf("CPU window#%d: DDR target window is not " "supposed to be reprogrammed!\n", i); rv = 0; } if (cpu_wins[i].remap != ~0 && win_cpu_can_remap(i) != 1) { printf("CPU window#%d: not capable of remapping, but " "val 0x%08x defined\n", i, cpu_wins[i].remap); rv = 0; } s = cpu_wins[i].size; b = cpu_wins[i].base; e = b + s - 1; if (s > (0xFFFFFFFF - b + 1)) { /* * XXX this boundary check should account for 64bit * and remapping.. */ printf("CPU window#%d: no space for size 0x%08x at " "0x%08x\n", i, s, b); rv = 0; continue; } if (b != rounddown2(b, s)) { printf("CPU window#%d: address 0x%08x is not aligned " "to 0x%08x\n", i, b, s); rv = 0; continue; } j = decode_win_overlap(i, cpu_wins_no, &cpu_wins[0]); if (j >= 0) { printf("CPU window#%d: (0x%08x - 0x%08x) overlaps " "with #%d (0x%08x - 0x%08x)\n", i, b, e, j, cpu_wins[j].base, cpu_wins[j].base + cpu_wins[j].size - 1); rv = 0; } } return (rv); } int decode_win_cpu_set(int target, int attr, vm_paddr_t base, uint32_t size, vm_paddr_t remap) { uint32_t br, cr; int win, i; if (remap == ~0) { win = MV_WIN_CPU_MAX - 1; i = -1; } else { win = 0; i = 1; } while ((win >= 0) && (win < MV_WIN_CPU_MAX)) { cr = win_cpu_cr_read(win); if ((cr & MV_WIN_CPU_ENABLE_BIT) == 0) break; if ((cr & ((0xff << MV_WIN_CPU_ATTR_SHIFT) | (0x1f << MV_WIN_CPU_TARGET_SHIFT))) == ((attr << MV_WIN_CPU_ATTR_SHIFT) | (target << MV_WIN_CPU_TARGET_SHIFT))) break; win += i; } if ((win < 0) || (win >= MV_WIN_CPU_MAX) || ((remap != ~0) && (win_cpu_can_remap(win) == 0))) return (-1); br = base & 0xffff0000; win_cpu_br_write(win, br); if (win_cpu_can_remap(win)) { if (remap != ~0) { win_cpu_remap_l_write(win, remap & 0xffff0000); win_cpu_remap_h_write(win, 0); } else { /* * Remap function is not used for a given window * (capable of remapping) - set remap field with the * same value as base. */ win_cpu_remap_l_write(win, base & 0xffff0000); win_cpu_remap_h_write(win, 0); } } cr = ((size - 1) & 0xffff0000) | (attr << MV_WIN_CPU_ATTR_SHIFT) | (target << MV_WIN_CPU_TARGET_SHIFT) | MV_WIN_CPU_ENABLE_BIT; win_cpu_cr_write(win, cr); return (0); } static void decode_win_cpu_setup(void) { int i; /* Disable all CPU windows */ for (i = 0; i < MV_WIN_CPU_MAX; i++) { win_cpu_cr_write(i, 0); win_cpu_br_write(i, 0); if (win_cpu_can_remap(i)) { win_cpu_remap_l_write(i, 0); win_cpu_remap_h_write(i, 0); } } for (i = 0; i < cpu_wins_no; i++) if (cpu_wins[i].target > 0) decode_win_cpu_set(cpu_wins[i].target, cpu_wins[i].attr, cpu_wins[i].base, cpu_wins[i].size, cpu_wins[i].remap); } #endif #ifdef SOC_MV_ARMADAXP static int decode_win_sdram_fixup(void) { struct mem_region mr[FDT_MEM_REGIONS]; uint8_t window_valid[MV_WIN_DDR_MAX]; int mr_cnt, err, i, j; uint32_t valid_win_num = 0; /* Grab physical memory regions information from device tree. */ err = fdt_get_mem_regions(mr, &mr_cnt, NULL); if (err != 0) return (err); for (i = 0; i < MV_WIN_DDR_MAX; i++) window_valid[i] = 0; /* Try to match entries from device tree with settings from u-boot */ for (i = 0; i < mr_cnt; i++) { for (j = 0; j < MV_WIN_DDR_MAX; j++) { if (ddr_is_active(j) && (ddr_base(j) == mr[i].mr_start) && (ddr_size(j) == mr[i].mr_size)) { window_valid[j] = 1; valid_win_num++; } } } if (mr_cnt != valid_win_num) return (EINVAL); /* Destroy windows without corresponding device tree entry */ for (j = 0; j < MV_WIN_DDR_MAX; j++) { if (ddr_is_active(j) && (window_valid[j] != 1)) { printf("Disabling SDRAM decoding window: %d\n", j); ddr_disable(j); } } return (0); } #endif /* * Check if we're able to cover all active DDR banks. */ static int decode_win_can_cover_ddr(int max) { int i, c; c = 0; for (i = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) c++; if (c > max) { printf("Unable to cover all active DDR banks: " "%d, available windows: %d\n", c, max); return (0); } return (1); } /************************************************************************** * DDR windows routines **************************************************************************/ int ddr_is_active(int i) { if (ddr_sz_read(i) & 0x1) return (1); return (0); } void ddr_disable(int i) { ddr_sz_write(i, 0); ddr_br_write(i, 0); } uint32_t ddr_base(int i) { return (ddr_br_read(i) & 0xff000000); } uint32_t ddr_size(int i) { return ((ddr_sz_read(i) | 0x00ffffff) + 1); } uint32_t ddr_attr(int i) { uint32_t dev, rev; soc_id(&dev, &rev); if (dev == MV_DEV_88RC8180) return ((ddr_sz_read(i) & 0xf0) >> 4); if (dev == MV_DEV_88F6781) return (0); return (i == 0 ? 0xe : (i == 1 ? 0xd : (i == 2 ? 0xb : (i == 3 ? 0x7 : 0xff)))); } uint32_t ddr_target(int i) { uint32_t dev, rev; soc_id(&dev, &rev); if (dev == MV_DEV_88RC8180) { i = (ddr_sz_read(i) & 0xf0) >> 4; return (i == 0xe ? 0xc : (i == 0xd ? 0xd : (i == 0xb ? 0xe : (i == 0x7 ? 0xf : 0xc)))); } /* * On SOCs other than 88RC8180 Mbus unit ID for * DDR SDRAM controller is always 0x0. */ return (0); } /************************************************************************** * USB windows routines **************************************************************************/ static int decode_win_usb_valid(void) { return (decode_win_can_cover_ddr(MV_WIN_USB_MAX)); } static void decode_win_usb_dump(u_long base) { int i; if (pm_is_disabled(CPU_PM_CTRL_USB(usb_port - 1))) return; for (i = 0; i < MV_WIN_USB_MAX; i++) printf("USB window#%d: c 0x%08x, b 0x%08x\n", i, win_usb_cr_read(base, i), win_usb_br_read(base, i)); } /* * Set USB decode windows. */ static void decode_win_usb_setup(u_long base) { uint32_t br, cr; int i, j; if (pm_is_disabled(CPU_PM_CTRL_USB(usb_port))) return; usb_port++; for (i = 0; i < MV_WIN_USB_MAX; i++) { win_usb_cr_write(base, i, 0); win_usb_br_write(base, i, 0); } /* Only access to active DRAM banks is required */ for (i = 0; i < MV_WIN_DDR_MAX; i++) { if (ddr_is_active(i)) { br = ddr_base(i); /* * XXX for 6281 we should handle Mbus write * burst limit field in the ctrl reg */ cr = (((ddr_size(i) - 1) & 0xffff0000) | (ddr_attr(i) << 8) | (ddr_target(i) << 4) | 1); /* Set the first free USB window */ for (j = 0; j < MV_WIN_USB_MAX; j++) { if (win_usb_cr_read(base, j) & 0x1) continue; win_usb_br_write(base, j, br); win_usb_cr_write(base, j, cr); break; } } } } /************************************************************************** * USB3 windows routines **************************************************************************/ #ifdef SOC_MV_ARMADA38X static int decode_win_usb3_valid(void) { return (decode_win_can_cover_ddr(MV_WIN_USB3_MAX)); } static void decode_win_usb3_dump(u_long base) { int i; for (i = 0; i < MV_WIN_USB3_MAX; i++) printf("USB3.0 window#%d: c 0x%08x, b 0x%08x\n", i, win_usb3_cr_read(base, i), win_usb3_br_read(base, i)); } /* * Set USB3 decode windows */ static void decode_win_usb3_setup(u_long base) { uint32_t br, cr; int i, j; for (i = 0; i < MV_WIN_USB3_MAX; i++) { win_usb3_cr_write(base, i, 0); win_usb3_br_write(base, i, 0); } /* Only access to active DRAM banks is required */ for (i = 0; i < MV_WIN_DDR_MAX; i++) { if (ddr_is_active(i)) { br = ddr_base(i); cr = (((ddr_size(i) - 1) & (IO_WIN_SIZE_MASK << IO_WIN_SIZE_SHIFT)) | (ddr_attr(i) << IO_WIN_ATTR_SHIFT) | (ddr_target(i) << IO_WIN_TGT_SHIFT) | IO_WIN_ENA_MASK); /* Set the first free USB3.0 window */ for (j = 0; j < MV_WIN_USB3_MAX; j++) { if (win_usb3_cr_read(base, j) & IO_WIN_ENA_MASK) continue; win_usb3_br_write(base, j, br); win_usb3_cr_write(base, j, cr); break; } } } } #else /* * Provide dummy functions to satisfy the build * for SoCs not equipped with USB3 */ static int decode_win_usb3_valid(void) { return (1); } static void decode_win_usb3_setup(u_long base) { } static void decode_win_usb3_dump(u_long base) { } #endif /************************************************************************** * ETH windows routines **************************************************************************/ static int win_eth_can_remap(int i) { /* ETH encode windows 0-3 have remap capability */ if (i < 4) return (1); return (0); } static int eth_bare_read(uint32_t base, int i) { uint32_t v; v = win_eth_bare_read(base); v &= (1 << i); return (v >> i); } static void eth_bare_write(uint32_t base, int i, int val) { uint32_t v; v = win_eth_bare_read(base); v &= ~(1 << i); v |= (val << i); win_eth_bare_write(base, v); } static void eth_epap_write(uint32_t base, int i, int val) { uint32_t v; v = win_eth_epap_read(base); v &= ~(0x3 << (i * 2)); v |= (val << (i * 2)); win_eth_epap_write(base, v); } static void decode_win_eth_dump(u_long base) { int i; if (pm_is_disabled(CPU_PM_CTRL_GE(eth_port - 1))) return; for (i = 0; i < MV_WIN_ETH_MAX; i++) { printf("ETH window#%d: b 0x%08x, s 0x%08x", i, win_eth_br_read(base, i), win_eth_sz_read(base, i)); if (win_eth_can_remap(i)) printf(", ha 0x%08x", win_eth_har_read(base, i)); printf("\n"); } printf("ETH windows: bare 0x%08x, epap 0x%08x\n", win_eth_bare_read(base), win_eth_epap_read(base)); } #if defined(SOC_MV_LOKIPLUS) #define MV_WIN_ETH_DDR_TRGT(n) 0 #else #define MV_WIN_ETH_DDR_TRGT(n) ddr_target(n) #endif static void decode_win_eth_setup(u_long base) { uint32_t br, sz; int i, j; if (pm_is_disabled(CPU_PM_CTRL_GE(eth_port))) return; eth_port++; /* Disable, clear and revoke protection for all ETH windows */ for (i = 0; i < MV_WIN_ETH_MAX; i++) { eth_bare_write(base, i, 1); eth_epap_write(base, i, 0); win_eth_br_write(base, i, 0); win_eth_sz_write(base, i, 0); if (win_eth_can_remap(i)) win_eth_har_write(base, i, 0); } /* Only access to active DRAM banks is required */ for (i = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) { br = ddr_base(i) | (ddr_attr(i) << 8) | MV_WIN_ETH_DDR_TRGT(i); sz = ((ddr_size(i) - 1) & 0xffff0000); /* Set the first free ETH window */ for (j = 0; j < MV_WIN_ETH_MAX; j++) { if (eth_bare_read(base, j) == 0) continue; win_eth_br_write(base, j, br); win_eth_sz_write(base, j, sz); /* XXX remapping ETH windows not supported */ /* Set protection RW */ eth_epap_write(base, j, 0x3); /* Enable window */ eth_bare_write(base, j, 0); break; } } } static int decode_win_eth_valid(void) { return (decode_win_can_cover_ddr(MV_WIN_ETH_MAX)); } /************************************************************************** * PCIE windows routines **************************************************************************/ void decode_win_pcie_setup(u_long base) { uint32_t size = 0, ddrbase = ~0; uint32_t cr, br; int i, j; for (i = 0; i < MV_PCIE_BAR_MAX; i++) { pcie_bar_br_write(base, i, MV_PCIE_BAR_64BIT | MV_PCIE_BAR_PREFETCH_EN); if (i < 3) pcie_bar_brh_write(base, i, 0); if (i > 0) pcie_bar_cr_write(base, i, 0); } for (i = 0; i < MV_WIN_PCIE_MAX; i++) { win_pcie_cr_write(base, i, 0); win_pcie_br_write(base, i, 0); win_pcie_remap_write(base, i, 0); } /* On End-Point only set BAR size to 1MB regardless of DDR size */ if ((bus_space_read_4(fdtbus_bs_tag, base, MV_PCIE_CONTROL) & MV_PCIE_ROOT_CMPLX) == 0) { pcie_bar_cr_write(base, 1, 0xf0000 | 1); return; } for (i = 0; i < MV_WIN_DDR_MAX; i++) { if (ddr_is_active(i)) { /* Map DDR to BAR 1 */ cr = (ddr_size(i) - 1) & 0xffff0000; size += ddr_size(i) & 0xffff0000; cr |= (ddr_attr(i) << 8) | (ddr_target(i) << 4) | 1; br = ddr_base(i); if (br < ddrbase) ddrbase = br; /* Use the first available PCIE window */ for (j = 0; j < MV_WIN_PCIE_MAX; j++) { if (win_pcie_cr_read(base, j) != 0) continue; win_pcie_br_write(base, j, br); win_pcie_cr_write(base, j, cr); break; } } } /* * Upper 16 bits in BAR register is interpreted as BAR size - * (in 64 kB units) plus 64kB, so substract 0x10000 + * (in 64 kB units) plus 64kB, so subtract 0x10000 * form value passed to register to get correct value. */ size -= 0x10000; pcie_bar_cr_write(base, 1, size | 1); pcie_bar_br_write(base, 1, ddrbase | MV_PCIE_BAR_64BIT | MV_PCIE_BAR_PREFETCH_EN); pcie_bar_br_write(base, 0, fdt_immr_pa | MV_PCIE_BAR_64BIT | MV_PCIE_BAR_PREFETCH_EN); } static int decode_win_pcie_valid(void) { return (decode_win_can_cover_ddr(MV_WIN_PCIE_MAX)); } /************************************************************************** * IDMA windows routines **************************************************************************/ #if defined(SOC_MV_ORION) || defined(SOC_MV_DISCOVERY) static int idma_bare_read(u_long base, int i) { uint32_t v; v = win_idma_bare_read(base); v &= (1 << i); return (v >> i); } static void idma_bare_write(u_long base, int i, int val) { uint32_t v; v = win_idma_bare_read(base); v &= ~(1 << i); v |= (val << i); win_idma_bare_write(base, v); } /* * Sets channel protection 'val' for window 'w' on channel 'c' */ static void idma_cap_write(u_long base, int c, int w, int val) { uint32_t v; v = win_idma_cap_read(base, c); v &= ~(0x3 << (w * 2)); v |= (val << (w * 2)); win_idma_cap_write(base, c, v); } /* * Set protection 'val' on all channels for window 'w' */ static void idma_set_prot(u_long base, int w, int val) { int c; for (c = 0; c < MV_IDMA_CHAN_MAX; c++) idma_cap_write(base, c, w, val); } static int win_idma_can_remap(int i) { /* IDMA decode windows 0-3 have remap capability */ if (i < 4) return (1); return (0); } void decode_win_idma_setup(u_long base) { uint32_t br, sz; int i, j; if (pm_is_disabled(CPU_PM_CTRL_IDMA)) return; /* * Disable and clear all IDMA windows, revoke protection for all channels */ for (i = 0; i < MV_WIN_IDMA_MAX; i++) { idma_bare_write(base, i, 1); win_idma_br_write(base, i, 0); win_idma_sz_write(base, i, 0); if (win_idma_can_remap(i) == 1) win_idma_har_write(base, i, 0); } for (i = 0; i < MV_IDMA_CHAN_MAX; i++) win_idma_cap_write(base, i, 0); /* * Set up access to all active DRAM banks */ for (i = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) { br = ddr_base(i) | (ddr_attr(i) << 8) | ddr_target(i); sz = ((ddr_size(i) - 1) & 0xffff0000); /* Place DDR entries in non-remapped windows */ for (j = 0; j < MV_WIN_IDMA_MAX; j++) if (win_idma_can_remap(j) != 1 && idma_bare_read(base, j) == 1) { /* Configure window */ win_idma_br_write(base, j, br); win_idma_sz_write(base, j, sz); /* Set protection RW on all channels */ idma_set_prot(base, j, 0x3); /* Enable window */ idma_bare_write(base, j, 0); break; } } /* * Remaining targets -- from statically defined table */ for (i = 0; i < idma_wins_no; i++) if (idma_wins[i].target > 0) { br = (idma_wins[i].base & 0xffff0000) | (idma_wins[i].attr << 8) | idma_wins[i].target; sz = ((idma_wins[i].size - 1) & 0xffff0000); /* Set the first free IDMA window */ for (j = 0; j < MV_WIN_IDMA_MAX; j++) { if (idma_bare_read(base, j) == 0) continue; /* Configure window */ win_idma_br_write(base, j, br); win_idma_sz_write(base, j, sz); if (win_idma_can_remap(j) && idma_wins[j].remap >= 0) win_idma_har_write(base, j, idma_wins[j].remap); /* Set protection RW on all channels */ idma_set_prot(base, j, 0x3); /* Enable window */ idma_bare_write(base, j, 0); break; } } } int decode_win_idma_valid(void) { const struct decode_win *wintab; int c, i, j, rv; uint32_t b, e, s; if (idma_wins_no > MV_WIN_IDMA_MAX) { printf("IDMA windows: too many entries: %d\n", idma_wins_no); return (0); } for (i = 0, c = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) c++; if (idma_wins_no > (MV_WIN_IDMA_MAX - c)) { printf("IDMA windows: too many entries: %d, available: %d\n", idma_wins_no, MV_WIN_IDMA_MAX - c); return (0); } wintab = idma_wins; rv = 1; for (i = 0; i < idma_wins_no; i++, wintab++) { if (wintab->target == 0) { printf("IDMA window#%d: DDR target window is not " "supposed to be reprogrammed!\n", i); rv = 0; } if (wintab->remap >= 0 && win_cpu_can_remap(i) != 1) { printf("IDMA window#%d: not capable of remapping, but " "val 0x%08x defined\n", i, wintab->remap); rv = 0; } s = wintab->size; b = wintab->base; e = b + s - 1; if (s > (0xFFFFFFFF - b + 1)) { /* XXX this boundary check should account for 64bit and * remapping.. */ printf("IDMA window#%d: no space for size 0x%08x at " "0x%08x\n", i, s, b); rv = 0; continue; } j = decode_win_overlap(i, idma_wins_no, &idma_wins[0]); if (j >= 0) { printf("IDMA window#%d: (0x%08x - 0x%08x) overlaps " "with #%d (0x%08x - 0x%08x)\n", i, b, e, j, idma_wins[j].base, idma_wins[j].base + idma_wins[j].size - 1); rv = 0; } } return (rv); } void decode_win_idma_dump(u_long base) { int i; if (pm_is_disabled(CPU_PM_CTRL_IDMA)) return; for (i = 0; i < MV_WIN_IDMA_MAX; i++) { printf("IDMA window#%d: b 0x%08x, s 0x%08x", i, win_idma_br_read(base, i), win_idma_sz_read(base, i)); if (win_idma_can_remap(i)) printf(", ha 0x%08x", win_idma_har_read(base, i)); printf("\n"); } for (i = 0; i < MV_IDMA_CHAN_MAX; i++) printf("IDMA channel#%d: ap 0x%08x\n", i, win_idma_cap_read(base, i)); printf("IDMA windows: bare 0x%08x\n", win_idma_bare_read(base)); } #else /* Provide dummy functions to satisfy the build for SoCs not equipped with IDMA */ int decode_win_idma_valid(void) { return (1); } void decode_win_idma_setup(u_long base) { } void decode_win_idma_dump(u_long base) { } #endif /************************************************************************** * XOR windows routines **************************************************************************/ #if defined(SOC_MV_KIRKWOOD) || defined(SOC_MV_DISCOVERY) static int xor_ctrl_read(u_long base, int i, int c, int e) { uint32_t v; v = win_xor_ctrl_read(base, c, e); v &= (1 << i); return (v >> i); } static void xor_ctrl_write(u_long base, int i, int c, int e, int val) { uint32_t v; v = win_xor_ctrl_read(base, c, e); v &= ~(1 << i); v |= (val << i); win_xor_ctrl_write(base, c, e, v); } /* * Set channel protection 'val' for window 'w' on channel 'c' */ static void xor_chan_write(u_long base, int c, int e, int w, int val) { uint32_t v; v = win_xor_ctrl_read(base, c, e); v &= ~(0x3 << (w * 2 + 16)); v |= (val << (w * 2 + 16)); win_xor_ctrl_write(base, c, e, v); } /* * Set protection 'val' on all channels for window 'w' on engine 'e' */ static void xor_set_prot(u_long base, int w, int e, int val) { int c; for (c = 0; c < MV_XOR_CHAN_MAX; c++) xor_chan_write(base, c, e, w, val); } static int win_xor_can_remap(int i) { /* XOR decode windows 0-3 have remap capability */ if (i < 4) return (1); return (0); } static int xor_max_eng(void) { uint32_t dev, rev; soc_id(&dev, &rev); switch (dev) { case MV_DEV_88F6281: case MV_DEV_88F6282: case MV_DEV_MV78130: case MV_DEV_MV78160: case MV_DEV_MV78230: case MV_DEV_MV78260: case MV_DEV_MV78460: return (2); case MV_DEV_MV78100: case MV_DEV_MV78100_Z0: return (1); default: return (0); } } static void xor_active_dram(u_long base, int c, int e, int *window) { uint32_t br, sz; int i, m, w; /* * Set up access to all active DRAM banks */ m = xor_max_eng(); for (i = 0; i < m; i++) if (ddr_is_active(i)) { br = ddr_base(i) | (ddr_attr(i) << 8) | ddr_target(i); sz = ((ddr_size(i) - 1) & 0xffff0000); /* Place DDR entries in non-remapped windows */ for (w = 0; w < MV_WIN_XOR_MAX; w++) if (win_xor_can_remap(w) != 1 && (xor_ctrl_read(base, w, c, e) == 0) && w > *window) { /* Configure window */ win_xor_br_write(base, w, e, br); win_xor_sz_write(base, w, e, sz); /* Set protection RW on all channels */ xor_set_prot(base, w, e, 0x3); /* Enable window */ xor_ctrl_write(base, w, c, e, 1); (*window)++; break; } } } void decode_win_xor_setup(u_long base) { uint32_t br, sz; int i, j, z, e = 1, m, window; if (pm_is_disabled(CPU_PM_CTRL_XOR)) return; /* * Disable and clear all XOR windows, revoke protection for all * channels */ m = xor_max_eng(); for (j = 0; j < m; j++, e--) { /* Number of non-remaped windows */ window = MV_XOR_NON_REMAP - 1; for (i = 0; i < MV_WIN_XOR_MAX; i++) { win_xor_br_write(base, i, e, 0); win_xor_sz_write(base, i, e, 0); } if (win_xor_can_remap(i) == 1) win_xor_har_write(base, i, e, 0); for (i = 0; i < MV_XOR_CHAN_MAX; i++) { win_xor_ctrl_write(base, i, e, 0); xor_active_dram(base, i, e, &window); } /* * Remaining targets -- from a statically defined table */ for (i = 0; i < xor_wins_no; i++) if (xor_wins[i].target > 0) { br = (xor_wins[i].base & 0xffff0000) | (xor_wins[i].attr << 8) | xor_wins[i].target; sz = ((xor_wins[i].size - 1) & 0xffff0000); /* Set the first free XOR window */ for (z = 0; z < MV_WIN_XOR_MAX; z++) { if (xor_ctrl_read(base, z, 0, e) && xor_ctrl_read(base, z, 1, e)) continue; /* Configure window */ win_xor_br_write(base, z, e, br); win_xor_sz_write(base, z, e, sz); if (win_xor_can_remap(z) && xor_wins[z].remap >= 0) win_xor_har_write(base, z, e, xor_wins[z].remap); /* Set protection RW on all channels */ xor_set_prot(base, z, e, 0x3); /* Enable window */ xor_ctrl_write(base, z, 0, e, 1); xor_ctrl_write(base, z, 1, e, 1); break; } } } } int decode_win_xor_valid(void) { const struct decode_win *wintab; int c, i, j, rv; uint32_t b, e, s; if (xor_wins_no > MV_WIN_XOR_MAX) { printf("XOR windows: too many entries: %d\n", xor_wins_no); return (0); } for (i = 0, c = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) c++; if (xor_wins_no > (MV_WIN_XOR_MAX - c)) { printf("XOR windows: too many entries: %d, available: %d\n", xor_wins_no, MV_WIN_IDMA_MAX - c); return (0); } wintab = xor_wins; rv = 1; for (i = 0; i < xor_wins_no; i++, wintab++) { if (wintab->target == 0) { printf("XOR window#%d: DDR target window is not " "supposed to be reprogrammed!\n", i); rv = 0; } if (wintab->remap >= 0 && win_cpu_can_remap(i) != 1) { printf("XOR window#%d: not capable of remapping, but " "val 0x%08x defined\n", i, wintab->remap); rv = 0; } s = wintab->size; b = wintab->base; e = b + s - 1; if (s > (0xFFFFFFFF - b + 1)) { /* * XXX this boundary check should account for 64bit * and remapping.. */ printf("XOR window#%d: no space for size 0x%08x at " "0x%08x\n", i, s, b); rv = 0; continue; } j = decode_win_overlap(i, xor_wins_no, &xor_wins[0]); if (j >= 0) { printf("XOR window#%d: (0x%08x - 0x%08x) overlaps " "with #%d (0x%08x - 0x%08x)\n", i, b, e, j, xor_wins[j].base, xor_wins[j].base + xor_wins[j].size - 1); rv = 0; } } return (rv); } void decode_win_xor_dump(u_long base) { int i, j; int e = 1; if (pm_is_disabled(CPU_PM_CTRL_XOR)) return; for (j = 0; j < xor_max_eng(); j++, e--) { for (i = 0; i < MV_WIN_XOR_MAX; i++) { printf("XOR window#%d: b 0x%08x, s 0x%08x", i, win_xor_br_read(base, i, e), win_xor_sz_read(base, i, e)); if (win_xor_can_remap(i)) printf(", ha 0x%08x", win_xor_har_read(base, i, e)); printf("\n"); } for (i = 0; i < MV_XOR_CHAN_MAX; i++) printf("XOR control#%d: 0x%08x\n", i, win_xor_ctrl_read(base, i, e)); } } #else /* Provide dummy functions to satisfy the build for SoCs not equipped with XOR */ static int decode_win_xor_valid(void) { return (1); } static void decode_win_xor_setup(u_long base) { } static void decode_win_xor_dump(u_long base) { } #endif /************************************************************************** * SATA windows routines **************************************************************************/ static void decode_win_sata_setup(u_long base) { uint32_t cr, br; int i, j; if (pm_is_disabled(CPU_PM_CTRL_SATA)) return; for (i = 0; i < MV_WIN_SATA_MAX; i++) { win_sata_cr_write(base, i, 0); win_sata_br_write(base, i, 0); } for (i = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) { cr = ((ddr_size(i) - 1) & 0xffff0000) | (ddr_attr(i) << 8) | (ddr_target(i) << 4) | 1; br = ddr_base(i); /* Use the first available SATA window */ for (j = 0; j < MV_WIN_SATA_MAX; j++) { if ((win_sata_cr_read(base, j) & 1) != 0) continue; win_sata_br_write(base, j, br); win_sata_cr_write(base, j, cr); break; } } } static int decode_win_sata_valid(void) { uint32_t dev, rev; soc_id(&dev, &rev); if (dev == MV_DEV_88F5281) return (1); return (decode_win_can_cover_ddr(MV_WIN_SATA_MAX)); } /************************************************************************** * FDT parsing routines. **************************************************************************/ static int fdt_get_ranges(const char *nodename, void *buf, int size, int *tuples, int *tuplesize) { phandle_t node; pcell_t addr_cells, par_addr_cells, size_cells; int len, tuple_size, tuples_count; node = OF_finddevice(nodename); if (node == -1) return (EINVAL); if ((fdt_addrsize_cells(node, &addr_cells, &size_cells)) != 0) return (ENXIO); par_addr_cells = fdt_parent_addr_cells(node); if (par_addr_cells > 2) return (ERANGE); tuple_size = sizeof(pcell_t) * (addr_cells + par_addr_cells + size_cells); /* Note the OF_getprop_alloc() cannot be used at this early stage. */ len = OF_getprop(node, "ranges", buf, size); /* * XXX this does not handle the empty 'ranges;' case, which is * legitimate and should be allowed. */ tuples_count = len / tuple_size; if (tuples_count <= 0) return (ERANGE); if (par_addr_cells > 2 || addr_cells > 2 || size_cells > 2) return (ERANGE); *tuples = tuples_count; *tuplesize = tuple_size; return (0); } static int win_cpu_from_dt(void) { pcell_t ranges[48]; phandle_t node; int i, entry_size, err, t, tuple_size, tuples; u_long sram_base, sram_size; t = 0; /* Retrieve 'ranges' property of '/localbus' node. */ if ((err = fdt_get_ranges("/localbus", ranges, sizeof(ranges), &tuples, &tuple_size)) == 0) { /* * Fill CPU decode windows table. */ bzero((void *)&cpu_win_tbl, sizeof(cpu_win_tbl)); entry_size = tuple_size / sizeof(pcell_t); cpu_wins_no = tuples; for (i = 0, t = 0; t < tuples; i += entry_size, t++) { cpu_win_tbl[t].target = 1; cpu_win_tbl[t].attr = fdt32_to_cpu(ranges[i + 1]); cpu_win_tbl[t].base = fdt32_to_cpu(ranges[i + 2]); cpu_win_tbl[t].size = fdt32_to_cpu(ranges[i + 3]); cpu_win_tbl[t].remap = ~0; debugf("target = 0x%0x attr = 0x%0x base = 0x%0x " "size = 0x%0x remap = 0x%0x\n", cpu_win_tbl[t].target, cpu_win_tbl[t].attr, cpu_win_tbl[t].base, cpu_win_tbl[t].size, cpu_win_tbl[t].remap); } } /* * Retrieve CESA SRAM data. */ if ((node = OF_finddevice("sram")) != -1) if (fdt_is_compatible(node, "mrvl,cesa-sram")) goto moveon; if ((node = OF_finddevice("/")) == 0) return (ENXIO); if ((node = fdt_find_compatible(node, "mrvl,cesa-sram", 0)) == 0) /* SRAM block is not always present. */ return (0); moveon: sram_base = sram_size = 0; if (fdt_regsize(node, &sram_base, &sram_size) != 0) return (EINVAL); cpu_win_tbl[t].target = MV_WIN_CESA_TARGET; cpu_win_tbl[t].attr = MV_WIN_CESA_ATTR(1); cpu_win_tbl[t].base = sram_base; cpu_win_tbl[t].size = sram_size; cpu_win_tbl[t].remap = ~0; cpu_wins_no++; debugf("sram: base = 0x%0lx size = 0x%0lx\n", sram_base, sram_size); return (0); } static int fdt_win_setup(void) { phandle_t node, child; struct soc_node_spec *soc_node; u_long size, base; int err, i; node = OF_finddevice("/"); if (node == -1) panic("fdt_win_setup: no root node"); /* * Traverse through all children of root and simple-bus nodes. * For each found device retrieve decode windows data (if applicable). */ child = OF_child(node); while (child != 0) { for (i = 0; soc_nodes[i].compat != NULL; i++) { soc_node = &soc_nodes[i]; if (!fdt_is_compatible(child, soc_node->compat)) continue; err = fdt_regsize(child, &base, &size); if (err != 0) return (err); base = (base & 0x000fffff) | fdt_immr_va; if (soc_node->decode_handler != NULL) soc_node->decode_handler(base); else return (ENXIO); if (MV_DUMP_WIN && (soc_node->dump_handler != NULL)) soc_node->dump_handler(base); } /* * Once done with root-level children let's move down to * simple-bus and its children. */ child = OF_peer(child); if ((child == 0) && (node == OF_finddevice("/"))) { node = fdt_find_compatible(node, "simple-bus", 0); if (node == 0) return (ENXIO); child = OF_child(node); } /* * Next, move one more level down to internal-regs node (if * it is present) and its children. This node also have * "simple-bus" compatible. */ if ((child == 0) && (node == OF_finddevice("simple-bus"))) { node = fdt_find_compatible(node, "simple-bus", 0); if (node == 0) return (0); child = OF_child(node); } } return (0); } static void fdt_fixup_busfreq(phandle_t root) { phandle_t sb; pcell_t freq; freq = cpu_to_fdt32(get_tclk()); /* * Fix bus speed in cpu node */ if ((sb = OF_finddevice("cpu")) != 0) if (fdt_is_compatible_strict(sb, "ARM,88VS584")) OF_setprop(sb, "bus-frequency", (void *)&freq, sizeof(freq)); /* * This fixup sets the simple-bus bus-frequency property. */ if ((sb = fdt_find_compatible(root, "simple-bus", 1)) != 0) OF_setprop(sb, "bus-frequency", (void *)&freq, sizeof(freq)); } static void fdt_fixup_ranges(phandle_t root) { phandle_t node; pcell_t par_addr_cells, addr_cells, size_cells; pcell_t ranges[3], reg[2], *rangesptr; int len, tuple_size, tuples_count; uint32_t base; /* Fix-up SoC ranges according to real fdt_immr_pa */ if ((node = fdt_find_compatible(root, "simple-bus", 1)) != 0) { if (fdt_addrsize_cells(node, &addr_cells, &size_cells) == 0 && (par_addr_cells = fdt_parent_addr_cells(node) <= 2)) { tuple_size = sizeof(pcell_t) * (par_addr_cells + addr_cells + size_cells); len = OF_getprop(node, "ranges", ranges, sizeof(ranges)); tuples_count = len / tuple_size; /* Unexpected settings are not supported */ if (tuples_count != 1) goto fixup_failed; rangesptr = &ranges[0]; rangesptr += par_addr_cells; base = fdt_data_get((void *)rangesptr, addr_cells); *rangesptr = cpu_to_fdt32(fdt_immr_pa); if (OF_setprop(node, "ranges", (void *)&ranges[0], sizeof(ranges)) < 0) goto fixup_failed; } } /* Fix-up PCIe reg according to real PCIe registers' PA */ if ((node = fdt_find_compatible(root, "mrvl,pcie", 1)) != 0) { if (fdt_addrsize_cells(OF_parent(node), &par_addr_cells, &size_cells) == 0) { tuple_size = sizeof(pcell_t) * (par_addr_cells + size_cells); len = OF_getprop(node, "reg", reg, sizeof(reg)); tuples_count = len / tuple_size; /* Unexpected settings are not supported */ if (tuples_count != 1) goto fixup_failed; base = fdt_data_get((void *)®[0], par_addr_cells); base &= ~0xFF000000; base |= fdt_immr_pa; reg[0] = cpu_to_fdt32(base); if (OF_setprop(node, "reg", (void *)®[0], sizeof(reg)) < 0) goto fixup_failed; } } /* Fix-up succeeded. May return and continue */ return; fixup_failed: while (1) { /* * In case of any error while fixing ranges just hang. * 1. No message can be displayed yet since console * is not initialized. * 2. Going further will cause failure on bus_space_map() * relying on the wrong ranges or data abort when * accessing PCIe registers. */ } } struct fdt_fixup_entry fdt_fixup_table[] = { { "mrvl,DB-88F6281", &fdt_fixup_busfreq }, { "mrvl,DB-78460", &fdt_fixup_busfreq }, { "mrvl,DB-78460", &fdt_fixup_ranges }, { NULL, NULL } }; #ifndef INTRNG static int fdt_pic_decode_ic(phandle_t node, pcell_t *intr, int *interrupt, int *trig, int *pol) { if (!fdt_is_compatible(node, "mrvl,pic") && !fdt_is_compatible(node, "mrvl,mpic")) return (ENXIO); *interrupt = fdt32_to_cpu(intr[0]); *trig = INTR_TRIGGER_CONFORM; *pol = INTR_POLARITY_CONFORM; return (0); } fdt_pic_decode_t fdt_pic_table[] = { #ifdef SOC_MV_ARMADA38X &gic_decode_fdt, #endif &fdt_pic_decode_ic, NULL }; #endif uint64_t get_sar_value(void) { uint32_t sar_low, sar_high; #if defined(SOC_MV_ARMADAXP) sar_high = bus_space_read_4(fdtbus_bs_tag, MV_MISC_BASE, SAMPLE_AT_RESET_HI); sar_low = bus_space_read_4(fdtbus_bs_tag, MV_MISC_BASE, SAMPLE_AT_RESET_LO); #elif defined(SOC_MV_ARMADA38X) sar_high = 0; sar_low = bus_space_read_4(fdtbus_bs_tag, MV_MISC_BASE, SAMPLE_AT_RESET); #else /* * TODO: Add getting proper values for other SoC configurations */ sar_high = 0; sar_low = 0; #endif return (((uint64_t)sar_high << 32) | sar_low); } Index: head/sys/arm/samsung/exynos/exynos5_usb_phy.c =================================================================== --- head/sys/arm/samsung/exynos/exynos5_usb_phy.c (revision 299068) +++ head/sys/arm/samsung/exynos/exynos5_usb_phy.c (revision 299069) @@ -1,271 +1,271 @@ /*- * Copyright (c) 2014 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * DWC3 USB 3.0 DRD (dual role device) PHY */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #define USB_DRD_LINKSYSTEM 0x04 #define LINKSYSTEM_FLADJ_MASK (0x3f << 1) #define LINKSYSTEM_FLADJ(x) ((x) << 1) #define LINKSYSTEM_XHCI_VERSION_CTRL (1 << 27) #define USB_DRD_PHYUTMI 0x08 #define PHYUTMI_OTGDISABLE (1 << 6) #define PHYUTMI_FORCESUSPEND (1 << 1) #define PHYUTMI_FORCESLEEP (1 << 0) #define USB_DRD_PHYPIPE 0x0c #define USB_DRD_PHYCLKRST 0x10 #define PHYCLKRST_PORTRESET (1 << 1) #define PHYCLKRST_COMMONONN (1 << 0) #define PHYCLKRST_EN_UTMISUSPEND (1 << 31) #define PHYCLKRST_SSC_REFCLKSEL_MASK (0xff << 23) #define PHYCLKRST_SSC_REFCLKSEL(x) ((x) << 23) #define PHYCLKRST_SSC_RANGE_MASK (0x03 << 21) #define PHYCLKRST_SSC_RANGE(x) ((x) << 21) #define PHYCLKRST_SSC_EN (1 << 20) #define PHYCLKRST_REF_SSP_EN (1 << 19) #define PHYCLKRST_REF_CLKDIV2 (1 << 18) #define PHYCLKRST_MPLL_MLTPR_MASK (0x7f << 11) #define PHYCLKRST_MPLL_MLTPR_100MHZ (0x19 << 11) #define PHYCLKRST_MPLL_MLTPR_50M (0x32 << 11) #define PHYCLKRST_MPLL_MLTPR_24MHZ (0x68 << 11) #define PHYCLKRST_MPLL_MLTPR_20MHZ (0x7d << 11) #define PHYCLKRST_MPLL_MLTPR_19200KHZ (0x02 << 11) #define PHYCLKRST_FSEL_UTMI_MASK (0x7 << 5) #define PHYCLKRST_FSEL_PIPE_MASK (0x7 << 8) #define PHYCLKRST_FSEL(x) ((x) << 5) #define PHYCLKRST_FSEL_9MHZ6 0x0 #define PHYCLKRST_FSEL_10MHZ 0x1 #define PHYCLKRST_FSEL_12MHZ 0x2 #define PHYCLKRST_FSEL_19MHZ2 0x3 #define PHYCLKRST_FSEL_20MHZ 0x4 #define PHYCLKRST_FSEL_24MHZ 0x5 #define PHYCLKRST_FSEL_50MHZ 0x7 #define PHYCLKRST_RETENABLEN (1 << 4) #define PHYCLKRST_REFCLKSEL_MASK (0x03 << 2) #define PHYCLKRST_REFCLKSEL_PAD_REFCLK (0x2 << 2) #define PHYCLKRST_REFCLKSEL_EXT_REFCLK (0x3 << 2) #define USB_DRD_PHYREG0 0x14 #define USB_DRD_PHYREG1 0x18 #define USB_DRD_PHYPARAM0 0x1c #define PHYPARAM0_REF_USE_PAD (1 << 31) #define PHYPARAM0_REF_LOSLEVEL_MASK (0x1f << 26) #define PHYPARAM0_REF_LOSLEVEL (0x9 << 26) #define USB_DRD_PHYPARAM1 0x20 #define PHYPARAM1_PCS_TXDEEMPH_MASK (0x1f << 0) #define PHYPARAM1_PCS_TXDEEMPH (0x1c) #define USB_DRD_PHYTERM 0x24 #define USB_DRD_PHYTEST 0x28 #define PHYTEST_POWERDOWN_SSP (1 << 3) #define PHYTEST_POWERDOWN_HSP (1 << 2) #define USB_DRD_PHYADP 0x2c #define USB_DRD_PHYUTMICLKSEL 0x30 #define PHYUTMICLKSEL_UTMI_CLKSEL (1 << 2) #define USB_DRD_PHYRESUME 0x34 #define USB_DRD_LINKPORT 0x44 struct usb_phy_softc { struct resource *res[1]; bus_space_tag_t bst; bus_space_handle_t bsh; device_t dev; }; static struct resource_spec usb_phy_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { -1, 0 } }; static int usb_phy_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "samsung,exynos5420-usbdrd-phy")) return (ENXIO); device_set_desc(dev, "Samsung Exynos 5 USB PHY"); return (BUS_PROBE_DEFAULT); } static int vbus_on(struct usb_phy_softc *sc) { pcell_t dts_value[3]; device_t gpio_dev; phandle_t node; pcell_t pin; int len; if ((node = ofw_bus_get_node(sc->dev)) == -1) return (-1); /* Power pin */ if ((len = OF_getproplen(node, "vbus-supply")) <= 0) return (-1); OF_getencprop(node, "vbus-supply", dts_value, len); pin = dts_value[0]; gpio_dev = devclass_get_device(devclass_find("gpio"), 0); if (gpio_dev == NULL) { - device_printf(sc->dev, "cant find gpio_dev\n"); + device_printf(sc->dev, "can't find gpio_dev\n"); return (1); } GPIO_PIN_SETFLAGS(gpio_dev, pin, GPIO_PIN_OUTPUT); GPIO_PIN_SET(gpio_dev, pin, GPIO_PIN_HIGH); return (0); } static int usb3_phy_init(struct usb_phy_softc *sc) { int reg; /* Reset USB 3.0 PHY */ WRITE4(sc, USB_DRD_PHYREG0, 0); reg = READ4(sc, USB_DRD_PHYPARAM0); /* PHY CLK src */ reg &= ~(PHYPARAM0_REF_USE_PAD); reg &= ~(PHYPARAM0_REF_LOSLEVEL_MASK); reg |= (PHYPARAM0_REF_LOSLEVEL); WRITE4(sc, USB_DRD_PHYPARAM0, reg); WRITE4(sc, USB_DRD_PHYRESUME, 0); reg = (LINKSYSTEM_XHCI_VERSION_CTRL | LINKSYSTEM_FLADJ(0x20)); WRITE4(sc, USB_DRD_LINKSYSTEM, reg); reg = READ4(sc, USB_DRD_PHYPARAM1); reg &= ~(PHYPARAM1_PCS_TXDEEMPH_MASK); reg |= (PHYPARAM1_PCS_TXDEEMPH); WRITE4(sc, USB_DRD_PHYPARAM1, reg); reg = READ4(sc, USB_DRD_PHYUTMICLKSEL); reg |= (PHYUTMICLKSEL_UTMI_CLKSEL); WRITE4(sc, USB_DRD_PHYUTMICLKSEL, reg); reg = READ4(sc, USB_DRD_PHYTEST); reg &= ~(PHYTEST_POWERDOWN_HSP); reg &= ~(PHYTEST_POWERDOWN_SSP); WRITE4(sc, USB_DRD_PHYTEST, reg); WRITE4(sc, USB_DRD_PHYUTMI, PHYUTMI_OTGDISABLE); /* Clock */ reg = (PHYCLKRST_REFCLKSEL_EXT_REFCLK); reg |= (PHYCLKRST_FSEL(PHYCLKRST_FSEL_24MHZ)); reg |= (PHYCLKRST_MPLL_MLTPR_24MHZ); reg |= (PHYCLKRST_SSC_REFCLKSEL(0x88)); reg |= (PHYCLKRST_RETENABLEN | PHYCLKRST_REF_SSP_EN | /* Super speed */ PHYCLKRST_SSC_EN | /* Spread spectrum */ PHYCLKRST_COMMONONN | PHYCLKRST_PORTRESET); WRITE4(sc, USB_DRD_PHYCLKRST, reg); DELAY(50000); reg &= ~PHYCLKRST_PORTRESET; WRITE4(sc, USB_DRD_PHYCLKRST, reg); return (0); } static int usb_phy_attach(device_t dev) { struct usb_phy_softc *sc; sc = device_get_softc(dev); sc->dev = dev; if (bus_alloc_resources(dev, usb_phy_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* Memory interface */ sc->bst = rman_get_bustag(sc->res[0]); sc->bsh = rman_get_bushandle(sc->res[0]); vbus_on(sc); usbdrd_phy_power_on(); DELAY(100); usb3_phy_init(sc); return (0); } static device_method_t usb_phy_methods[] = { DEVMETHOD(device_probe, usb_phy_probe), DEVMETHOD(device_attach, usb_phy_attach), { 0, 0 } }; static driver_t usb_phy_driver = { "usb_phy", usb_phy_methods, sizeof(struct usb_phy_softc), }; static devclass_t usb_phy_devclass; DRIVER_MODULE(usb_phy, simplebus, usb_phy_driver, usb_phy_devclass, 0, 0); Index: head/sys/arm/ti/omap4/omap4_prcm_clks.c =================================================================== --- head/sys/arm/ti/omap4/omap4_prcm_clks.c (revision 299068) +++ head/sys/arm/ti/omap4/omap4_prcm_clks.c (revision 299069) @@ -1,1496 +1,1496 @@ /*- * Copyright (c) 2011 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * This file defines the clock configuration for the OMAP4xxx series of * devices. * * How This is Suppose to Work * =========================== * - There is a top level omap_prcm module that defines all OMAP SoC drivers * should use to enable/disable the system clocks regardless of the version * of OMAP device they are running on. This top level PRCM module is just * a thin shim to chip specific functions that perform the donkey work of * configuring the clock - this file is the 'donkey' for OMAP44xx devices. * * - The key bit in this file is the omap_clk_devmap array, it's * used by the omap_prcm driver to determine what clocks are valid and which * functions to call to manipulate them. * * - In essence you just need to define some callbacks for each of the * clocks and then you're done. * * - The other thing that is worth noting is that when the omap_prcm device * is registered you typically pass in some memory ranges which are the * SYS_MEMORY resources. These resources are in turn allocated using * bus_allocate_resources(...) and the resource handles are passed to all * individual clock callback handlers. * * * * OMAP4 devices are different from the previous OMAP3 devices in that there * is no longer a separate functional and interface clock for each module, * instead there is typically an interface clock that spans many modules. */ #define FREQ_96MHZ 96000000 #define FREQ_64MHZ 64000000 #define FREQ_48MHZ 48000000 #define FREQ_32KHZ 32000 #define PRM_INSTANCE 1 #define CM1_INSTANCE 2 #define CM2_INSTANCE 3 /** * Address offsets from the PRM memory region to the top level clock control * registers. */ #define CKGEN_PRM_OFFSET 0x00000100UL #define MPU_PRM_OFFSET 0x00000300UL #define DSP_PRM_OFFSET 0x00000400UL #define ABE_PRM_OFFSET 0x00000500UL #define ALWAYS_ON_PRM_OFFSET 0x00000600UL #define CORE_PRM_OFFSET 0x00000700UL #define IVAHD_PRM_OFFSET 0x00000F00UL #define CAM_PRM_OFFSET 0x00001000UL #define DSS_PRM_OFFSET 0x00001100UL #define SGX_PRM_OFFSET 0x00001200UL #define L3INIT_PRM_OFFSET 0x00001300UL #define L4PER_PRM_OFFSET 0x00001400UL #define WKUP_PRM_OFFSET 0x00001700UL #define WKUP_CM_OFFSET 0x00001800UL #define EMU_PRM_OFFSET 0x00001900UL #define EMU_CM_OFFSET 0x00001A00UL #define DEVICE_PRM_OFFSET 0x00001B00UL #define INSTR_PRM_OFFSET 0x00001F00UL #define CM_ABE_DSS_SYS_CLKSEL_OFFSET (CKGEN_PRM_OFFSET + 0x0000UL) #define CM_L4_WKUP_CLKSELL_OFFSET (CKGEN_PRM_OFFSET + 0x0008UL) #define CM_ABE_PLL_REF_CLKSEL_OFFSET (CKGEN_PRM_OFFSET + 0x000CUL) #define CM_SYS_CLKSEL_OFFSET (CKGEN_PRM_OFFSET + 0x0010UL) /** * Address offsets from the CM1 memory region to the top level clock control * registers. */ #define CKGEN_CM1_OFFSET 0x00000100UL #define MPU_CM1_OFFSET 0x00000300UL #define DSP_CM1_OFFSET 0x00000400UL #define ABE_CM1_OFFSET 0x00000500UL #define RESTORE_CM1_OFFSET 0x00000E00UL #define INSTR_CM1_OFFSET 0x00000F00UL #define CM_CLKSEL_DPLL_MPU (CKGEN_CM1_OFFSET + 0x006CUL) /** * Address offsets from the CM2 memory region to the top level clock control * registers. */ #define INTRCONN_SOCKET_CM2_OFFSET 0x00000000UL #define CKGEN_CM2_OFFSET 0x00000100UL #define ALWAYS_ON_CM2_OFFSET 0x00000600UL #define CORE_CM2_OFFSET 0x00000700UL #define IVAHD_CM2_OFFSET 0x00000F00UL #define CAM_CM2_OFFSET 0x00001000UL #define DSS_CM2_OFFSET 0x00001100UL #define SGX_CM2_OFFSET 0x00001200UL #define L3INIT_CM2_OFFSET 0x00001300UL #define L4PER_CM2_OFFSET 0x00001400UL #define RESTORE_CM2_OFFSET 0x00001E00UL #define INSTR_CM2_OFFSET 0x00001F00UL #define CLKCTRL_MODULEMODE_MASK 0x00000003UL #define CLKCTRL_MODULEMODE_DISABLE 0x00000000UL #define CLKCTRL_MODULEMODE_AUTO 0x00000001UL #define CLKCTRL_MODULEMODE_ENABLE 0x00000001UL #define CLKCTRL_IDLEST_MASK 0x00030000UL #define CLKCTRL_IDLEST_ENABLED 0x00000000UL #define CLKCTRL_IDLEST_WAKING 0x00010000UL #define CLKCTRL_IDLEST_IDLE 0x00020000UL #define CLKCTRL_IDLEST_DISABLED 0x00030000UL static struct ofw_compat_data compat_data[] = { {"ti,omap4-cm1", (uintptr_t)CM1_INSTANCE}, {"ti,omap4-cm2", (uintptr_t)CM2_INSTANCE}, {"ti,omap4-prm", (uintptr_t)PRM_INSTANCE}, {NULL, (uintptr_t)0}, }; struct omap4_prcm_softc { struct resource *sc_res; int sc_rid; int sc_instance; }; static int omap4_clk_generic_activate(struct ti_clock_dev *clkdev); static int omap4_clk_generic_deactivate(struct ti_clock_dev *clkdev); static int omap4_clk_generic_accessible(struct ti_clock_dev *clkdev); static int omap4_clk_generic_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc); static int omap4_clk_generic_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq); static int omap4_clk_gptimer_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc); static int omap4_clk_gptimer_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq); static int omap4_clk_hsmmc_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc); static int omap4_clk_hsmmc_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq); static int omap4_clk_hsusbhost_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc); static int omap4_clk_hsusbhost_activate(struct ti_clock_dev *clkdev); static int omap4_clk_hsusbhost_deactivate(struct ti_clock_dev *clkdev); static int omap4_clk_hsusbhost_accessible(struct ti_clock_dev *clkdev); static int omap4_clk_get_sysclk_freq(struct ti_clock_dev *clkdev, unsigned int *freq); static int omap4_clk_get_arm_fclk_freq(struct ti_clock_dev *clkdev, unsigned int *freq); /** * omap_clk_devmap - Array of clock devices available on OMAP4xxx devices * * This map only defines which clocks are valid and the callback functions * for clock activate, deactivate, etc. It is used by the top level omap_prcm * driver. * * The actual details of the clocks (config registers, bit fields, sources, * etc) are in the private g_omap3_clk_details array below. * */ #define OMAP4_GENERIC_CLOCK_DEV(i) \ { .id = (i), \ .clk_activate = omap4_clk_generic_activate, \ .clk_deactivate = omap4_clk_generic_deactivate, \ .clk_set_source = omap4_clk_generic_set_source, \ .clk_accessible = omap4_clk_generic_accessible, \ .clk_get_source_freq = omap4_clk_generic_get_source_freq, \ .clk_set_source_freq = NULL \ } #define OMAP4_GPTIMER_CLOCK_DEV(i) \ { .id = (i), \ .clk_activate = omap4_clk_generic_activate, \ .clk_deactivate = omap4_clk_generic_deactivate, \ .clk_set_source = omap4_clk_gptimer_set_source, \ .clk_accessible = omap4_clk_generic_accessible, \ .clk_get_source_freq = omap4_clk_gptimer_get_source_freq, \ .clk_set_source_freq = NULL \ } #define OMAP4_HSMMC_CLOCK_DEV(i) \ { .id = (i), \ .clk_activate = omap4_clk_generic_activate, \ .clk_deactivate = omap4_clk_generic_deactivate, \ .clk_set_source = omap4_clk_hsmmc_set_source, \ .clk_accessible = omap4_clk_generic_accessible, \ .clk_get_source_freq = omap4_clk_hsmmc_get_source_freq, \ .clk_set_source_freq = NULL \ } #define OMAP4_HSUSBHOST_CLOCK_DEV(i) \ { .id = (i), \ .clk_activate = omap4_clk_hsusbhost_activate, \ .clk_deactivate = omap4_clk_hsusbhost_deactivate, \ .clk_set_source = omap4_clk_hsusbhost_set_source, \ .clk_accessible = omap4_clk_hsusbhost_accessible, \ .clk_get_source_freq = NULL, \ .clk_set_source_freq = NULL \ } struct ti_clock_dev ti_omap4_clk_devmap[] = { /* System clocks */ { .id = SYS_CLK, .clk_activate = NULL, .clk_deactivate = NULL, .clk_set_source = NULL, .clk_accessible = NULL, .clk_get_source_freq = omap4_clk_get_sysclk_freq, .clk_set_source_freq = NULL, }, /* MPU (ARM) core clocks */ { .id = MPU_CLK, .clk_activate = NULL, .clk_deactivate = NULL, .clk_set_source = NULL, .clk_accessible = NULL, .clk_get_source_freq = omap4_clk_get_arm_fclk_freq, .clk_set_source_freq = NULL, }, /* UART device clocks */ OMAP4_GENERIC_CLOCK_DEV(UART1_CLK), OMAP4_GENERIC_CLOCK_DEV(UART2_CLK), OMAP4_GENERIC_CLOCK_DEV(UART3_CLK), OMAP4_GENERIC_CLOCK_DEV(UART4_CLK), /* Timer device source clocks */ OMAP4_GPTIMER_CLOCK_DEV(TIMER1_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER2_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER3_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER4_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER5_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER6_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER7_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER8_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER9_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER10_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER11_CLK), /* MMC device clocks (MMC1 and MMC2 can have different input clocks) */ OMAP4_HSMMC_CLOCK_DEV(MMC1_CLK), OMAP4_HSMMC_CLOCK_DEV(MMC2_CLK), OMAP4_GENERIC_CLOCK_DEV(MMC3_CLK), OMAP4_GENERIC_CLOCK_DEV(MMC4_CLK), OMAP4_GENERIC_CLOCK_DEV(MMC5_CLK), /* USB HS (high speed TLL, EHCI and OHCI) */ OMAP4_HSUSBHOST_CLOCK_DEV(USBTLL_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBHSHOST_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBFSHOST_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP1_PHY_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP2_PHY_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP1_UTMI_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP2_UTMI_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP1_HSIC_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP2_HSIC_CLK), /* GPIO */ OMAP4_GENERIC_CLOCK_DEV(GPIO1_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO2_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO3_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO4_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO5_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO6_CLK), /* sDMA */ OMAP4_GENERIC_CLOCK_DEV(SDMA_CLK), /* I2C */ OMAP4_GENERIC_CLOCK_DEV(I2C1_CLK), OMAP4_GENERIC_CLOCK_DEV(I2C2_CLK), OMAP4_GENERIC_CLOCK_DEV(I2C3_CLK), OMAP4_GENERIC_CLOCK_DEV(I2C4_CLK), { INVALID_CLK_IDENT, NULL, NULL, NULL, NULL } }; /** * omap4_clk_details - Stores details for all the different clocks supported * * Whenever an operation on a clock is being performed (activated, deactivated, * etc) this array is looked up to find the correct register and bit(s) we * should be modifying. * */ struct omap4_clk_details { clk_ident_t id; uint32_t instance; uint32_t clksel_reg; int32_t src_freq; uint32_t enable_mode; }; #define OMAP4_GENERIC_CLOCK_DETAILS(i, f, di, r, e) \ { .id = (i), \ .instance = (di), \ .clksel_reg = (r), \ .src_freq = (f), \ .enable_mode = (e), \ } static struct omap4_clk_details g_omap4_clk_details[] = { /* UART */ OMAP4_GENERIC_CLOCK_DETAILS(UART1_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0140), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(UART2_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0148), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(UART3_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0150), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(UART4_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0158), CLKCTRL_MODULEMODE_ENABLE), /* General purpose timers */ OMAP4_GENERIC_CLOCK_DETAILS(TIMER1_CLK, -1, PRM_INSTANCE, (WKUP_CM_OFFSET + 0x040), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER2_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x038), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER3_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x040), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER4_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x048), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER5_CLK, -1, CM1_INSTANCE, (ABE_CM1_OFFSET + 0x068), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER6_CLK, -1, CM1_INSTANCE, (ABE_CM1_OFFSET + 0x070), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER7_CLK, -1, CM1_INSTANCE, (ABE_CM1_OFFSET + 0x078), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER8_CLK, -1, CM1_INSTANCE, (ABE_CM1_OFFSET + 0x080), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER9_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x050), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER10_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x028), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER11_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x030), CLKCTRL_MODULEMODE_ENABLE), /* HSMMC (MMC1 and MMC2 can have different input clocks) */ OMAP4_GENERIC_CLOCK_DETAILS(MMC1_CLK, -1, CM2_INSTANCE, (L3INIT_CM2_OFFSET + 0x028), /*CLKCTRL_MODULEMODE_ENABLE*/2), OMAP4_GENERIC_CLOCK_DETAILS(MMC2_CLK, -1, CM2_INSTANCE, (L3INIT_CM2_OFFSET + 0x030), /*CLKCTRL_MODULEMODE_ENABLE*/2), OMAP4_GENERIC_CLOCK_DETAILS(MMC3_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x120), /*CLKCTRL_MODULEMODE_ENABLE*/2), OMAP4_GENERIC_CLOCK_DETAILS(MMC4_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x128), /*CLKCTRL_MODULEMODE_ENABLE*/2), OMAP4_GENERIC_CLOCK_DETAILS(MMC5_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x160), /*CLKCTRL_MODULEMODE_ENABLE*/1), /* GPIO modules */ OMAP4_GENERIC_CLOCK_DETAILS(GPIO1_CLK, -1, PRM_INSTANCE, (WKUP_CM_OFFSET + 0x038), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO2_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x060), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO3_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x068), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO4_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x070), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO5_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x078), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO6_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x080), CLKCTRL_MODULEMODE_AUTO), /* sDMA block */ OMAP4_GENERIC_CLOCK_DETAILS(SDMA_CLK, -1, CM2_INSTANCE, (CORE_CM2_OFFSET + 0x300), CLKCTRL_MODULEMODE_AUTO), /* I2C modules */ OMAP4_GENERIC_CLOCK_DETAILS(I2C1_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0A0), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(I2C2_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0A8), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(I2C3_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0B0), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(I2C4_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0B8), CLKCTRL_MODULEMODE_ENABLE), { INVALID_CLK_IDENT, 0, 0, 0, 0 }, }; /** * MAX_MODULE_ENABLE_WAIT - the number of loops to wait for the module to come * alive. * */ #define MAX_MODULE_ENABLE_WAIT 100 /** * ARRAY_SIZE - Macro to return the number of elements in a static const array. * */ #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0])) /** * omap4_clk_details - writes a 32-bit value to one of the timer registers * @timer: Timer device context * @off: The offset of a register from the timer register address range * @val: The value to write into the register * * * RETURNS: * nothing */ static struct omap4_clk_details* omap4_clk_details(clk_ident_t id) { struct omap4_clk_details *walker; for (walker = g_omap4_clk_details; walker->id != INVALID_CLK_IDENT; walker++) { if (id == walker->id) return (walker); } return NULL; } static struct omap4_prcm_softc * omap4_prcm_get_instance_softc(int module_instance) { int i, maxunit; devclass_t prcm_devclass; device_t dev; struct omap4_prcm_softc *sc; prcm_devclass = devclass_find("omap4_prcm"); maxunit = devclass_get_maxunit(prcm_devclass); for (i = 0; i < maxunit; i++) { dev = devclass_get_device(prcm_devclass, i); sc = device_get_softc(dev); if (sc->sc_instance == module_instance) return (sc); } return (NULL); } /** * omap4_clk_generic_activate - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ static int omap4_clk_generic_activate(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; unsigned int i; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* All the 'generic' clocks have a CLKCTRL register which is more or less * generic - the have at least two fielda called MODULEMODE and IDLEST. */ clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= clk_details->enable_mode; bus_write_4(clk_mem_res, clk_details->clksel_reg, clksel); /* Now poll on the IDLEST register to tell us if the module has come up. * TODO: We need to take into account the parent clocks. */ /* Try MAX_MODULE_ENABLE_WAIT number of times to check if enabled */ for (i = 0; i < MAX_MODULE_ENABLE_WAIT; i++) { clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); if ((clksel & CLKCTRL_IDLEST_MASK) == CLKCTRL_IDLEST_ENABLED) break; DELAY(10); } /* Check the enabled state */ if ((clksel & CLKCTRL_IDLEST_MASK) != CLKCTRL_IDLEST_ENABLED) { printf("Error: failed to enable module with clock %d\n", clkdev->id); printf("Error: 0x%08x => 0x%08x\n", clk_details->clksel_reg, clksel); return (ETIMEDOUT); } return (0); } /** * omap4_clk_generic_deactivate - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ static int omap4_clk_generic_deactivate(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* All the 'generic' clocks have a CLKCTRL register which is more or less * generic - the have at least two fielda called MODULEMODE and IDLEST. */ clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= CLKCTRL_MODULEMODE_DISABLE; bus_write_4(clk_mem_res, clk_details->clksel_reg, clksel); return (0); } /** * omap4_clk_generic_set_source - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ static int omap4_clk_generic_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc) { return (0); } /** * omap4_clk_generic_accessible - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_generic_accessible(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); /* Check the enabled state */ if ((clksel & CLKCTRL_IDLEST_MASK) != CLKCTRL_IDLEST_ENABLED) return (0); return (1); } /** * omap4_clk_generic_get_source_freq - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_generic_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq ) { struct omap4_clk_details* clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); /* Simply return the stored frequency */ if (freq) *freq = (unsigned int)clk_details->src_freq; return (0); } /** * omap4_clk_gptimer_set_source - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_gptimer_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* TODO: Implement */ return (0); } /** * omap4_clk_gptimer_get_source_freq - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_gptimer_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq ) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; unsigned int src_freq; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* Need to read the CLKSEL field to determine the clock source */ clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); if (clksel & (0x1UL << 24)) src_freq = FREQ_32KHZ; else omap4_clk_get_sysclk_freq(NULL, &src_freq); /* Return the frequency */ if (freq) *freq = src_freq; return (0); } /** * omap4_clk_hsmmc_set_source - sets the source clock (freq) * @clkdev: pointer to the clockdev structure (id field will contain clock id) * * The MMC 1 and 2 clocks can be source from either a 64MHz or 96MHz clock. * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_hsmmc_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* For MMC modules 3, 4 & 5 you can't change the freq, it's always 48MHz */ if ((clkdev->id == MMC3_CLK) || (clkdev->id == MMC4_CLK) || (clkdev->id == MMC5_CLK)) { if (clksrc != F48MHZ_CLK) return (EINVAL); return 0; } clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); /* Bit 24 is set if 96MHz clock or cleared for 64MHz clock */ if (clksrc == F64MHZ_CLK) clksel &= ~(0x1UL << 24); else if (clksrc == F96MHZ_CLK) clksel |= (0x1UL << 24); else return (EINVAL); bus_write_4(clk_mem_res, clk_details->clksel_reg, clksel); return (0); } /** * omap4_clk_hsmmc_get_source_freq - checks if a module is accessible * @clkdev: pointer to the clockdev structure (id field will contain clock id) * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_hsmmc_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq ) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; unsigned int src_freq; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); switch (clkdev->id) { case MMC1_CLK: case MMC2_CLK: /* Need to read the CLKSEL field to determine the clock source */ clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); if (clksel & (0x1UL << 24)) src_freq = FREQ_96MHZ; else src_freq = FREQ_64MHZ; break; case MMC3_CLK: case MMC4_CLK: case MMC5_CLK: src_freq = FREQ_48MHZ; break; default: return (EINVAL); } /* Return the frequency */ if (freq) *freq = src_freq; return (0); } /** * omap4_clk_get_sysclk_freq - gets the sysclk frequency * @sc: pointer to the clk module/device context * * Read the clocking information from the power-control/boot-strap registers, * and stored in two global variables. * * RETURNS: * nothing, values are saved in global variables */ static int omap4_clk_get_sysclk_freq(struct ti_clock_dev *clkdev, unsigned int *freq) { uint32_t clksel; uint32_t sysclk; struct omap4_prcm_softc *sc; sc = omap4_prcm_get_instance_softc(PRM_INSTANCE); if (sc == NULL) return ENXIO; /* Read the input clock freq from the configuration register (CM_SYS_CLKSEL) */ clksel = bus_read_4(sc->sc_res, CM_SYS_CLKSEL_OFFSET); switch (clksel & 0x7) { case 0x1: /* 12Mhz */ sysclk = 12000000; break; case 0x3: /* 16.8Mhz */ sysclk = 16800000; break; case 0x4: /* 19.2Mhz */ sysclk = 19200000; break; case 0x5: /* 26Mhz */ sysclk = 26000000; break; case 0x7: /* 38.4Mhz */ sysclk = 38400000; break; default: panic("%s: Invalid clock freq", __func__); } /* Return the value */ if (freq) *freq = sysclk; return (0); } /** * omap4_clk_get_arm_fclk_freq - gets the MPU clock frequency * @clkdev: ignored * @freq: pointer which upon return will contain the freq in hz * @mem_res: array of allocated memory resources * * Reads the frequency setting information registers and returns the value * in the freq variable. * * RETURNS: * returns 0 on success, a positive error code on failure. */ static int omap4_clk_get_arm_fclk_freq(struct ti_clock_dev *clkdev, unsigned int *freq) { uint32_t clksel; uint32_t pll_mult, pll_div; uint32_t mpuclk, sysclk; struct omap4_prcm_softc *sc; sc = omap4_prcm_get_instance_softc(CM1_INSTANCE); if (sc == NULL) return ENXIO; /* Read the clksel register which contains the DPLL multiple and divide * values. These are applied to the sysclk. */ clksel = bus_read_4(sc->sc_res, CM_CLKSEL_DPLL_MPU); pll_mult = ((clksel >> 8) & 0x7ff); pll_div = (clksel & 0x7f) + 1; /* Get the system clock freq */ omap4_clk_get_sysclk_freq(NULL, &sysclk); /* Calculate the MPU freq */ mpuclk = ((uint64_t)sysclk * pll_mult) / pll_div; /* Return the value */ if (freq) *freq = mpuclk; return (0); } /** * omap4_clk_hsusbhost_activate - activates the USB clocks for the given module * @clkdev: pointer to the clock device structure. * @mem_res: array of memory resources allocated by the top level PRCM driver. * * The USB clocking setup seems to be a bit more tricky than the other modules, * to start with the clocking diagram for the HS host module shows 13 different * clocks. So to try and make it easier to follow the clocking activation * and deactivation is handled in it's own set of callbacks. * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ struct dpll_param { unsigned int m; unsigned int n; unsigned int m2; unsigned int m3; unsigned int m4; unsigned int m5; unsigned int m6; unsigned int m7; }; /* USB parameters */ struct dpll_param usb_dpll_param[7] = { /* 12M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 13M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 16.8M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 19.2M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 26M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 27M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 38.4M values */ #ifdef CONFIG_OMAP4_SDC {0x32, 0x1, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0}, #else {0x32, 0x1, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0}, #endif }; static int omap4_clk_hsusbhost_activate(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct resource* clk_mem_res; uint32_t clksel_reg_off; uint32_t clksel; unsigned int i; sc = omap4_prcm_get_instance_softc(CM2_INSTANCE); if (sc == NULL) return ENXIO; switch (clkdev->id) { case USBTLL_CLK: /* For the USBTLL module we need to enable the following clocks: * - INIT_L4_ICLK (will be enabled by bootloader) * - TLL_CH0_FCLK * - TLL_CH1_FCLK */ /* We need the CM_L3INIT_HSUSBTLL_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x68; /* Enable the module and also enable the optional func clocks for * channels 0 & 1 (is this needed ?) */ clksel = bus_read_4(clk_mem_res, clksel_reg_off); clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= CLKCTRL_MODULEMODE_ENABLE; clksel |= (0x1 << 8); /* USB-HOST optional clock: USB_CH0_CLK */ clksel |= (0x1 << 9); /* USB-HOST optional clock: USB_CH1_CLK */ break; case USBHSHOST_CLK: case USBP1_PHY_CLK: case USBP2_PHY_CLK: case USBP1_UTMI_CLK: case USBP2_UTMI_CLK: case USBP1_HSIC_CLK: case USBP2_HSIC_CLK: /* For the USB HS HOST module we need to enable the following clocks: * - INIT_L4_ICLK (will be enabled by bootloader) * - INIT_L3_ICLK (will be enabled by bootloader) * - INIT_48MC_FCLK * - UTMI_ROOT_GFCLK (UTMI only, create a new clock for that ?) * - UTMI_P1_FCLK (UTMI only, create a new clock for that ?) * - UTMI_P2_FCLK (UTMI only, create a new clock for that ?) * - HSIC_P1_60 (HSIC only, create a new clock for that ?) * - HSIC_P1_480 (HSIC only, create a new clock for that ?) * - HSIC_P2_60 (HSIC only, create a new clock for that ?) * - HSIC_P2_480 (HSIC only, create a new clock for that ?) */ /* We need the CM_L3INIT_HSUSBHOST_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x58; clksel = bus_read_4(clk_mem_res, clksel_reg_off); /* Enable the module and also enable the optional func clocks */ if (clkdev->id == USBHSHOST_CLK) { clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= /*CLKCTRL_MODULEMODE_ENABLE*/2; clksel |= (0x1 << 15); /* USB-HOST clock control: FUNC48MCLK */ } else if (clkdev->id == USBP1_UTMI_CLK) clksel |= (0x1 << 8); /* UTMI_P1_CLK */ else if (clkdev->id == USBP2_UTMI_CLK) clksel |= (0x1 << 9); /* UTMI_P2_CLK */ else if (clkdev->id == USBP1_HSIC_CLK) clksel |= (0x5 << 11); /* HSIC60M_P1_CLK + HSIC480M_P1_CLK */ else if (clkdev->id == USBP2_HSIC_CLK) clksel |= (0x5 << 12); /* HSIC60M_P2_CLK + HSIC480M_P2_CLK */ break; default: return (EINVAL); } bus_write_4(clk_mem_res, clksel_reg_off, clksel); /* Try MAX_MODULE_ENABLE_WAIT number of times to check if enabled */ for (i = 0; i < MAX_MODULE_ENABLE_WAIT; i++) { clksel = bus_read_4(clk_mem_res, clksel_reg_off); if ((clksel & CLKCTRL_IDLEST_MASK) == CLKCTRL_IDLEST_ENABLED) break; } /* Check the enabled state */ if ((clksel & CLKCTRL_IDLEST_MASK) != CLKCTRL_IDLEST_ENABLED) { printf("Error: HERE failed to enable module with clock %d\n", clkdev->id); printf("Error: 0x%08x => 0x%08x\n", clksel_reg_off, clksel); return (ETIMEDOUT); } return (0); } /** * omap4_clk_generic_deactivate - checks if a module is accessible * @clkdev: pointer to the clock device structure. * @mem_res: array of memory resources allocated by the top level PRCM driver. * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ static int omap4_clk_hsusbhost_deactivate(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct resource* clk_mem_res; uint32_t clksel_reg_off; uint32_t clksel; sc = omap4_prcm_get_instance_softc(CM2_INSTANCE); if (sc == NULL) return ENXIO; switch (clkdev->id) { case USBTLL_CLK: /* We need the CM_L3INIT_HSUSBTLL_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x68; clksel = bus_read_4(clk_mem_res, clksel_reg_off); clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= CLKCTRL_MODULEMODE_DISABLE; break; case USBHSHOST_CLK: case USBP1_PHY_CLK: case USBP2_PHY_CLK: case USBP1_UTMI_CLK: case USBP2_UTMI_CLK: case USBP1_HSIC_CLK: case USBP2_HSIC_CLK: /* For the USB HS HOST module we need to enable the following clocks: * - INIT_L4_ICLK (will be enabled by bootloader) * - INIT_L3_ICLK (will be enabled by bootloader) * - INIT_48MC_FCLK * - UTMI_ROOT_GFCLK (UTMI only, create a new clock for that ?) * - UTMI_P1_FCLK (UTMI only, create a new clock for that ?) * - UTMI_P2_FCLK (UTMI only, create a new clock for that ?) * - HSIC_P1_60 (HSIC only, create a new clock for that ?) * - HSIC_P1_480 (HSIC only, create a new clock for that ?) * - HSIC_P2_60 (HSIC only, create a new clock for that ?) * - HSIC_P2_480 (HSIC only, create a new clock for that ?) */ /* We need the CM_L3INIT_HSUSBHOST_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x58; clksel = bus_read_4(clk_mem_res, clksel_reg_off); /* Enable the module and also enable the optional func clocks */ if (clkdev->id == USBHSHOST_CLK) { clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= CLKCTRL_MODULEMODE_DISABLE; clksel &= ~(0x1 << 15); /* USB-HOST clock control: FUNC48MCLK */ } else if (clkdev->id == USBP1_UTMI_CLK) clksel &= ~(0x1 << 8); /* UTMI_P1_CLK */ else if (clkdev->id == USBP2_UTMI_CLK) clksel &= ~(0x1 << 9); /* UTMI_P2_CLK */ else if (clkdev->id == USBP1_HSIC_CLK) clksel &= ~(0x5 << 11); /* HSIC60M_P1_CLK + HSIC480M_P1_CLK */ else if (clkdev->id == USBP2_HSIC_CLK) clksel &= ~(0x5 << 12); /* HSIC60M_P2_CLK + HSIC480M_P2_CLK */ break; default: return (EINVAL); } bus_write_4(clk_mem_res, clksel_reg_off, clksel); return (0); } /** * omap4_clk_hsusbhost_accessible - checks if a module is accessible * @clkdev: pointer to the clock device structure. * @mem_res: array of memory resources allocated by the top level PRCM driver. * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 if module is not enable, 1 if module is enabled or a negative * error code on failure. */ static int omap4_clk_hsusbhost_accessible(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct resource* clk_mem_res; uint32_t clksel_reg_off; uint32_t clksel; sc = omap4_prcm_get_instance_softc(CM2_INSTANCE); if (sc == NULL) return ENXIO; if (clkdev->id == USBTLL_CLK) { /* We need the CM_L3INIT_HSUSBTLL_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x68; } else if (clkdev->id == USBHSHOST_CLK) { /* We need the CM_L3INIT_HSUSBHOST_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x58; } else { return (EINVAL); } clksel = bus_read_4(clk_mem_res, clksel_reg_off); /* Check the enabled state */ if ((clksel & CLKCTRL_IDLEST_MASK) != CLKCTRL_IDLEST_ENABLED) return (0); return (1); } /** * omap4_clk_hsusbhost_set_source - sets the source clocks * @clkdev: pointer to the clock device structure. * @clksrc: the clock source ID for the given clock. * @mem_res: array of memory resources allocated by the top level PRCM driver. * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: - * Returns 0 if sucessful otherwise a negative error code on failure. + * Returns 0 if successful otherwise a negative error code on failure. */ static int omap4_clk_hsusbhost_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc) { struct omap4_prcm_softc *sc; struct resource* clk_mem_res; uint32_t clksel_reg_off; uint32_t clksel; unsigned int bit; sc = omap4_prcm_get_instance_softc(CM2_INSTANCE); if (sc == NULL) return ENXIO; if (clkdev->id == USBP1_PHY_CLK) bit = 24; else if (clkdev->id != USBP2_PHY_CLK) bit = 25; else return (EINVAL); /* We need the CM_L3INIT_HSUSBHOST_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x58; clksel = bus_read_4(clk_mem_res, clksel_reg_off); /* Set the clock source to either external or internal */ if (clksrc == EXT_CLK) clksel |= (0x1 << bit); else clksel &= ~(0x1 << bit); bus_write_4(clk_mem_res, clksel_reg_off, clksel); return (0); } #define PRM_RSTCTRL 0x1b00 #define PRM_RSTCTRL_RESET 0x2 static void omap4_prcm_reset(void) { struct omap4_prcm_softc *sc; sc = omap4_prcm_get_instance_softc(PRM_INSTANCE); if (sc == NULL) return; bus_write_4(sc->sc_res, PRM_RSTCTRL, bus_read_4(sc->sc_res, PRM_RSTCTRL) | PRM_RSTCTRL_RESET); bus_read_4(sc->sc_res, PRM_RSTCTRL); } /** * omap4_prcm_probe - probe function for the driver * @dev: prcm device handle * * Simply sets the name of the driver module. * * LOCKING: * None * * RETURNS: * Always returns 0 */ static int omap4_prcm_probe(device_t dev) { const struct ofw_compat_data *ocd; if (!ofw_bus_status_okay(dev)) return (ENXIO); ocd = ofw_bus_search_compatible(dev, compat_data); if ((int)ocd->ocd_data == 0) return (ENXIO); switch ((int)ocd->ocd_data) { case PRM_INSTANCE: device_set_desc(dev, "TI OMAP Power, Reset and Clock Management (PRM)"); break; case CM1_INSTANCE: device_set_desc(dev, "TI OMAP Power, Reset and Clock Management (C1)"); break; case CM2_INSTANCE: device_set_desc(dev, "TI OMAP Power, Reset and Clock Management (C2)"); break; default: device_printf(dev, "unknown instance type: %d\n", (int)ocd->ocd_data); return (ENXIO); } return (BUS_PROBE_DEFAULT); } /** * omap_prcm_attach - attach function for the driver * @dev: prcm device handle * * Allocates and sets up the driver context, this simply entails creating a * bus mappings for the PRCM register set. * * LOCKING: * None * * RETURNS: * Always returns 0 */ extern uint32_t platform_arm_tmr_freq; static int omap4_prcm_attach(device_t dev) { struct omap4_prcm_softc *sc; unsigned int freq; const struct ofw_compat_data *ocd; sc = device_get_softc(dev); ocd = ofw_bus_search_compatible(dev, compat_data); sc->sc_instance = (int)ocd->ocd_data; sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rid, RF_ACTIVE); if (sc->sc_res == NULL) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } ti_cpu_reset = omap4_prcm_reset; /* * In order to determine ARM frequency we need both RPM and CM1 * instances up and running. So wait until all CRM devices are * initialized. Should be replaced with proper clock framework */ if (device_get_unit(dev) == 2) { omap4_clk_get_arm_fclk_freq(NULL, &freq); arm_tmr_change_frequency(freq / 2); } return (0); } static device_method_t omap4_prcm_methods[] = { DEVMETHOD(device_probe, omap4_prcm_probe), DEVMETHOD(device_attach, omap4_prcm_attach), {0, 0}, }; static driver_t omap4_prcm_driver = { "omap4_prcm", omap4_prcm_methods, sizeof(struct omap4_prcm_softc), }; static devclass_t omap4_prcm_devclass; EARLY_DRIVER_MODULE(omap4_prcm, simplebus, omap4_prcm_driver, omap4_prcm_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_EARLY); MODULE_VERSION(omap4_prcm, 1); Index: head/sys/arm/ti/ti_i2c.c =================================================================== --- head/sys/arm/ti/ti_i2c.c (revision 299068) +++ head/sys/arm/ti/ti_i2c.c (revision 299069) @@ -1,988 +1,988 @@ /*- * Copyright (c) 2011 Ben Gray . * Copyright (c) 2014 Luiz Otavio O Souza . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /** * Driver for the I2C module on the TI SoC. * * This driver is heavily based on the TWI driver for the AT91 (at91_twi.c). * * CAUTION: The I2Ci registers are limited to 16 bit and 8 bit data accesses, * 32 bit data access is not allowed and can corrupt register content. * * This driver currently doesn't use DMA for the transfer, although I hope to * incorporate that sometime in the future. The idea being that for transaction * larger than a certain size the DMA engine is used, for anything less the * normal interrupt/fifo driven option is used. * * * WARNING: This driver uses mtx_sleep and interrupts to perform transactions, * which means you can't do a transaction during startup before the interrupts * have been enabled. Hint - the freebsd function config_intrhook_establish(). */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "iicbus_if.h" /** * I2C device driver context, a pointer to this is stored in the device * driver structure. */ struct ti_i2c_softc { device_t sc_dev; clk_ident_t clk_id; struct resource* sc_irq_res; struct resource* sc_mem_res; device_t sc_iicbus; void* sc_irq_h; struct mtx sc_mtx; struct iic_msg* sc_buffer; int sc_bus_inuse; int sc_buffer_pos; int sc_error; int sc_fifo_trsh; int sc_timeout; uint16_t sc_con_reg; uint16_t sc_rev; }; struct ti_i2c_clock_config { u_int frequency; /* Bus frequency in Hz */ uint8_t psc; /* Fast/Standard mode prescale divider */ uint8_t scll; /* Fast/Standard mode SCL low time */ uint8_t sclh; /* Fast/Standard mode SCL high time */ uint8_t hsscll; /* High Speed mode SCL low time */ uint8_t hssclh; /* High Speed mode SCL high time */ }; #if defined(SOC_OMAP4) /* * OMAP4 i2c bus clock is 96MHz / ((psc + 1) * (scll + 7 + sclh + 5)). * The prescaler values for 100KHz and 400KHz modes come from the table in the * OMAP4 TRM. The table doesn't list 1MHz; these values should give that speed. */ static struct ti_i2c_clock_config ti_omap4_i2c_clock_configs[] = { { 100000, 23, 13, 15, 0, 0}, { 400000, 9, 5, 7, 0, 0}, { 1000000, 3, 5, 7, 0, 0}, /* { 3200000, 1, 113, 115, 7, 10}, - HS mode */ { 0 /* Table terminator */ } }; #endif #if defined(SOC_TI_AM335X) /* * AM335x i2c bus clock is 48MHZ / ((psc + 1) * (scll + 7 + sclh + 5)) * In all cases we prescale the clock to 24MHz as recommended in the manual. */ static struct ti_i2c_clock_config ti_am335x_i2c_clock_configs[] = { { 100000, 1, 111, 117, 0, 0}, { 400000, 1, 23, 25, 0, 0}, { 1000000, 1, 5, 7, 0, 0}, { 0 /* Table terminator */ } }; #endif /** * Locking macros used throughout the driver */ #define TI_I2C_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define TI_I2C_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define TI_I2C_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), \ "ti_i2c", MTX_DEF) #define TI_I2C_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx) #define TI_I2C_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED) #define TI_I2C_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED) #ifdef DEBUG #define ti_i2c_dbg(_sc, fmt, args...) \ device_printf((_sc)->sc_dev, fmt, ##args) #else #define ti_i2c_dbg(_sc, fmt, args...) #endif /** * ti_i2c_read_2 - reads a 16-bit value from one of the I2C registers * @sc: I2C device context * @off: the byte offset within the register bank to read from. * * * LOCKING: * No locking required * * RETURNS: * 16-bit value read from the register. */ static inline uint16_t ti_i2c_read_2(struct ti_i2c_softc *sc, bus_size_t off) { return (bus_read_2(sc->sc_mem_res, off)); } /** * ti_i2c_write_2 - writes a 16-bit value to one of the I2C registers * @sc: I2C device context * @off: the byte offset within the register bank to read from. * @val: the value to write into the register * * LOCKING: * No locking required * * RETURNS: * 16-bit value read from the register. */ static inline void ti_i2c_write_2(struct ti_i2c_softc *sc, bus_size_t off, uint16_t val) { bus_write_2(sc->sc_mem_res, off, val); } static int ti_i2c_transfer_intr(struct ti_i2c_softc* sc, uint16_t status) { int amount, done, i; done = 0; amount = 0; /* Check for the error conditions. */ if (status & I2C_STAT_NACK) { /* No ACK from slave. */ ti_i2c_dbg(sc, "NACK\n"); ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_NACK); sc->sc_error = ENXIO; } else if (status & I2C_STAT_AL) { /* Arbitration lost. */ ti_i2c_dbg(sc, "Arbitration lost\n"); ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_AL); sc->sc_error = ENXIO; } /* Check if we have finished. */ if (status & I2C_STAT_ARDY) { /* Register access ready - transaction complete basically. */ ti_i2c_dbg(sc, "ARDY transaction complete\n"); if (sc->sc_error != 0 && sc->sc_buffer->flags & IIC_M_NOSTOP) { ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg | I2C_CON_STP); } ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_ARDY | I2C_STAT_RDR | I2C_STAT_RRDY | I2C_STAT_XDR | I2C_STAT_XRDY); return (1); } if (sc->sc_buffer->flags & IIC_M_RD) { /* Read some data. */ if (status & I2C_STAT_RDR) { /* * Receive draining interrupt - last data received. - * The set FIFO threshold wont be reached to trigger + * The set FIFO threshold won't be reached to trigger * RRDY. */ ti_i2c_dbg(sc, "Receive draining interrupt\n"); /* * Drain the FIFO. Read the pending data in the FIFO. */ amount = sc->sc_buffer->len - sc->sc_buffer_pos; } else if (status & I2C_STAT_RRDY) { /* * Receive data ready interrupt - FIFO has reached the * set threshold. */ ti_i2c_dbg(sc, "Receive data ready interrupt\n"); amount = min(sc->sc_fifo_trsh, sc->sc_buffer->len - sc->sc_buffer_pos); } /* Read the bytes from the fifo. */ for (i = 0; i < amount; i++) sc->sc_buffer->buf[sc->sc_buffer_pos++] = (uint8_t)(ti_i2c_read_2(sc, I2C_REG_DATA) & 0xff); if (status & I2C_STAT_RDR) ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_RDR); if (status & I2C_STAT_RRDY) ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_RRDY); } else { /* Write some data. */ if (status & I2C_STAT_XDR) { /* * Transmit draining interrupt - FIFO level is below * the set threshold and the amount of data still to - * be transferred wont reach the set FIFO threshold. + * be transferred won't reach the set FIFO threshold. */ ti_i2c_dbg(sc, "Transmit draining interrupt\n"); /* * Drain the TX data. Write the pending data in the * FIFO. */ amount = sc->sc_buffer->len - sc->sc_buffer_pos; } else if (status & I2C_STAT_XRDY) { /* * Transmit data ready interrupt - the FIFO level * is below the set threshold. */ ti_i2c_dbg(sc, "Transmit data ready interrupt\n"); amount = min(sc->sc_fifo_trsh, sc->sc_buffer->len - sc->sc_buffer_pos); } /* Write the bytes from the fifo. */ for (i = 0; i < amount; i++) ti_i2c_write_2(sc, I2C_REG_DATA, sc->sc_buffer->buf[sc->sc_buffer_pos++]); if (status & I2C_STAT_XDR) ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_XDR); if (status & I2C_STAT_XRDY) ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_XRDY); } return (done); } /** * ti_i2c_intr - interrupt handler for the I2C module * @dev: i2c device handle * * * * LOCKING: * Called from timer context * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ static void ti_i2c_intr(void *arg) { int done; struct ti_i2c_softc *sc; uint16_t events, status; sc = (struct ti_i2c_softc *)arg; TI_I2C_LOCK(sc); status = ti_i2c_read_2(sc, I2C_REG_STATUS); if (status == 0) { TI_I2C_UNLOCK(sc); return; } /* Save enabled interrupts. */ events = ti_i2c_read_2(sc, I2C_REG_IRQENABLE_SET); /* We only care about enabled interrupts. */ status &= events; done = 0; if (sc->sc_buffer != NULL) done = ti_i2c_transfer_intr(sc, status); else { ti_i2c_dbg(sc, "Transfer interrupt without buffer\n"); sc->sc_error = EINVAL; done = 1; } if (done) /* Wakeup the process that started the transaction. */ wakeup(sc); TI_I2C_UNLOCK(sc); } /** * ti_i2c_transfer - called to perform the transfer * @dev: i2c device handle * @msgs: the messages to send/receive * @nmsgs: the number of messages in the msgs array * * * LOCKING: * Internally locked * * RETURNS: * 0 on function succeeded * EINVAL if invalid message is passed as an arg */ static int ti_i2c_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs) { int err, i, repstart, timeout; struct ti_i2c_softc *sc; uint16_t reg; sc = device_get_softc(dev); TI_I2C_LOCK(sc); /* If the controller is busy wait until it is available. */ while (sc->sc_bus_inuse == 1) mtx_sleep(sc, &sc->sc_mtx, 0, "i2cbuswait", 0); /* Now we have control over the I2C controller. */ sc->sc_bus_inuse = 1; err = 0; repstart = 0; for (i = 0; i < nmsgs; i++) { sc->sc_buffer = &msgs[i]; sc->sc_buffer_pos = 0; sc->sc_error = 0; /* Zero byte transfers aren't allowed. */ if (sc->sc_buffer == NULL || sc->sc_buffer->buf == NULL || sc->sc_buffer->len == 0) { err = EINVAL; break; } /* Check if the i2c bus is free. */ if (repstart == 0) { /* * On repeated start we send the START condition while * the bus _is_ busy. */ timeout = 0; while (ti_i2c_read_2(sc, I2C_REG_STATUS_RAW) & I2C_STAT_BB) { if (timeout++ > 100) { err = EBUSY; goto out; } DELAY(1000); } timeout = 0; } else repstart = 0; if (sc->sc_buffer->flags & IIC_M_NOSTOP) repstart = 1; /* Set the slave address. */ ti_i2c_write_2(sc, I2C_REG_SA, msgs[i].slave >> 1); /* Write the data length. */ ti_i2c_write_2(sc, I2C_REG_CNT, sc->sc_buffer->len); /* Clear the RX and the TX FIFO. */ reg = ti_i2c_read_2(sc, I2C_REG_BUF); reg |= I2C_BUF_RXFIFO_CLR | I2C_BUF_TXFIFO_CLR; ti_i2c_write_2(sc, I2C_REG_BUF, reg); reg = sc->sc_con_reg | I2C_CON_STT; if (repstart == 0) reg |= I2C_CON_STP; if ((sc->sc_buffer->flags & IIC_M_RD) == 0) reg |= I2C_CON_TRX; ti_i2c_write_2(sc, I2C_REG_CON, reg); /* Wait for an event. */ err = mtx_sleep(sc, &sc->sc_mtx, 0, "i2ciowait", sc->sc_timeout); if (err == 0) err = sc->sc_error; if (err) break; } out: if (timeout == 0) { while (ti_i2c_read_2(sc, I2C_REG_STATUS_RAW) & I2C_STAT_BB) { if (timeout++ > 100) break; DELAY(1000); } } /* Put the controller in master mode again. */ if ((ti_i2c_read_2(sc, I2C_REG_CON) & I2C_CON_MST) == 0) ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg); sc->sc_buffer = NULL; sc->sc_bus_inuse = 0; /* Wake up the processes that are waiting for the bus. */ wakeup(sc); TI_I2C_UNLOCK(sc); return (err); } static int ti_i2c_reset(struct ti_i2c_softc *sc, u_char speed) { int timeout; struct ti_i2c_clock_config *clkcfg; u_int busfreq; uint16_t fifo_trsh, reg, scll, sclh; switch (ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: clkcfg = ti_omap4_i2c_clock_configs; break; #endif #ifdef SOC_TI_AM335X case CHIP_AM335X: clkcfg = ti_am335x_i2c_clock_configs; break; #endif default: panic("Unknown TI SoC, unable to reset the i2c"); } /* * If we haven't attached the bus yet, just init at the default slow * speed. This lets us get the hardware initialized enough to attach * the bus which is where the real speed configuration is handled. After * the bus is attached, get the configured speed from it. Search the * configuration table for the best speed we can do that doesn't exceed * the requested speed. */ if (sc->sc_iicbus == NULL) busfreq = 100000; else busfreq = IICBUS_GET_FREQUENCY(sc->sc_iicbus, speed); for (;;) { if (clkcfg[1].frequency == 0 || clkcfg[1].frequency > busfreq) break; clkcfg++; } /* * 23.1.4.3 - HS I2C Software Reset * From OMAP4 TRM at page 4068. * * 1. Ensure that the module is disabled. */ sc->sc_con_reg = 0; ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg); /* 2. Issue a softreset to the controller. */ bus_write_2(sc->sc_mem_res, I2C_REG_SYSC, I2C_REG_SYSC_SRST); /* * 3. Enable the module. * The I2Ci.I2C_SYSS[0] RDONE bit is asserted only after the module * is enabled by setting the I2Ci.I2C_CON[15] I2C_EN bit to 1. */ ti_i2c_write_2(sc, I2C_REG_CON, I2C_CON_I2C_EN); /* 4. Wait for the software reset to complete. */ timeout = 0; while ((ti_i2c_read_2(sc, I2C_REG_SYSS) & I2C_SYSS_RDONE) == 0) { if (timeout++ > 100) return (EBUSY); DELAY(100); } /* * Disable the I2C controller once again, now that the reset has * finished. */ ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg); /* * The following sequence is taken from the OMAP4 TRM at page 4077. * * 1. Enable the functional and interface clocks (see Section * 23.1.5.1.1.1.1). Done at ti_i2c_activate(). * * 2. Program the prescaler to obtain an approximately 12MHz internal * sampling clock (I2Ci_INTERNAL_CLK) by programming the * corresponding value in the I2Ci.I2C_PSC[3:0] PSC field. * This value depends on the frequency of the functional clock * (I2Ci_FCLK). Because this frequency is 96MHz, the * I2Ci.I2C_PSC[7:0] PSC field value is 0x7. */ ti_i2c_write_2(sc, I2C_REG_PSC, clkcfg->psc); /* * 3. Program the I2Ci.I2C_SCLL[7:0] SCLL and I2Ci.I2C_SCLH[7:0] SCLH * bit fields to obtain a bit rate of 100 Kbps, 400 Kbps or 1Mbps. * These values depend on the internal sampling clock frequency * (see Table 23-8). */ scll = clkcfg->scll & I2C_SCLL_MASK; sclh = clkcfg->sclh & I2C_SCLH_MASK; /* * 4. (Optional) Program the I2Ci.I2C_SCLL[15:8] HSSCLL and * I2Ci.I2C_SCLH[15:8] HSSCLH fields to obtain a bit rate of * 400K bps or 3.4M bps (for the second phase of HS mode). These * values depend on the internal sampling clock frequency (see * Table 23-8). * * 5. (Optional) If a bit rate of 3.4M bps is used and the bus line * capacitance exceeds 45 pF, (see Section 18.4.8, PAD Functional * Multiplexing and Configuration). */ switch (ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: if ((clkcfg->hsscll + clkcfg->hssclh) > 0) { scll |= clkcfg->hsscll << I2C_HSSCLL_SHIFT; sclh |= clkcfg->hssclh << I2C_HSSCLH_SHIFT; sc->sc_con_reg |= I2C_CON_OPMODE_HS; } break; #endif } /* Write the selected bit rate. */ ti_i2c_write_2(sc, I2C_REG_SCLL, scll); ti_i2c_write_2(sc, I2C_REG_SCLH, sclh); /* * 6. Configure the Own Address of the I2C controller by storing it in * the I2Ci.I2C_OA0 register. Up to four Own Addresses can be * programmed in the I2Ci.I2C_OAi registers (where i = 0, 1, 2, 3) * for each I2C controller. * * Note: For a 10-bit address, set the corresponding expand Own Address * bit in the I2Ci.I2C_CON register. * * Driver currently always in single master mode so ignore this step. */ /* * 7. Set the TX threshold (in transmitter mode) and the RX threshold * (in receiver mode) by setting the I2Ci.I2C_BUF[5:0]XTRSH field to * (TX threshold - 1) and the I2Ci.I2C_BUF[13:8]RTRSH field to (RX * threshold - 1), where the TX and RX thresholds are greater than * or equal to 1. * * The threshold is set to 5 for now. */ fifo_trsh = (sc->sc_fifo_trsh - 1) & I2C_BUF_TRSH_MASK; reg = fifo_trsh | (fifo_trsh << I2C_BUF_RXTRSH_SHIFT); ti_i2c_write_2(sc, I2C_REG_BUF, reg); /* * 8. Take the I2C controller out of reset by setting the * I2Ci.I2C_CON[15] I2C_EN bit to 1. * * 23.1.5.1.1.1.2 - Initialize the I2C Controller * * To initialize the I2C controller, perform the following steps: * * 1. Configure the I2Ci.I2C_CON register: * . For master or slave mode, set the I2Ci.I2C_CON[10] MST bit * (0: slave, 1: master). * . For transmitter or receiver mode, set the I2Ci.I2C_CON[9] TRX * bit (0: receiver, 1: transmitter). */ /* Enable the I2C controller in master mode. */ sc->sc_con_reg |= I2C_CON_I2C_EN | I2C_CON_MST; ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg); /* * 2. If using an interrupt to transmit/receive data, set the * corresponding bit in the I2Ci.I2C_IE register (the I2Ci.I2C_IE[4] * XRDY_IE bit for the transmit interrupt, the I2Ci.I2C_IE[3] RRDY * bit for the receive interrupt). */ /* Set the interrupts we want to be notified. */ reg = I2C_IE_XDR | /* Transmit draining interrupt. */ I2C_IE_XRDY | /* Transmit Data Ready interrupt. */ I2C_IE_RDR | /* Receive draining interrupt. */ I2C_IE_RRDY | /* Receive Data Ready interrupt. */ I2C_IE_ARDY | /* Register Access Ready interrupt. */ I2C_IE_NACK | /* No Acknowledgment interrupt. */ I2C_IE_AL; /* Arbitration lost interrupt. */ /* Enable the interrupts. */ ti_i2c_write_2(sc, I2C_REG_IRQENABLE_SET, reg); /* * 3. If using DMA to receive/transmit data, set to 1 the corresponding * bit in the I2Ci.I2C_BUF register (the I2Ci.I2C_BUF[15] RDMA_EN * bit for the receive DMA channel, the I2Ci.I2C_BUF[7] XDMA_EN bit * for the transmit DMA channel). * * Not using DMA for now, so ignore this. */ return (0); } static int ti_i2c_iicbus_reset(device_t dev, u_char speed, u_char addr, u_char *oldaddr) { struct ti_i2c_softc *sc; int err; sc = device_get_softc(dev); TI_I2C_LOCK(sc); err = ti_i2c_reset(sc, speed); TI_I2C_UNLOCK(sc); if (err) return (err); return (IIC_ENOADDR); } static int ti_i2c_activate(device_t dev) { int err; struct ti_i2c_softc *sc; sc = (struct ti_i2c_softc*)device_get_softc(dev); /* * 1. Enable the functional and interface clocks (see Section * 23.1.5.1.1.1.1). */ err = ti_prcm_clk_enable(sc->clk_id); if (err) return (err); return (ti_i2c_reset(sc, IIC_UNKNOWN)); } /** * ti_i2c_deactivate - deactivates the controller and releases resources * @dev: i2c device handle * * * * LOCKING: * Assumed called in an atomic context. * * RETURNS: * nothing */ static void ti_i2c_deactivate(device_t dev) { struct ti_i2c_softc *sc = device_get_softc(dev); /* Disable the controller - cancel all transactions. */ ti_i2c_write_2(sc, I2C_REG_IRQENABLE_CLR, 0xffff); ti_i2c_write_2(sc, I2C_REG_STATUS, 0xffff); ti_i2c_write_2(sc, I2C_REG_CON, 0); /* Release the interrupt handler. */ if (sc->sc_irq_h != NULL) { bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_irq_h); sc->sc_irq_h = NULL; } bus_generic_detach(sc->sc_dev); /* Unmap the I2C controller registers. */ if (sc->sc_mem_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); sc->sc_mem_res = NULL; } /* Release the IRQ resource. */ if (sc->sc_irq_res != NULL) { bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res); sc->sc_irq_res = NULL; } /* Finally disable the functional and interface clocks. */ ti_prcm_clk_disable(sc->clk_id); } static int ti_i2c_sysctl_clk(SYSCTL_HANDLER_ARGS) { int clk, psc, sclh, scll; struct ti_i2c_softc *sc; sc = arg1; TI_I2C_LOCK(sc); /* Get the system prescaler value. */ psc = (int)ti_i2c_read_2(sc, I2C_REG_PSC) + 1; /* Get the bitrate. */ scll = (int)ti_i2c_read_2(sc, I2C_REG_SCLL) & I2C_SCLL_MASK; sclh = (int)ti_i2c_read_2(sc, I2C_REG_SCLH) & I2C_SCLH_MASK; clk = I2C_CLK / psc / (scll + 7 + sclh + 5); TI_I2C_UNLOCK(sc); return (sysctl_handle_int(oidp, &clk, 0, req)); } static int ti_i2c_sysctl_timeout(SYSCTL_HANDLER_ARGS) { struct ti_i2c_softc *sc; unsigned int val; int err; sc = arg1; /* * MTX_DEF lock can't be held while doing uimove in * sysctl_handle_int */ TI_I2C_LOCK(sc); val = sc->sc_timeout; TI_I2C_UNLOCK(sc); err = sysctl_handle_int(oidp, &val, 0, req); /* Write request? */ if ((err == 0) && (req->newptr != NULL)) { TI_I2C_LOCK(sc); sc->sc_timeout = val; TI_I2C_UNLOCK(sc); } return (err); } static int ti_i2c_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "ti,omap4-i2c")) return (ENXIO); device_set_desc(dev, "TI I2C Controller"); return (0); } static int ti_i2c_attach(device_t dev) { int err, rid; phandle_t node; struct ti_i2c_softc *sc; struct sysctl_ctx_list *ctx; struct sysctl_oid_list *tree; uint16_t fifosz; sc = device_get_softc(dev); sc->sc_dev = dev; /* Get the i2c device id from FDT. */ node = ofw_bus_get_node(dev); /* i2c ti,hwmods bindings is special: it start with index 1 */ sc->clk_id = ti_hwmods_get_clock(dev); if (sc->clk_id == INVALID_CLK_IDENT) { device_printf(dev, "failed to get device id using ti,hwmod\n"); return (ENXIO); } /* Get the memory resource for the register mapping. */ rid = 0; sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem_res == NULL) { device_printf(dev, "Cannot map registers.\n"); return (ENXIO); } /* Allocate our IRQ resource. */ rid = 0; sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | RF_SHAREABLE); if (sc->sc_irq_res == NULL) { bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); device_printf(dev, "Cannot allocate interrupt.\n"); return (ENXIO); } TI_I2C_LOCK_INIT(sc); /* First of all, we _must_ activate the H/W. */ err = ti_i2c_activate(dev); if (err) { device_printf(dev, "ti_i2c_activate failed\n"); goto out; } /* Read the version number of the I2C module */ sc->sc_rev = ti_i2c_read_2(sc, I2C_REG_REVNB_HI) & 0xff; /* Get the fifo size. */ fifosz = ti_i2c_read_2(sc, I2C_REG_BUFSTAT); fifosz >>= I2C_BUFSTAT_FIFODEPTH_SHIFT; fifosz &= I2C_BUFSTAT_FIFODEPTH_MASK; device_printf(dev, "I2C revision %d.%d FIFO size: %d bytes\n", sc->sc_rev >> 4, sc->sc_rev & 0xf, 8 << fifosz); /* Set the FIFO threshold to 5 for now. */ sc->sc_fifo_trsh = 5; /* Set I2C bus timeout */ sc->sc_timeout = 5*hz; ctx = device_get_sysctl_ctx(dev); tree = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "i2c_clock", CTLFLAG_RD | CTLTYPE_UINT | CTLFLAG_MPSAFE, sc, 0, ti_i2c_sysctl_clk, "IU", "I2C bus clock"); SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "i2c_timeout", CTLFLAG_RW | CTLTYPE_UINT | CTLFLAG_MPSAFE, sc, 0, ti_i2c_sysctl_timeout, "IU", "I2C bus timeout (in ticks)"); /* Activate the interrupt. */ err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, ti_i2c_intr, sc, &sc->sc_irq_h); if (err) goto out; /* Attach the iicbus. */ if ((sc->sc_iicbus = device_add_child(dev, "iicbus", -1)) == NULL) { device_printf(dev, "could not allocate iicbus instance\n"); err = ENXIO; goto out; } /* Probe and attach the iicbus */ bus_generic_attach(dev); out: if (err) { ti_i2c_deactivate(dev); TI_I2C_LOCK_DESTROY(sc); } return (err); } static int ti_i2c_detach(device_t dev) { struct ti_i2c_softc *sc; int rv; sc = device_get_softc(dev); ti_i2c_deactivate(dev); TI_I2C_LOCK_DESTROY(sc); if (sc->sc_iicbus && (rv = device_delete_child(dev, sc->sc_iicbus)) != 0) return (rv); return (0); } static phandle_t ti_i2c_get_node(device_t bus, device_t dev) { /* Share controller node with iibus device. */ return (ofw_bus_get_node(bus)); } static device_method_t ti_i2c_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ti_i2c_probe), DEVMETHOD(device_attach, ti_i2c_attach), DEVMETHOD(device_detach, ti_i2c_detach), /* Bus interface */ DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_alloc_resource, bus_generic_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource), DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), /* OFW methods */ DEVMETHOD(ofw_bus_get_node, ti_i2c_get_node), /* iicbus interface */ DEVMETHOD(iicbus_callback, iicbus_null_callback), DEVMETHOD(iicbus_reset, ti_i2c_iicbus_reset), DEVMETHOD(iicbus_transfer, ti_i2c_transfer), DEVMETHOD_END }; static driver_t ti_i2c_driver = { "iichb", ti_i2c_methods, sizeof(struct ti_i2c_softc), }; static devclass_t ti_i2c_devclass; DRIVER_MODULE(ti_iic, simplebus, ti_i2c_driver, ti_i2c_devclass, 0, 0); DRIVER_MODULE(iicbus, ti_iic, iicbus_driver, iicbus_devclass, 0, 0); MODULE_DEPEND(ti_iic, ti_prcm, 1, 1, 1); MODULE_DEPEND(ti_iic, iicbus, 1, 1, 1); Index: head/sys/arm/ti/ti_pinmux.c =================================================================== --- head/sys/arm/ti/ti_pinmux.c (revision 299068) +++ head/sys/arm/ti/ti_pinmux.c (revision 299069) @@ -1,445 +1,445 @@ /* * Copyright (c) 2010 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Ben Gray. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** * Exposes pinmux module to pinctrl-compatible interface */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ti_pinmux.h" struct pincfg { uint32_t reg; uint32_t conf; }; static struct resource_spec ti_pinmux_res_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Control memory window */ { -1, 0 } }; static struct ti_pinmux_softc *ti_pinmux_sc; #define ti_pinmux_read_2(sc, reg) \ bus_space_read_2((sc)->sc_bst, (sc)->sc_bsh, (reg)) #define ti_pinmux_write_2(sc, reg, val) \ bus_space_write_2((sc)->sc_bst, (sc)->sc_bsh, (reg), (val)) #define ti_pinmux_read_4(sc, reg) \ bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg)) #define ti_pinmux_write_4(sc, reg, val) \ bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val)) /** * ti_padconf_devmap - Array of pins, should be defined one per SoC * * This array is typically defined in one of the targeted *_scm_pinumx.c * files and is specific to the given SoC platform. Each entry in the array * corresponds to an individual pin. */ extern const struct ti_pinmux_device ti_pinmux_dev; /** * ti_pinmux_padconf_from_name - searches the list of pads and returns entry * with matching ball name. * @ballname: the name of the ball * * RETURNS: * A pointer to the matching padconf or NULL if the ball wasn't found. */ static const struct ti_pinmux_padconf* ti_pinmux_padconf_from_name(const char *ballname) { const struct ti_pinmux_padconf *padconf; padconf = ti_pinmux_dev.padconf; while (padconf->ballname != NULL) { if (strcmp(ballname, padconf->ballname) == 0) return(padconf); padconf++; } return (NULL); } /** * ti_pinmux_padconf_set_internal - sets the muxmode and state for a pad/pin * @padconf: pointer to the pad structure * @muxmode: the name of the mode to use for the pin, i.e. "uart1_rx" * @state: the state to put the pad/pin in, i.e. PADCONF_PIN_??? * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or already in use. */ static int ti_pinmux_padconf_set_internal(struct ti_pinmux_softc *sc, const struct ti_pinmux_padconf *padconf, const char *muxmode, unsigned int state) { unsigned int mode; uint16_t reg_val; /* populate the new value for the PADCONF register */ reg_val = (uint16_t)(state & ti_pinmux_dev.padconf_sate_mask); /* find the new mode requested */ for (mode = 0; mode < 8; mode++) { if ((padconf->muxmodes[mode] != NULL) && (strcmp(padconf->muxmodes[mode], muxmode) == 0)) { break; } } /* couldn't find the mux mode */ if (mode >= 8) { printf("Invalid mode \"%s\"\n", muxmode); return (EINVAL); } /* set the mux mode */ reg_val |= (uint16_t)(mode & ti_pinmux_dev.padconf_muxmode_mask); if (bootverbose) device_printf(sc->sc_dev, "setting internal %x for %s\n", reg_val, muxmode); /* write the register value (16-bit writes) */ ti_pinmux_write_2(sc, padconf->reg_off, reg_val); return (0); } /** * ti_pinmux_padconf_set - sets the muxmode and state for a pad/pin * @padname: the name of the pad, i.e. "c12" * @muxmode: the name of the mode to use for the pin, i.e. "uart1_rx" * @state: the state to put the pad/pin in, i.e. PADCONF_PIN_??? * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or already in use. */ int ti_pinmux_padconf_set(const char *padname, const char *muxmode, unsigned int state) { const struct ti_pinmux_padconf *padconf; if (!ti_pinmux_sc) return (ENXIO); /* find the pin in the devmap */ padconf = ti_pinmux_padconf_from_name(padname); if (padconf == NULL) return (EINVAL); return (ti_pinmux_padconf_set_internal(ti_pinmux_sc, padconf, muxmode, state)); } /** * ti_pinmux_padconf_get - gets the muxmode and state for a pad/pin * @padname: the name of the pad, i.e. "c12" * @muxmode: upon return will contain the name of the muxmode of the pin * @state: upon return will contain the state of the pad/pin * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or already in use. */ int ti_pinmux_padconf_get(const char *padname, const char **muxmode, unsigned int *state) { const struct ti_pinmux_padconf *padconf; uint16_t reg_val; if (!ti_pinmux_sc) return (ENXIO); /* find the pin in the devmap */ padconf = ti_pinmux_padconf_from_name(padname); if (padconf == NULL) return (EINVAL); /* read the register value (16-bit reads) */ reg_val = ti_pinmux_read_2(ti_pinmux_sc, padconf->reg_off); /* save the state */ if (state) *state = (reg_val & ti_pinmux_dev.padconf_sate_mask); /* save the mode */ if (muxmode) *muxmode = padconf->muxmodes[(reg_val & ti_pinmux_dev.padconf_muxmode_mask)]; return (0); } /** * ti_pinmux_padconf_set_gpiomode - converts a pad to GPIO mode. * @gpio: the GPIO pin number (0-195) * @state: the state to put the pad/pin in, i.e. PADCONF_PIN_??? * * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or already in use. */ int ti_pinmux_padconf_set_gpiomode(uint32_t gpio, unsigned int state) { const struct ti_pinmux_padconf *padconf; uint16_t reg_val; if (!ti_pinmux_sc) return (ENXIO); /* find the gpio pin in the padconf array */ padconf = ti_pinmux_dev.padconf; while (padconf->ballname != NULL) { if (padconf->gpio_pin == gpio) break; padconf++; } if (padconf->ballname == NULL) return (EINVAL); /* populate the new value for the PADCONF register */ reg_val = (uint16_t)(state & ti_pinmux_dev.padconf_sate_mask); /* set the mux mode */ reg_val |= (uint16_t)(padconf->gpio_mode & ti_pinmux_dev.padconf_muxmode_mask); /* write the register value (16-bit writes) */ ti_pinmux_write_2(ti_pinmux_sc, padconf->reg_off, reg_val); return (0); } /** * ti_pinmux_padconf_get_gpiomode - gets the current GPIO mode of the pin * @gpio: the GPIO pin number (0-195) * @state: upon return will contain the state * * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or not configured as GPIO. */ int ti_pinmux_padconf_get_gpiomode(uint32_t gpio, unsigned int *state) { const struct ti_pinmux_padconf *padconf; uint16_t reg_val; if (!ti_pinmux_sc) return (ENXIO); /* find the gpio pin in the padconf array */ padconf = ti_pinmux_dev.padconf; while (padconf->ballname != NULL) { if (padconf->gpio_pin == gpio) break; padconf++; } if (padconf->ballname == NULL) return (EINVAL); /* read the current register settings */ reg_val = ti_pinmux_read_2(ti_pinmux_sc, padconf->reg_off); /* check to make sure the pins is configured as GPIO in the first state */ if ((reg_val & ti_pinmux_dev.padconf_muxmode_mask) != padconf->gpio_mode) return (EINVAL); /* read and store the reset of the state, i.e. pull-up, pull-down, etc */ if (state) *state = (reg_val & ti_pinmux_dev.padconf_sate_mask); return (0); } static int ti_pinmux_configure_pins(device_t dev, phandle_t cfgxref) { struct pincfg *cfgtuples, *cfg; phandle_t cfgnode; int i, ntuples; static struct ti_pinmux_softc *sc; sc = device_get_softc(dev); cfgnode = OF_node_from_xref(cfgxref); ntuples = OF_getencprop_alloc(cfgnode, "pinctrl-single,pins", sizeof(*cfgtuples), (void **)&cfgtuples); if (ntuples < 0) return (ENOENT); if (ntuples == 0) return (0); /* Empty property is not an error. */ for (i = 0, cfg = cfgtuples; i < ntuples; i++, cfg++) { if (bootverbose) { char name[32]; OF_getprop(cfgnode, "name", &name, sizeof(name)); printf("%16s: muxreg 0x%04x muxval 0x%02x\n", name, cfg->reg, cfg->conf); } /* write the register value (16-bit writes) */ ti_pinmux_write_2(sc, cfg->reg, cfg->conf); } free(cfgtuples, M_OFWPROP); return (0); } /* * Device part of OMAP SCM driver */ static int ti_pinmux_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "pinctrl-single")) return (ENXIO); if (ti_pinmux_sc) { printf("%s: multiple pinctrl modules in device tree data, ignoring\n", __func__); return (EEXIST); } device_set_desc(dev, "TI Pinmux Module"); return (BUS_PROBE_DEFAULT); } /** * ti_pinmux_attach - attaches the pinmux to the simplebus * @dev: new device * * RETURNS - * Zero on sucess or ENXIO if an error occuried. + * Zero on success or ENXIO if an error occuried. */ static int ti_pinmux_attach(device_t dev) { struct ti_pinmux_softc *sc = device_get_softc(dev); #if 0 if (ti_pinmux_sc) return (ENXIO); #endif sc->sc_dev = dev; if (bus_alloc_resources(dev, ti_pinmux_res_spec, sc->sc_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } sc->sc_bst = rman_get_bustag(sc->sc_res[0]); sc->sc_bsh = rman_get_bushandle(sc->sc_res[0]); if (ti_pinmux_sc == NULL) ti_pinmux_sc = sc; fdt_pinctrl_register(dev, "pinctrl-single,pins"); fdt_pinctrl_configure_tree(dev); return (0); } static device_method_t ti_pinmux_methods[] = { DEVMETHOD(device_probe, ti_pinmux_probe), DEVMETHOD(device_attach, ti_pinmux_attach), /* fdt_pinctrl interface */ DEVMETHOD(fdt_pinctrl_configure, ti_pinmux_configure_pins), { 0, 0 } }; static driver_t ti_pinmux_driver = { "ti_pinmux", ti_pinmux_methods, sizeof(struct ti_pinmux_softc), }; static devclass_t ti_pinmux_devclass; DRIVER_MODULE(ti_pinmux, simplebus, ti_pinmux_driver, ti_pinmux_devclass, 0, 0); Index: head/sys/arm/ti/ti_prcm.c =================================================================== --- head/sys/arm/ti/ti_prcm.c (revision 299068) +++ head/sys/arm/ti/ti_prcm.c (revision 299069) @@ -1,357 +1,357 @@ /* * Copyright (c) 2010 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Ben Gray. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** - * Power, Reset and Clock Managment Module + * Power, Reset and Clock Management Module * * This is a very simple driver wrapper around the PRCM set of registers in * the OMAP3 chip. It allows you to turn on and off things like the functional * and interface clocks to the various on-chip modules. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /** * ti_*_clk_devmap - Array of clock devices, should be defined one per SoC * * This array is typically defined in one of the targeted *_prcm_clk.c * files and is specific to the given SoC platform. Each entry in the array * corresponds to an individual clock device. */ extern struct ti_clock_dev ti_omap4_clk_devmap[]; extern struct ti_clock_dev ti_am335x_clk_devmap[]; /** * ti_prcm_clk_dev - returns a pointer to the clock device with given id * @clk: the ID of the clock device to get * * Simply iterates through the clk_devmap global array and returns a pointer * to the clock device if found. * * LOCKING: * None * * RETURNS: * The pointer to the clock device on success, on failure NULL is returned. */ static struct ti_clock_dev * ti_prcm_clk_dev(clk_ident_t clk) { struct ti_clock_dev *clk_dev; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = NULL; switch(ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: clk_dev = &(ti_omap4_clk_devmap[0]); break; #endif #ifdef SOC_TI_AM335X case CHIP_AM335X: clk_dev = &(ti_am335x_clk_devmap[0]); break; #endif } if (clk_dev == NULL) panic("No clock devmap found"); while (clk_dev->id != INVALID_CLK_IDENT) { if (clk_dev->id == clk) { return (clk_dev); } clk_dev++; } /* Sanity check we managed to find the clock */ printf("ti_prcm: Failed to find clock device (%d)\n", clk); return (NULL); } /** * ti_prcm_clk_valid - enables a clock for a particular module * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK. * * This function can enable either a functional or interface clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_valid(clk_ident_t clk) { int ret = 0; if (ti_prcm_clk_dev(clk) == NULL) ret = EINVAL; return (ret); } /** * ti_prcm_clk_enable - enables a clock for a particular module * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK. * * This function can enable either a functional or interface clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_enable(clk_ident_t clk) { struct ti_clock_dev *clk_dev; int ret; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Activate the clock */ if (clk_dev->clk_activate) ret = clk_dev->clk_activate(clk_dev); else ret = EINVAL; return (ret); } /** * ti_prcm_clk_disable - disables a clock for a particular module * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK. * * This function can enable either a functional or interface clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_disable(clk_ident_t clk) { struct ti_clock_dev *clk_dev; int ret; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Activate the clock */ if (clk_dev->clk_deactivate) ret = clk_dev->clk_deactivate(clk_dev); else ret = EINVAL; return (ret); } /** * ti_prcm_clk_set_source - sets the source * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK. * * This function can enable either a functional or interface clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_set_source(clk_ident_t clk, clk_src_t clksrc) { struct ti_clock_dev *clk_dev; int ret; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Activate the clock */ if (clk_dev->clk_set_source) ret = clk_dev->clk_set_source(clk_dev, clksrc); else ret = EINVAL; return (ret); } /** * ti_prcm_clk_get_source_freq - gets the source clock frequency * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * @freq: pointer to an integer that upon return will contain the src freq * * This function returns the frequency of the source clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_get_source_freq(clk_ident_t clk, unsigned int *freq) { struct ti_clock_dev *clk_dev; int ret; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Get the source frequency of the clock */ if (clk_dev->clk_get_source_freq) ret = clk_dev->clk_get_source_freq(clk_dev, freq); else ret = EINVAL; return (ret); } /** * ti_prcm_clk_set_source_freq - sets the source clock frequency as close to freq as possible * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * @freq: requested freq * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_set_source_freq(clk_ident_t clk, unsigned int freq) { struct ti_clock_dev *clk_dev; int ret; clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Get the source frequency of the clock */ if (clk_dev->clk_set_source_freq) ret = clk_dev->clk_set_source_freq(clk_dev, freq); else ret = EINVAL; return (ret); } Index: head/sys/arm/ti/ti_scm.c =================================================================== --- head/sys/arm/ti/ti_scm.c (revision 299068) +++ head/sys/arm/ti/ti_scm.c (revision 299069) @@ -1,174 +1,174 @@ /* * Copyright (c) 2010 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Ben Gray. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** * SCM - System Control Module * * Hopefully in the end this module will contain a bunch of utility functions * for configuring and querying the general system control registers, but for * now it only does pin(pad) multiplexing. * * This is different from the GPIO module in that it is used to configure the * pins between modules not just GPIO input/output. * * This file contains the generic top level driver, however it relies on chip * specific settings and therefore expects an array of ti_scm_padconf structs * call ti_padconf_devmap to be located somewhere in the kernel. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ti_scm.h" static struct resource_spec ti_scm_res_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Control memory window */ { -1, 0 } }; static struct ti_scm_softc *ti_scm_sc; #define ti_scm_read_4(sc, reg) \ bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg)) #define ti_scm_write_4(sc, reg, val) \ bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val)) /* * Device part of OMAP SCM driver */ static int ti_scm_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "syscon")) return (ENXIO); if (ti_scm_sc) { return (EEXIST); } device_set_desc(dev, "TI Control Module"); return (BUS_PROBE_DEFAULT); } /** * ti_scm_attach - attaches the timer to the simplebus * @dev: new device * * Reserves memory and interrupt resources, stores the softc structure * globally and registers both the timecount and eventtimer objects. * * RETURNS - * Zero on sucess or ENXIO if an error occuried. + * Zero on success or ENXIO if an error occuried. */ static int ti_scm_attach(device_t dev) { struct ti_scm_softc *sc = device_get_softc(dev); sc->sc_dev = dev; if (bus_alloc_resources(dev, ti_scm_res_spec, sc->sc_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* Global timer interface */ sc->sc_bst = rman_get_bustag(sc->sc_res[0]); sc->sc_bsh = rman_get_bushandle(sc->sc_res[0]); ti_scm_sc = sc; return (0); } int ti_scm_reg_read_4(uint32_t reg, uint32_t *val) { if (!ti_scm_sc) return (ENXIO); *val = ti_scm_read_4(ti_scm_sc, reg); return (0); } int ti_scm_reg_write_4(uint32_t reg, uint32_t val) { if (!ti_scm_sc) return (ENXIO); ti_scm_write_4(ti_scm_sc, reg, val); return (0); } static device_method_t ti_scm_methods[] = { DEVMETHOD(device_probe, ti_scm_probe), DEVMETHOD(device_attach, ti_scm_attach), { 0, 0 } }; static driver_t ti_scm_driver = { "ti_scm", ti_scm_methods, sizeof(struct ti_scm_softc), }; static devclass_t ti_scm_devclass; EARLY_DRIVER_MODULE(ti_scm, simplebus, ti_scm_driver, ti_scm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE); Index: head/sys/arm/ti/ti_sdma.c =================================================================== --- head/sys/arm/ti/ti_sdma.c (revision 299068) +++ head/sys/arm/ti/ti_sdma.c (revision 299069) @@ -1,1250 +1,1250 @@ /*- * Copyright (c) 2011 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /** * Kernel functions for using the DMA controller * * * DMA TRANSFERS: * A DMA transfer block consists of a number of frames (FN). Each frame * consists of a number of elements, and each element can have a size of 8, 16, * or 32 bits. * * OMAP44xx and newer chips support linked list (aka scatter gather) transfers, * where a linked list of source/destination pairs can be placed in memory * for the H/W to process. Earlier chips only allowed you to chain multiple * channels together. However currently this linked list feature is not * supported by the driver. * */ /** * Data structure per DMA channel. * * */ struct ti_sdma_channel { /* * The configuration registers for the given channel, these are modified * by the set functions and only written to the actual registers when a * transaction is started. */ uint32_t reg_csdp; uint32_t reg_ccr; uint32_t reg_cicr; /* Set when one of the configuration registers above change */ uint32_t need_reg_write; /* Callback function used when an interrupt is tripped on the given channel */ void (*callback)(unsigned int ch, uint32_t ch_status, void *data); /* Callback data passed in the callback ... duh */ void* callback_data; }; /** * DMA driver context, allocated and stored globally, this driver is not * intetned to ever be unloaded (see ti_sdma_sc). * */ struct ti_sdma_softc { device_t sc_dev; struct resource* sc_irq_res; struct resource* sc_mem_res; /* * I guess in theory we should have a mutex per DMA channel for register * modifications. But since we know we are never going to be run on a SMP * system, we can use just the single lock for all channels. */ struct mtx sc_mtx; /* Stores the H/W revision read from the registers */ uint32_t sc_hw_rev; /* * Bits in the sc_active_channels data field indicate if the channel has * been activated. */ uint32_t sc_active_channels; struct ti_sdma_channel sc_channel[NUM_DMA_CHANNELS]; }; static struct ti_sdma_softc *ti_sdma_sc = NULL; /** * Macros for driver mutex locking */ #define TI_SDMA_LOCK(_sc) mtx_lock_spin(&(_sc)->sc_mtx) #define TI_SDMA_UNLOCK(_sc) mtx_unlock_spin(&(_sc)->sc_mtx) #define TI_SDMA_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), \ "ti_sdma", MTX_SPIN) #define TI_SDMA_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define TI_SDMA_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define TI_SDMA_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); /** * Function prototypes * */ static void ti_sdma_intr(void *); /** * ti_sdma_read_4 - reads a 32-bit value from one of the DMA registers * @sc: DMA device context * @off: The offset of a register from the DMA register address range * * * RETURNS: * 32-bit value read from the register. */ static inline uint32_t ti_sdma_read_4(struct ti_sdma_softc *sc, bus_size_t off) { return bus_read_4(sc->sc_mem_res, off); } /** * ti_sdma_write_4 - writes a 32-bit value to one of the DMA registers * @sc: DMA device context * @off: The offset of a register from the DMA register address range * * * RETURNS: * 32-bit value read from the register. */ static inline void ti_sdma_write_4(struct ti_sdma_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(sc->sc_mem_res, off, val); } /** * ti_sdma_is_omap3_rev - returns true if H/W is from OMAP3 series * @sc: DMA device context * */ static inline int ti_sdma_is_omap3_rev(struct ti_sdma_softc *sc) { return (sc->sc_hw_rev == DMA4_OMAP3_REV); } /** * ti_sdma_is_omap4_rev - returns true if H/W is from OMAP4 series * @sc: DMA device context * */ static inline int ti_sdma_is_omap4_rev(struct ti_sdma_softc *sc) { return (sc->sc_hw_rev == DMA4_OMAP4_REV); } /** * ti_sdma_intr - interrupt handler for all 4 DMA IRQs * @arg: ignored * * Called when any of the four DMA IRQs are triggered. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * nothing */ static void ti_sdma_intr(void *arg) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t intr; uint32_t csr; unsigned int ch, j; struct ti_sdma_channel* channel; TI_SDMA_LOCK(sc); for (j = 0; j < NUM_DMA_IRQS; j++) { /* Get the flag interrupts (enabled) */ intr = ti_sdma_read_4(sc, DMA4_IRQSTATUS_L(j)); intr &= ti_sdma_read_4(sc, DMA4_IRQENABLE_L(j)); if (intr == 0x00000000) continue; /* Loop through checking the status bits */ for (ch = 0; ch < NUM_DMA_CHANNELS; ch++) { if (intr & (1 << ch)) { channel = &sc->sc_channel[ch]; /* Read the CSR regsiter and verify we don't have a spurious IRQ */ csr = ti_sdma_read_4(sc, DMA4_CSR(ch)); if (csr == 0) { device_printf(sc->sc_dev, "Spurious DMA IRQ for channel " "%d\n", ch); continue; } /* Sanity check this channel is active */ if ((sc->sc_active_channels & (1 << ch)) == 0) { device_printf(sc->sc_dev, "IRQ %d for a non-activated " "channel %d\n", j, ch); continue; } /* Check the status error codes */ if (csr & DMA4_CSR_DROP) device_printf(sc->sc_dev, "Synchronization event drop " "occurred during the transfer on channel %u\n", ch); if (csr & DMA4_CSR_SECURE_ERR) device_printf(sc->sc_dev, "Secure transaction error event " "on channel %u\n", ch); if (csr & DMA4_CSR_MISALIGNED_ADRS_ERR) device_printf(sc->sc_dev, "Misaligned address error event " "on channel %u\n", ch); if (csr & DMA4_CSR_TRANS_ERR) { device_printf(sc->sc_dev, "Transaction error event on " "channel %u\n", ch); /* * Apparently according to linux code, there is an errata * that says the channel is not disabled upon this error. * They explicitly disable the channel here .. since I * haven't seen the errata, I'm going to ignore for now. */ } /* Clear the status flags for the IRQ */ ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK); ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch)); /* Call the callback for the given channel */ if (channel->callback) channel->callback(ch, csr, channel->callback_data); } } } TI_SDMA_UNLOCK(sc); return; } /** * ti_sdma_activate_channel - activates a DMA channel * @ch: upon return contains the channel allocated * @callback: a callback function to associate with the channel * @data: optional data supplied when the callback is called * * Simply activates a channel be enabling and writing default values to the * channel's register set. It doesn't start a transaction, just populates the * internal data structures and sets defaults. * * Note this function doesn't enable interrupts, for that you need to call * ti_sdma_enable_channel_irq(). If not using IRQ to detect the end of the * transfer, you can use ti_sdma_status_poll() to detect a change in the * status. * * A channel must be activated before any of the other DMA functions can be * called on it. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * 0 on success, otherwise an error code */ int ti_sdma_activate_channel(unsigned int *ch, void (*callback)(unsigned int ch, uint32_t status, void *data), void *data) { struct ti_sdma_softc *sc = ti_sdma_sc; struct ti_sdma_channel *channel = NULL; uint32_t addr; unsigned int i; /* Sanity check */ if (sc == NULL) return (ENOMEM); if (ch == NULL) return (EINVAL); TI_SDMA_LOCK(sc); /* Check to see if all channels are in use */ if (sc->sc_active_channels == 0xffffffff) { TI_SDMA_UNLOCK(sc); return (ENOMEM); } /* Find the first non-active channel */ for (i = 0; i < NUM_DMA_CHANNELS; i++) { if (!(sc->sc_active_channels & (0x1 << i))) { sc->sc_active_channels |= (0x1 << i); *ch = i; break; } } /* Get the channel struct and populate the fields */ channel = &sc->sc_channel[*ch]; channel->callback = callback; channel->callback_data = data; channel->need_reg_write = 1; /* Set the default configuration for the DMA channel */ channel->reg_csdp = DMA4_CSDP_DATA_TYPE(0x2) | DMA4_CSDP_SRC_BURST_MODE(0) | DMA4_CSDP_DST_BURST_MODE(0) | DMA4_CSDP_SRC_ENDIANISM(0) | DMA4_CSDP_DST_ENDIANISM(0) | DMA4_CSDP_WRITE_MODE(0) | DMA4_CSDP_SRC_PACKED(0) | DMA4_CSDP_DST_PACKED(0); channel->reg_ccr = DMA4_CCR_DST_ADDRESS_MODE(1) | DMA4_CCR_SRC_ADDRESS_MODE(1) | DMA4_CCR_READ_PRIORITY(0) | DMA4_CCR_WRITE_PRIORITY(0) | DMA4_CCR_SYNC_TRIGGER(0) | DMA4_CCR_FRAME_SYNC(0) | DMA4_CCR_BLOCK_SYNC(0); channel->reg_cicr = DMA4_CICR_TRANS_ERR_IE | DMA4_CICR_SECURE_ERR_IE | DMA4_CICR_SUPERVISOR_ERR_IE | DMA4_CICR_MISALIGNED_ADRS_ERR_IE; /* Clear all the channel registers, this should abort any transaction */ for (addr = DMA4_CCR(*ch); addr <= DMA4_COLOR(*ch); addr += 4) ti_sdma_write_4(sc, addr, 0x00000000); TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_deactivate_channel - deactivates a channel * @ch: the channel to deactivate * * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_deactivate_channel(unsigned int ch) { struct ti_sdma_softc *sc = ti_sdma_sc; unsigned int j; unsigned int addr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); /* First check if the channel is currently active */ if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EBUSY); } /* Mark the channel as inactive */ sc->sc_active_channels &= ~(1 << ch); /* Disable all DMA interrupts for the channel. */ ti_sdma_write_4(sc, DMA4_CICR(ch), 0); /* Make sure the DMA transfer is stopped. */ ti_sdma_write_4(sc, DMA4_CCR(ch), 0); /* Clear the CSR register and IRQ status register */ ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK); for (j = 0; j < NUM_DMA_IRQS; j++) { ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch)); } /* Clear all the channel registers, this should abort any transaction */ for (addr = DMA4_CCR(ch); addr <= DMA4_COLOR(ch); addr += 4) ti_sdma_write_4(sc, addr, 0x00000000); TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_disable_channel_irq - disables IRQ's on the given channel * @ch: the channel to disable IRQ's on * - * Disable interupt generation for the given channel. + * Disable interrupt generation for the given channel. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_disable_channel_irq(unsigned int ch) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t irq_enable; unsigned int j; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } /* Disable all the individual error conditions */ sc->sc_channel[ch].reg_cicr = 0x0000; ti_sdma_write_4(sc, DMA4_CICR(ch), 0x0000); /* Disable the channel interrupt enable */ for (j = 0; j < NUM_DMA_IRQS; j++) { irq_enable = ti_sdma_read_4(sc, DMA4_IRQENABLE_L(j)); irq_enable &= ~(1 << ch); ti_sdma_write_4(sc, DMA4_IRQENABLE_L(j), irq_enable); } /* Indicate the registers need to be rewritten on the next transaction */ sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return (0); } /** * ti_sdma_disable_channel_irq - enables IRQ's on the given channel * @ch: the channel to enable IRQ's on * @flags: bitmask of interrupt types to enable * * Flags can be a bitmask of the following options: * DMA_IRQ_FLAG_DROP * DMA_IRQ_FLAG_HALF_FRAME_COMPL * DMA_IRQ_FLAG_FRAME_COMPL * DMA_IRQ_FLAG_START_LAST_FRAME * DMA_IRQ_FLAG_BLOCK_COMPL * DMA_IRQ_FLAG_ENDOF_PKT * DMA_IRQ_FLAG_DRAIN * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_enable_channel_irq(unsigned int ch, uint32_t flags) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t irq_enable; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } /* Always enable the error interrupts if we have interrupts enabled */ flags |= DMA4_CICR_TRANS_ERR_IE | DMA4_CICR_SECURE_ERR_IE | DMA4_CICR_SUPERVISOR_ERR_IE | DMA4_CICR_MISALIGNED_ADRS_ERR_IE; sc->sc_channel[ch].reg_cicr = flags; /* Write the values to the register */ ti_sdma_write_4(sc, DMA4_CICR(ch), flags); /* Enable the channel interrupt enable */ irq_enable = ti_sdma_read_4(sc, DMA4_IRQENABLE_L(0)); irq_enable |= (1 << ch); ti_sdma_write_4(sc, DMA4_IRQENABLE_L(0), irq_enable); /* Indicate the registers need to be rewritten on the next transaction */ sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return (0); } /** * ti_sdma_get_channel_status - returns the status of a given channel * @ch: the channel number to get the status of * @status: upon return will contain the status bitmask, see below for possible * values. * * DMA_STATUS_DROP * DMA_STATUS_HALF * DMA_STATUS_FRAME * DMA_STATUS_LAST * DMA_STATUS_BLOCK * DMA_STATUS_SYNC * DMA_STATUS_PKT * DMA_STATUS_TRANS_ERR * DMA_STATUS_SECURE_ERR * DMA_STATUS_SUPERVISOR_ERR * DMA_STATUS_MISALIGNED_ADRS_ERR * DMA_STATUS_DRAIN_END * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_get_channel_status(unsigned int ch, uint32_t *status) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t csr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } TI_SDMA_UNLOCK(sc); csr = ti_sdma_read_4(sc, DMA4_CSR(ch)); if (status != NULL) *status = csr; return (0); } /** * ti_sdma_start_xfer - starts a DMA transfer - * @ch: the channel number to set the endianess of + * @ch: the channel number to set the endianness of * @src_paddr: the source phsyical address * @dst_paddr: the destination phsyical address * @frmcnt: the number of frames per block * @elmcnt: the number of elements in a frame, an element is either an 8, 16 * or 32-bit value as defined by ti_sdma_set_xfer_burst() * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_start_xfer(unsigned int ch, unsigned int src_paddr, unsigned long dst_paddr, unsigned int frmcnt, unsigned int elmcnt) { struct ti_sdma_softc *sc = ti_sdma_sc; struct ti_sdma_channel *channel; uint32_t ccr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } channel = &sc->sc_channel[ch]; /* a) Write the CSDP register */ ti_sdma_write_4(sc, DMA4_CSDP(ch), channel->reg_csdp | DMA4_CSDP_WRITE_MODE(1)); /* b) Set the number of element per frame CEN[23:0] */ ti_sdma_write_4(sc, DMA4_CEN(ch), elmcnt); /* c) Set the number of frame per block CFN[15:0] */ ti_sdma_write_4(sc, DMA4_CFN(ch), frmcnt); /* d) Set the Source/dest start address index CSSA[31:0]/CDSA[31:0] */ ti_sdma_write_4(sc, DMA4_CSSA(ch), src_paddr); ti_sdma_write_4(sc, DMA4_CDSA(ch), dst_paddr); /* e) Write the CCR register */ ti_sdma_write_4(sc, DMA4_CCR(ch), channel->reg_ccr); /* f) - Set the source element index increment CSEI[15:0] */ ti_sdma_write_4(sc, DMA4_CSE(ch), 0x0001); /* - Set the source frame index increment CSFI[15:0] */ ti_sdma_write_4(sc, DMA4_CSF(ch), 0x0001); /* - Set the destination element index increment CDEI[15:0]*/ ti_sdma_write_4(sc, DMA4_CDE(ch), 0x0001); /* - Set the destination frame index increment CDFI[31:0] */ ti_sdma_write_4(sc, DMA4_CDF(ch), 0x0001); /* Clear the status register */ ti_sdma_write_4(sc, DMA4_CSR(ch), 0x1FFE); /* Write the start-bit and away we go */ ccr = ti_sdma_read_4(sc, DMA4_CCR(ch)); ccr |= (1 << 7); ti_sdma_write_4(sc, DMA4_CCR(ch), ccr); /* Clear the reg write flag */ channel->need_reg_write = 0; TI_SDMA_UNLOCK(sc); return (0); } /** * ti_sdma_start_xfer_packet - starts a packet DMA transfer * @ch: the channel number to use for the transfer * @src_paddr: the source physical address * @dst_paddr: the destination physical address * @frmcnt: the number of frames to transfer * @elmcnt: the number of elements in a frame, an element is either an 8, 16 * or 32-bit value as defined by ti_sdma_set_xfer_burst() * @pktsize: the number of elements in each transfer packet * * The @frmcnt and @elmcnt define the overall number of bytes to transfer, * typically @frmcnt is 1 and @elmcnt contains the total number of elements. * @pktsize is the size of each individual packet, there might be multiple * packets per transfer. i.e. for the following with element size of 32-bits * * frmcnt = 1, elmcnt = 512, pktsize = 128 * * Total transfer bytes = 1 * 512 = 512 elements or 2048 bytes - * Packets transfered = 128 / 512 = 4 + * Packets transferred = 128 / 512 = 4 * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_start_xfer_packet(unsigned int ch, unsigned int src_paddr, unsigned long dst_paddr, unsigned int frmcnt, unsigned int elmcnt, unsigned int pktsize) { struct ti_sdma_softc *sc = ti_sdma_sc; struct ti_sdma_channel *channel; uint32_t ccr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } channel = &sc->sc_channel[ch]; /* a) Write the CSDP register */ if (channel->need_reg_write) ti_sdma_write_4(sc, DMA4_CSDP(ch), channel->reg_csdp | DMA4_CSDP_WRITE_MODE(1)); /* b) Set the number of elements to transfer CEN[23:0] */ ti_sdma_write_4(sc, DMA4_CEN(ch), elmcnt); /* c) Set the number of frames to transfer CFN[15:0] */ ti_sdma_write_4(sc, DMA4_CFN(ch), frmcnt); /* d) Set the Source/dest start address index CSSA[31:0]/CDSA[31:0] */ ti_sdma_write_4(sc, DMA4_CSSA(ch), src_paddr); ti_sdma_write_4(sc, DMA4_CDSA(ch), dst_paddr); /* e) Write the CCR register */ ti_sdma_write_4(sc, DMA4_CCR(ch), channel->reg_ccr | DMA4_CCR_PACKET_TRANS); /* f) - Set the source element index increment CSEI[15:0] */ ti_sdma_write_4(sc, DMA4_CSE(ch), 0x0001); /* - Set the packet size, this is dependent on the sync source */ if (channel->reg_ccr & DMA4_CCR_SEL_SRC_DST_SYNC(1)) ti_sdma_write_4(sc, DMA4_CSF(ch), pktsize); else ti_sdma_write_4(sc, DMA4_CDF(ch), pktsize); /* - Set the destination frame index increment CDFI[31:0] */ ti_sdma_write_4(sc, DMA4_CDE(ch), 0x0001); /* Clear the status register */ ti_sdma_write_4(sc, DMA4_CSR(ch), 0x1FFE); /* Write the start-bit and away we go */ ccr = ti_sdma_read_4(sc, DMA4_CCR(ch)); ccr |= (1 << 7); ti_sdma_write_4(sc, DMA4_CCR(ch), ccr); /* Clear the reg write flag */ channel->need_reg_write = 0; TI_SDMA_UNLOCK(sc); return (0); } /** * ti_sdma_stop_xfer - stops any currently active transfers - * @ch: the channel number to set the endianess of + * @ch: the channel number to set the endianness of * * This function call is effectively a NOP if no transaction is in progress. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_stop_xfer(unsigned int ch) { struct ti_sdma_softc *sc = ti_sdma_sc; unsigned int j; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } /* Disable all DMA interrupts for the channel. */ ti_sdma_write_4(sc, DMA4_CICR(ch), 0); /* Make sure the DMA transfer is stopped. */ ti_sdma_write_4(sc, DMA4_CCR(ch), 0); /* Clear the CSR register and IRQ status register */ ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK); for (j = 0; j < NUM_DMA_IRQS; j++) { ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch)); } /* Configuration registers need to be re-written on the next xfer */ sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return (0); } /** - * ti_sdma_set_xfer_endianess - sets the endianess of subsequent transfers - * @ch: the channel number to set the endianess of - * @src: the source endianess (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG) - * @dst: the destination endianess (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG) + * ti_sdma_set_xfer_endianess - sets the endianness of subsequent transfers + * @ch: the channel number to set the endianness of + * @src: the source endianness (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG) + * @dst: the destination endianness (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG) * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_xfer_endianess(unsigned int ch, unsigned int src, unsigned int dst) { struct ti_sdma_softc *sc = ti_sdma_sc; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_SRC_ENDIANISM(1); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_SRC_ENDIANISM(src); sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DST_ENDIANISM(1); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_DST_ENDIANISM(dst); sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_set_xfer_burst - sets the source and destination element size * @ch: the channel number to set the burst settings of - * @src: the source endianess (either DMA_BURST_NONE, DMA_BURST_16, DMA_BURST_32 + * @src: the source endianness (either DMA_BURST_NONE, DMA_BURST_16, DMA_BURST_32 * or DMA_BURST_64) - * @dst: the destination endianess (either DMA_BURST_NONE, DMA_BURST_16, + * @dst: the destination endianness (either DMA_BURST_NONE, DMA_BURST_16, * DMA_BURST_32 or DMA_BURST_64) * * This function sets the size of the elements for all subsequent transfers. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_xfer_burst(unsigned int ch, unsigned int src, unsigned int dst) { struct ti_sdma_softc *sc = ti_sdma_sc; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_SRC_BURST_MODE(0x3); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_SRC_BURST_MODE(src); sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DST_BURST_MODE(0x3); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_DST_BURST_MODE(dst); sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_set_xfer_data_type - driver attach function - * @ch: the channel number to set the endianess of + * @ch: the channel number to set the endianness of * @type: the xfer data type (either DMA_DATA_8BITS_SCALAR, DMA_DATA_16BITS_SCALAR * or DMA_DATA_32BITS_SCALAR) * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_xfer_data_type(unsigned int ch, unsigned int type) { struct ti_sdma_softc *sc = ti_sdma_sc; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DATA_TYPE(0x3); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_DATA_TYPE(type); sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_set_callback - driver attach function * @dev: dma device handle * * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_callback(unsigned int ch, void (*callback)(unsigned int ch, uint32_t status, void *data), void *data) { struct ti_sdma_softc *sc = ti_sdma_sc; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } sc->sc_channel[ch].callback = callback; sc->sc_channel[ch].callback_data = data; sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_sync_params - sets channel sync settings * @ch: the channel number to set the sync on * @trigger: the number of the sync trigger, this depends on what other H/W * module is triggering/receiving the DMA transactions * @mode: flags describing the sync mode to use, it may have one or more of * the following bits set; TI_SDMA_SYNC_FRAME, * TI_SDMA_SYNC_BLOCK, TI_SDMA_SYNC_TRIG_ON_SRC. * * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_sync_params(unsigned int ch, unsigned int trigger, unsigned int mode) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t ccr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } ccr = sc->sc_channel[ch].reg_ccr; ccr &= ~DMA4_CCR_SYNC_TRIGGER(0x7F); ccr |= DMA4_CCR_SYNC_TRIGGER(trigger + 1); if (mode & TI_SDMA_SYNC_FRAME) ccr |= DMA4_CCR_FRAME_SYNC(1); else ccr &= ~DMA4_CCR_FRAME_SYNC(1); if (mode & TI_SDMA_SYNC_BLOCK) ccr |= DMA4_CCR_BLOCK_SYNC(1); else ccr &= ~DMA4_CCR_BLOCK_SYNC(1); if (mode & TI_SDMA_SYNC_TRIG_ON_SRC) ccr |= DMA4_CCR_SEL_SRC_DST_SYNC(1); else ccr &= ~DMA4_CCR_SEL_SRC_DST_SYNC(1); sc->sc_channel[ch].reg_ccr = ccr; sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_set_addr_mode - driver attach function - * @ch: the channel number to set the endianess of + * @ch: the channel number to set the endianness of * @rd_mode: the xfer source addressing mode (either DMA_ADDR_CONSTANT, * DMA_ADDR_POST_INCREMENT, DMA_ADDR_SINGLE_INDEX or * DMA_ADDR_DOUBLE_INDEX) * @wr_mode: the xfer destination addressing mode (either DMA_ADDR_CONSTANT, * DMA_ADDR_POST_INCREMENT, DMA_ADDR_SINGLE_INDEX or * DMA_ADDR_DOUBLE_INDEX) * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_addr_mode(unsigned int ch, unsigned int src_mode, unsigned int dst_mode) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t ccr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } ccr = sc->sc_channel[ch].reg_ccr; ccr &= ~DMA4_CCR_SRC_ADDRESS_MODE(0x3); ccr |= DMA4_CCR_SRC_ADDRESS_MODE(src_mode); ccr &= ~DMA4_CCR_DST_ADDRESS_MODE(0x3); ccr |= DMA4_CCR_DST_ADDRESS_MODE(dst_mode); sc->sc_channel[ch].reg_ccr = ccr; sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_probe - driver probe function * @dev: dma device handle * * * * RETURNS: * Always returns 0. */ static int ti_sdma_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "ti,omap4430-sdma")) return (ENXIO); device_set_desc(dev, "TI sDMA Controller"); return (0); } /** * ti_sdma_attach - driver attach function * @dev: dma device handle * * Initialises memory mapping/pointers to the DMA register set and requests * IRQs. This is effectively the setup function for the driver. * * RETURNS: * 0 on success or a negative error code failure. */ static int ti_sdma_attach(device_t dev) { struct ti_sdma_softc *sc = device_get_softc(dev); unsigned int timeout; unsigned int i; int rid; void *ihl; int err; /* Setup the basics */ sc->sc_dev = dev; /* No channels active at the moment */ sc->sc_active_channels = 0x00000000; /* Mutex to protect the shared data structures */ TI_SDMA_LOCK_INIT(sc); /* Get the memory resource for the register mapping */ rid = 0; sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem_res == NULL) panic("%s: Cannot map registers", device_get_name(dev)); /* Enable the interface and functional clocks */ ti_prcm_clk_enable(SDMA_CLK); /* Read the sDMA revision register and sanity check it's known */ sc->sc_hw_rev = ti_sdma_read_4(sc, DMA4_REVISION); device_printf(dev, "sDMA revision %08x\n", sc->sc_hw_rev); if (!ti_sdma_is_omap4_rev(sc) && !ti_sdma_is_omap3_rev(sc)) { device_printf(sc->sc_dev, "error - unknown sDMA H/W revision\n"); return (EINVAL); } /* Disable all interrupts */ for (i = 0; i < NUM_DMA_IRQS; i++) { ti_sdma_write_4(sc, DMA4_IRQENABLE_L(i), 0x00000000); } /* Soft-reset is only supported on pre-OMAP44xx devices */ if (ti_sdma_is_omap3_rev(sc)) { /* Soft-reset */ ti_sdma_write_4(sc, DMA4_OCP_SYSCONFIG, 0x0002); /* Set the timeout to 100ms*/ timeout = (hz < 10) ? 1 : ((100 * hz) / 1000); /* Wait for DMA reset to complete */ while ((ti_sdma_read_4(sc, DMA4_SYSSTATUS) & 0x1) == 0x0) { /* Sleep for a tick */ pause("DMARESET", 1); if (timeout-- == 0) { device_printf(sc->sc_dev, "sDMA reset operation timed out\n"); return (EINVAL); } } } /* * Install interrupt handlers for the for possible interrupts. Any channel * can trip one of the four IRQs */ rid = 0; sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | RF_SHAREABLE); if (sc->sc_irq_res == NULL) panic("Unable to setup the dma irq handler.\n"); err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, ti_sdma_intr, NULL, &ihl); if (err) panic("%s: Cannot register IRQ", device_get_name(dev)); /* Store the DMA structure globally ... this driver should never be unloaded */ ti_sdma_sc = sc; return (0); } static device_method_t ti_sdma_methods[] = { DEVMETHOD(device_probe, ti_sdma_probe), DEVMETHOD(device_attach, ti_sdma_attach), {0, 0}, }; static driver_t ti_sdma_driver = { "ti_sdma", ti_sdma_methods, sizeof(struct ti_sdma_softc), }; static devclass_t ti_sdma_devclass; DRIVER_MODULE(ti_sdma, simplebus, ti_sdma_driver, ti_sdma_devclass, 0, 0); MODULE_DEPEND(ti_sdma, ti_prcm, 1, 1, 1); Index: head/sys/arm/xilinx/zy7_slcr.c =================================================================== --- head/sys/arm/xilinx/zy7_slcr.c (revision 299068) +++ head/sys/arm/xilinx/zy7_slcr.c (revision 299069) @@ -1,712 +1,712 @@ /*- * Copyright (c) 2013 Thomas Skibo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * Zynq-700 SLCR driver. Provides hooks for cpu_reset and PL control stuff. * In the future, maybe MIO control, clock control, etc. could go here. * * Reference: Zynq-7000 All Programmable SoC Technical Reference Manual. * (v1.4) November 16, 2012. Xilinx doc UG585. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct zy7_slcr_softc { device_t dev; struct mtx sc_mtx; struct resource *mem_res; }; static struct zy7_slcr_softc *zy7_slcr_softc_p; extern void (*zynq7_cpu_reset); #define ZSLCR_LOCK(sc) mtx_lock(&(sc)->sc_mtx) #define ZSLCR_UNLOCK(sc) mtx_unlock(&(sc)->sc_mtx) #define ZSLCR_LOCK_INIT(sc) \ mtx_init(&(sc)->sc_mtx, device_get_nameunit((sc)->dev), \ "zy7_slcr", MTX_DEF) #define ZSLCR_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define RD4(sc, off) (bus_read_4((sc)->mem_res, (off))) #define WR4(sc, off, val) (bus_write_4((sc)->mem_res, (off), (val))) #define ZYNQ_DEFAULT_PS_CLK_FREQUENCY 33333333 /* 33.3 Mhz */ SYSCTL_NODE(_hw, OID_AUTO, zynq, CTLFLAG_RD, 0, "Xilinx Zynq-7000"); static char zynq_bootmode[64]; SYSCTL_STRING(_hw_zynq, OID_AUTO, bootmode, CTLFLAG_RD, zynq_bootmode, 0, "Zynq boot mode"); static char zynq_pssid[100]; SYSCTL_STRING(_hw_zynq, OID_AUTO, pssid, CTLFLAG_RD, zynq_pssid, 0, "Zynq PSS IDCODE"); static uint32_t zynq_reboot_status; SYSCTL_INT(_hw_zynq, OID_AUTO, reboot_status, CTLFLAG_RD, &zynq_reboot_status, 0, "Zynq REBOOT_STATUS register"); static int ps_clk_frequency; SYSCTL_INT(_hw_zynq, OID_AUTO, ps_clk_frequency, CTLFLAG_RD, &ps_clk_frequency, 0, "Zynq PS_CLK Frequency"); static int io_pll_frequency; SYSCTL_INT(_hw_zynq, OID_AUTO, io_pll_frequency, CTLFLAG_RD, &io_pll_frequency, 0, "Zynq IO PLL Frequency"); static int arm_pll_frequency; SYSCTL_INT(_hw_zynq, OID_AUTO, arm_pll_frequency, CTLFLAG_RD, &arm_pll_frequency, 0, "Zynq ARM PLL Frequency"); static int ddr_pll_frequency; SYSCTL_INT(_hw_zynq, OID_AUTO, ddr_pll_frequency, CTLFLAG_RD, &ddr_pll_frequency, 0, "Zynq DDR PLL Frequency"); static void zy7_slcr_unlock(struct zy7_slcr_softc *sc) { /* Unlock SLCR with magic number. */ WR4(sc, ZY7_SLCR_UNLOCK, ZY7_SLCR_UNLOCK_MAGIC); } static void zy7_slcr_lock(struct zy7_slcr_softc *sc) { /* Lock SLCR with magic number. */ WR4(sc, ZY7_SLCR_LOCK, ZY7_SLCR_LOCK_MAGIC); } static void zy7_slcr_cpu_reset(void) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* This has something to do with a work-around so the fsbl will load * the bitstream after soft-reboot. It's very important. */ WR4(sc, ZY7_SLCR_REBOOT_STAT, RD4(sc, ZY7_SLCR_REBOOT_STAT) & 0xf0ffffff); /* Soft reset */ WR4(sc, ZY7_SLCR_PSS_RST_CTRL, ZY7_SLCR_PSS_RST_CTRL_SOFT_RESET); for (;;) ; } /* Assert PL resets and disable level shifters in preparation of programming * the PL (FPGA) section. Called from zy7_devcfg.c. */ void zy7_slcr_preload_pl(void) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return; ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* Assert top level output resets. */ WR4(sc, ZY7_SLCR_FPGA_RST_CTRL, ZY7_SLCR_FPGA_RST_CTRL_RST_ALL); /* Disable all level shifters. */ WR4(sc, ZY7_SLCR_LVL_SHFTR_EN, 0); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); } /* After PL configuration, enable level shifters and deassert top-level * PL resets. Called from zy7_devcfg.c. Optionally, the level shifters * can be left disabled but that's rare of an FPGA application. That option - * is controled by a sysctl in the devcfg driver. + * is controlled by a sysctl in the devcfg driver. */ void zy7_slcr_postload_pl(int en_level_shifters) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return; ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); if (en_level_shifters) /* Enable level shifters. */ WR4(sc, ZY7_SLCR_LVL_SHFTR_EN, ZY7_SLCR_LVL_SHFTR_EN_ALL); /* Deassert top level output resets. */ WR4(sc, ZY7_SLCR_FPGA_RST_CTRL, 0); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); } /* Override cgem_set_refclk() in gigabit ethernet driver * (sys/dev/cadence/if_cgem.c). This function is called to * request a change in the gem's reference clock speed. */ int cgem_set_ref_clk(int unit, int frequency) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; int div0, div1; if (!sc) return (-1); /* Find suitable divisor pairs. Round result to nearest khz * to test for match. */ for (div1 = 1; div1 <= ZY7_SLCR_GEM_CLK_CTRL_DIVISOR1_MAX; div1++) { div0 = (io_pll_frequency + div1 * frequency / 2) / div1 / frequency; if (div0 > 0 && div0 <= ZY7_SLCR_GEM_CLK_CTRL_DIVISOR_MAX && ((io_pll_frequency / div0 / div1) + 500) / 1000 == (frequency + 500) / 1000) break; } if (div1 > ZY7_SLCR_GEM_CLK_CTRL_DIVISOR1_MAX) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* Modify GEM reference clock. */ WR4(sc, unit ? ZY7_SLCR_GEM1_CLK_CTRL : ZY7_SLCR_GEM0_CLK_CTRL, (div1 << ZY7_SLCR_GEM_CLK_CTRL_DIVISOR1_SHIFT) | (div0 << ZY7_SLCR_GEM_CLK_CTRL_DIVISOR_SHIFT) | ZY7_SLCR_GEM_CLK_CTRL_SRCSEL_IO_PLL | ZY7_SLCR_GEM_CLK_CTRL_CLKACT); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (0); } /* * PL clocks management function */ int zy7_pl_fclk_set_source(int unit, int source) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; uint32_t reg; if (!sc) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* Modify FPGAx source. */ reg = RD4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit)); reg &= ~(ZY7_SLCR_FPGA_CLK_CTRL_SRCSEL_MASK); reg |= (source << ZY7_SLCR_FPGA_CLK_CTRL_SRCSEL_SHIFT); WR4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit), reg); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (0); } int zy7_pl_fclk_get_source(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; uint32_t reg; int source; if (!sc) return (-1); ZSLCR_LOCK(sc); /* Modify GEM reference clock. */ reg = RD4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit)); source = (reg & ZY7_SLCR_FPGA_CLK_CTRL_SRCSEL_MASK) >> ZY7_SLCR_FPGA_CLK_CTRL_SRCSEL_SHIFT; /* ZY7_PL_FCLK_SRC_IO is actually b0x */ if ((source & 2) == 0) source = ZY7_PL_FCLK_SRC_IO; ZSLCR_UNLOCK(sc); return (source); } int zy7_pl_fclk_set_freq(int unit, int frequency) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; int div0, div1; int base_frequency; uint32_t reg; int source; if (!sc) return (-1); source = zy7_pl_fclk_get_source(unit); switch (source) { case ZY7_PL_FCLK_SRC_IO: base_frequency = io_pll_frequency; break; case ZY7_PL_FCLK_SRC_ARM: base_frequency = arm_pll_frequency; break; case ZY7_PL_FCLK_SRC_DDR: base_frequency = ddr_pll_frequency; break; default: return (-1); } /* Find suitable divisor pairs. Round result to nearest khz * to test for match. */ for (div1 = 1; div1 <= ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR_MAX; div1++) { div0 = (base_frequency + div1 * frequency / 2) / div1 / frequency; if (div0 > 0 && div0 <= ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR_MAX && ((base_frequency / div0 / div1) + 500) / 1000 == (frequency + 500) / 1000) break; } if (div1 > ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR_MAX) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* Modify FPGAx reference clock. */ reg = RD4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit)); reg &= ~(ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR1_MASK | ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR0_MASK); reg |= (div1 << ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR1_SHIFT) | (div0 << ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR0_SHIFT); WR4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit), reg); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (base_frequency / div0 / div1); } int zy7_pl_fclk_get_freq(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; int div0, div1; int base_frequency; int frequency; uint32_t reg; int source; if (!sc) return (-1); source = zy7_pl_fclk_get_source(unit); switch (source) { case ZY7_PL_FCLK_SRC_IO: base_frequency = io_pll_frequency; break; case ZY7_PL_FCLK_SRC_ARM: base_frequency = arm_pll_frequency; break; case ZY7_PL_FCLK_SRC_DDR: base_frequency = ddr_pll_frequency; break; default: return (-1); } ZSLCR_LOCK(sc); /* Modify FPGAx reference clock. */ reg = RD4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit)); div1 = (reg & ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR1_MASK) >> ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR1_SHIFT; div0 = (reg & ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR0_MASK) >> ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR0_SHIFT; ZSLCR_UNLOCK(sc); if (div0 == 0) div0 = 1; if (div1 == 0) div1 = 1; frequency = (base_frequency / div0 / div1); /* Round to KHz */ frequency = (frequency + 500) / 1000; frequency = frequency * 1000; return (frequency); } int zy7_pl_fclk_enable(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); WR4(sc, ZY7_SLCR_FPGA_THR_CTRL(unit), 0); WR4(sc, ZY7_SLCR_FPGA_THR_CNT(unit), 0); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (0); } int zy7_pl_fclk_disable(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); WR4(sc, ZY7_SLCR_FPGA_THR_CTRL(unit), 0); WR4(sc, ZY7_SLCR_FPGA_THR_CNT(unit), 1); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (0); } int zy7_pl_fclk_enabled(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; uint32_t reg; if (!sc) return (-1); ZSLCR_LOCK(sc); reg = RD4(sc, ZY7_SLCR_FPGA_THR_CNT(unit)); ZSLCR_UNLOCK(sc); return !(reg & 1); } int zy7_pl_level_shifters_enabled() { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; uint32_t reg; if (!sc) return (-1); ZSLCR_LOCK(sc); reg = RD4(sc, ZY7_SLCR_LVL_SHFTR_EN); ZSLCR_UNLOCK(sc); return (reg == ZY7_SLCR_LVL_SHFTR_EN_ALL); } void zy7_pl_level_shifters_enable() { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return; ZSLCR_LOCK(sc); zy7_slcr_unlock(sc); WR4(sc, ZY7_SLCR_LVL_SHFTR_EN, ZY7_SLCR_LVL_SHFTR_EN_ALL); zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); } void zy7_pl_level_shifters_disable() { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return; ZSLCR_LOCK(sc); zy7_slcr_unlock(sc); WR4(sc, ZY7_SLCR_LVL_SHFTR_EN, 0); zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); } static int zy7_slcr_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "xlnx,zy7_slcr")) return (ENXIO); device_set_desc(dev, "Zynq-7000 slcr block"); return (0); } static int zy7_slcr_attach(device_t dev) { struct zy7_slcr_softc *sc = device_get_softc(dev); int rid; phandle_t node; pcell_t cell; uint32_t bootmode; uint32_t pss_idcode; uint32_t arm_pll_ctrl; uint32_t ddr_pll_ctrl; uint32_t io_pll_ctrl; static char *bootdev_names[] = { "JTAG", "Quad-SPI", "NOR", "(3?)", "NAND", "SD Card", "(6?)", "(7?)" }; /* Allow only one attach. */ if (zy7_slcr_softc_p != NULL) return (ENXIO); sc->dev = dev; ZSLCR_LOCK_INIT(sc); /* Get memory resource. */ rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) { device_printf(dev, "could not allocate memory resources.\n"); return (ENOMEM); } /* Hook up cpu_reset. */ zy7_slcr_softc_p = sc; zynq7_cpu_reset = zy7_slcr_cpu_reset; /* Read info and set sysctls. */ bootmode = RD4(sc, ZY7_SLCR_BOOT_MODE); snprintf(zynq_bootmode, sizeof(zynq_bootmode), "0x%x: boot device: %s", bootmode, bootdev_names[bootmode & ZY7_SLCR_BOOT_MODE_BOOTDEV_MASK]); pss_idcode = RD4(sc, ZY7_SLCR_PSS_IDCODE); snprintf(zynq_pssid, sizeof(zynq_pssid), "0x%x: manufacturer: 0x%x device: 0x%x " "family: 0x%x sub-family: 0x%x rev: 0x%x", pss_idcode, (pss_idcode & ZY7_SLCR_PSS_IDCODE_MNFR_ID_MASK) >> ZY7_SLCR_PSS_IDCODE_MNFR_ID_SHIFT, (pss_idcode & ZY7_SLCR_PSS_IDCODE_DEVICE_MASK) >> ZY7_SLCR_PSS_IDCODE_DEVICE_SHIFT, (pss_idcode & ZY7_SLCR_PSS_IDCODE_FAMILY_MASK) >> ZY7_SLCR_PSS_IDCODE_FAMILY_SHIFT, (pss_idcode & ZY7_SLCR_PSS_IDCODE_SUB_FAMILY_MASK) >> ZY7_SLCR_PSS_IDCODE_SUB_FAMILY_SHIFT, (pss_idcode & ZY7_SLCR_PSS_IDCODE_REVISION_MASK) >> ZY7_SLCR_PSS_IDCODE_REVISION_SHIFT); zynq_reboot_status = RD4(sc, ZY7_SLCR_REBOOT_STAT); /* Derive PLL frequencies from PS_CLK. */ node = ofw_bus_get_node(dev); if (OF_getprop(node, "clock-frequency", &cell, sizeof(cell)) > 0) ps_clk_frequency = fdt32_to_cpu(cell); else ps_clk_frequency = ZYNQ_DEFAULT_PS_CLK_FREQUENCY; arm_pll_ctrl = RD4(sc, ZY7_SLCR_ARM_PLL_CTRL); ddr_pll_ctrl = RD4(sc, ZY7_SLCR_DDR_PLL_CTRL); io_pll_ctrl = RD4(sc, ZY7_SLCR_IO_PLL_CTRL); /* Determine ARM PLL frequency. */ if (((arm_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) == 0 && (arm_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_FORCE) != 0) || ((arm_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) != 0 && (bootmode & ZY7_SLCR_BOOT_MODE_PLL_BYPASS) != 0)) /* PLL is bypassed. */ arm_pll_frequency = ps_clk_frequency; else arm_pll_frequency = ps_clk_frequency * ((arm_pll_ctrl & ZY7_SLCR_PLL_CTRL_FDIV_MASK) >> ZY7_SLCR_PLL_CTRL_FDIV_SHIFT); /* Determine DDR PLL frequency. */ if (((ddr_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) == 0 && (ddr_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_FORCE) != 0) || ((ddr_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) != 0 && (bootmode & ZY7_SLCR_BOOT_MODE_PLL_BYPASS) != 0)) /* PLL is bypassed. */ ddr_pll_frequency = ps_clk_frequency; else ddr_pll_frequency = ps_clk_frequency * ((ddr_pll_ctrl & ZY7_SLCR_PLL_CTRL_FDIV_MASK) >> ZY7_SLCR_PLL_CTRL_FDIV_SHIFT); /* Determine IO PLL frequency. */ if (((io_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) == 0 && (io_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_FORCE) != 0) || ((io_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) != 0 && (bootmode & ZY7_SLCR_BOOT_MODE_PLL_BYPASS) != 0)) /* PLL is bypassed. */ io_pll_frequency = ps_clk_frequency; else io_pll_frequency = ps_clk_frequency * ((io_pll_ctrl & ZY7_SLCR_PLL_CTRL_FDIV_MASK) >> ZY7_SLCR_PLL_CTRL_FDIV_SHIFT); /* Lock SLCR registers. */ zy7_slcr_lock(sc); return (0); } static int zy7_slcr_detach(device_t dev) { struct zy7_slcr_softc *sc = device_get_softc(dev); bus_generic_detach(dev); /* Release memory resource. */ if (sc->mem_res != NULL) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->mem_res), sc->mem_res); zy7_slcr_softc_p = NULL; zynq7_cpu_reset = NULL; ZSLCR_LOCK_DESTROY(sc); return (0); } static device_method_t zy7_slcr_methods[] = { /* device_if */ DEVMETHOD(device_probe, zy7_slcr_probe), DEVMETHOD(device_attach, zy7_slcr_attach), DEVMETHOD(device_detach, zy7_slcr_detach), DEVMETHOD_END }; static driver_t zy7_slcr_driver = { "zy7_slcr", zy7_slcr_methods, sizeof(struct zy7_slcr_softc), }; static devclass_t zy7_slcr_devclass; DRIVER_MODULE(zy7_slcr, simplebus, zy7_slcr_driver, zy7_slcr_devclass, 0, 0); MODULE_VERSION(zy7_slcr, 1); Index: head/sys/arm/xscale/i8134x/crb_machdep.c =================================================================== --- head/sys/arm/xscale/i8134x/crb_machdep.c (revision 299068) +++ head/sys/arm/xscale/i8134x/crb_machdep.c (revision 299069) @@ -1,331 +1,331 @@ /* $NetBSD: hpc_machdep.c,v 1.70 2003/09/16 08:18:22 agc Exp $ */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * machdep.c * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * This file needs a lot of work. * * Created : 17/09/94 */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* For i80321_calibrate_delay() */ #include #include #include #define KERNEL_PT_SYS 0 /* Page table for mapping proc0 zero page */ #define KERNEL_PT_IOPXS 1 #define KERNEL_PT_BEFOREKERN 2 #define KERNEL_PT_AFKERNEL 3 /* L2 table for mapping after kernel */ #define KERNEL_PT_AFKERNEL_NUM 9 /* this should be evenly divisable by PAGE_SIZE / L2_TABLE_SIZE_REAL (or 4) */ #define NUM_KERNEL_PTS (KERNEL_PT_AFKERNEL + KERNEL_PT_AFKERNEL_NUM) struct pv_addr kernel_pt_table[NUM_KERNEL_PTS]; /* Physical and virtual addresses for some global pages */ struct pv_addr systempage; struct pv_addr msgbufpv; struct pv_addr irqstack; struct pv_addr undstack; struct pv_addr abtstack; struct pv_addr kernelstack; /* Static device mappings. */ static const struct devmap_entry iq81342_devmap[] = { { IOP34X_VADDR, IOP34X_HWADDR, IOP34X_SIZE, }, { /* * Cheat and map a whole section, this will bring * both PCI-X and PCI-E outbound I/O */ rounddown2(IOP34X_PCIX_OIOBAR_VADDR, 0x100000), rounddown2(IOP34X_PCIX_OIOBAR, 0x100000), 0x100000, }, { IOP34X_PCE1_VADDR, IOP34X_PCE1, IOP34X_PCE1_SIZE, }, { 0, 0, 0, } }; #define SDRAM_START 0x00000000 extern vm_offset_t xscale_cache_clean_addr; void * initarm(struct arm_boot_params *abp) { struct pv_addr kernel_l1pt; struct pv_addr dpcpu; int loop, i; u_int l1pagetable; vm_offset_t freemempos; vm_offset_t freemem_pt; vm_offset_t afterkern; vm_offset_t freemem_after; vm_offset_t lastaddr; uint32_t memsize, memstart; lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; set_cpufuncs(); pcpu_init(pcpup, 0, sizeof(struct pcpu)); PCPU_SET(curthread, &thread0); /* Do basic tuning, hz etc */ init_param1(); freemempos = 0x00200000; /* Define a macro to simplify memory allocation */ #define valloc_pages(var, np) \ alloc_pages((var).pv_pa, (np)); \ (var).pv_va = (var).pv_pa + 0xc0000000; #define alloc_pages(var, np) \ freemempos -= (np * PAGE_SIZE); \ (var) = freemempos; \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos -= PAGE_SIZE; valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); for (loop = 0; loop < NUM_KERNEL_PTS; ++loop) { if (!(loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[loop], L2_TABLE_SIZE / PAGE_SIZE); } else { kernel_pt_table[loop].pv_pa = freemempos + (loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL)) * L2_TABLE_SIZE_REAL; kernel_pt_table[loop].pv_va = kernel_pt_table[loop].pv_pa + 0xc0000000; } } freemem_pt = freemempos; freemempos = 0x00100000; /* * Allocate a page for the system page mapped to V0x00000000 * This page will just contain the system vectors and can be * shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE); valloc_pages(abtstack, ABT_STACK_SIZE); valloc_pages(undstack, UND_STACK_SIZE); valloc_pages(kernelstack, kstack_pages); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Now we start construction of the L1 page table * We start by mapping the L2 page tables into the L1. * This means that we can replace L1 mappings later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* Map the L2 pages tables in the L1 page table */ pmap_link_l2pt(l1pagetable, rounddown2(ARM_VECTORS_HIGH, 0x00100000), &kernel_pt_table[KERNEL_PT_SYS]); pmap_map_chunk(l1pagetable, KERNBASE, SDRAM_START, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, KERNBASE + 0x100000, SDRAM_START + 0x100000, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, KERNBASE + 0x200000, SDRAM_START + 0x200000, rounddown2(((uint32_t)(lastaddr) - KERNBASE - 0x200000) + L1_S_SIZE, L1_S_SIZE), VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); freemem_after = rounddown2((int)lastaddr + PAGE_SIZE, PAGE_SIZE); afterkern = round_page(rounddown2((vm_offset_t)lastaddr + L1_S_SIZE, L1_S_SIZE)); for (i = 0; i < KERNEL_PT_AFKERNEL_NUM; i++) { pmap_link_l2pt(l1pagetable, afterkern + i * 0x00100000, &kernel_pt_table[KERNEL_PT_AFKERNEL + i]); } /* Map the vector page. */ pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); devmap_bootstrap(l1pagetable, iq81342_devmap); /* * Give the XScale global cache clean code an appropriately * sized chunk of unmapped VA space starting at 0xff000000 * (our device mappings end before this address). */ xscale_cache_clean_addr = 0xff000000U; cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); cpu_setup(); i80321_calibrate_delay(); i81342_sdram_bounds(obio_bs_tag, IOP34X_VADDR, &memstart, &memsize); physmem = memsize / PAGE_SIZE; cninit(); /* Set stack for exception handlers */ undefined_init(); init_proc0(kernelstack.pv_va); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_curmaxkvaddr = afterkern + PAGE_SIZE; vm_max_kernel_address = 0xe0000000; pmap_bootstrap(pmap_curmaxkvaddr, &kernel_l1pt); msgbufp = (void*)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Add the physical ram we have available. * * Exclude the kernel (and all the things we allocated which immediately * follow the kernel) from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_hardware_region(SDRAM_START, memsize); arm_physmem_exclude_region(freemem_pt, abp->abp_physaddr - freemem_pt, EXFLAG_NOALLOC); arm_physmem_exclude_region(freemempos, abp->abp_physaddr - 0x100000 - freemempos, EXFLAG_NOALLOC); arm_physmem_exclude_region(abp->abp_physaddr, virtual_avail - KERNVIRTADDR, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); } Index: head/sys/arm/xscale/i8134x/i81342reg.h =================================================================== --- head/sys/arm/xscale/i8134x/i81342reg.h (revision 299068) +++ head/sys/arm/xscale/i8134x/i81342reg.h (revision 299069) @@ -1,348 +1,348 @@ /*- * Copyright (c) 2006 Olivier Houchard * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* $FreeBSD$ */ #ifndef I83142_REG_H_ #define I83142_REG_H_ /* Physical Memory Map */ /* * 0x000000000 - 0x07FFFFFFF SDRAM * 0x090100000 - 0x0901FFFFF ATUe Outbound IO Window * 0x0F0000000 - 0x0F1FFFFFF Flash * 0x0F2000000 - 0x0F20FFFFF PCE1 * 0x0F3000000 - 0x0FFCFFFFF Compact Flash * 0x0FFD00000 - 0x0FFDFFFFF MMR * 0x0FFFB0000 - 0x0FFFBFFFF ATU-X Outbound I/O Window * 0x0FFFD0000 - 0x0FFFDFFFF ATUe Outbound I/O Window * 0x100000000 - 0x1FFFFFFFF ATU-X outbound Memory Translation Window * 0x2FF000000 - 0x2FFFFFFFF ATUe Outbound Memory Translation Window */ #define IOP34X_VADDR 0xf0000000 #define IOP34X_HWADDR 0xffd00000 #define IOP34X_SIZE 0x100000 #define IOP34X_ADMA0_OFFSET 0x00080000 #define IOP34X_ADMA1_OFFSET 0x00080200 #define IOP34X_ADMA2_OFFSET 0x00080400 #define IOP34X_ADMA_SIZE 0x200 /* ADMA Registers */ #define IOP34X_ADMA_CCR 0x0000 /* Channel Control Register */ #define IOP34X_ADMA_CSR 0x0004 /* Channel Status Register */ #define IOP34X_ADMA_DAR 0x0008 /* Descriptor Address Register */ #define IOP34X_ADMA_IPCR 0x0018 /* Internal Interface Parity Ctrl Reg */ #define IOP34X_ADMA_NDAR 0x0024 /* Next Descriptor Register */ #define IOP34X_ADMA_DCR 0x0028 /* Descriptor Control Register */ #define IOP34X_ADMA_IE (1 << 0) /* Interrupt enable */ #define IOP34X_ADMA_TR (1 << 1) /* Transfert Direction */ /* * Source Destination * 00 Host I/O Interface Local Memory * 01 Local Memory Host I/O Interface * 10 Internal Bus Local Memory * 11 Local Memory Internal Bus */ #define IOP34X_ADMA_SS (1 << 3) /* Source selection */ /* 0000: Data Transfer / CRC / Memory Block Fill */ #define IOP34X_ADMA_ZRBCE (1 << 7) /* Zero Result Buffer Check Enable */ #define IOP34X_ADMA_MBFE (1 << 8) /* Memory Block Fill Enable */ #define IOP34X_ADMA_CGE (1 << 9) /* CRC Generation enable */ #define IOP34X_ADMA_CTD (1 << 10) /* CRC Transfer disable */ #define IOP34X_ADMA_CSFD (1 << 11) /* CRC Seed fetch disable */ #define IOP34X_ADMA_SWBE (1 << 12) /* Status write back enable */ #define IOP34X_ADMA_ESE (1 << 13) /* Endian swap enable */ #define IOP34X_ADMA_PQUTE (1 << 16) /* P+Q Update Transfer Enable */ #define IOP34X_ADMA_DXE (1 << 17) /* Dual XOR Enable */ #define IOP34X_ADMA_PQTE (1 << 18) /* P+Q Transfer Enable */ #define IOP34X_ADMA_PTD (1 << 19) /* P Transfer Disable */ #define IOP34X_ADMA_ROE (1 << 30) /* Relaxed Ordering Enable */ #define IOP34X_ADMA_NSE (1U << 31) /* No Snoop Enable */ #define IOP34X_PBBAR0 0x81588 /* PBI Base Address Register 0 */ #define IOP34X_PBBAR0_ADDRMASK 0xfffff000 #define IOP34X_PBBAR1 0x81590 #define IOP34X_PCE1 0xF2000000 #define IOP34X_PCE1_SIZE 0x00100000 #define IOP34X_PCE1_VADDR 0xF1000000 #define IOP34X_ESSTSR0 0x82188 #define IOP34X_CONTROLLER_ONLY (1 << 14) #define IOP34X_INT_SEL_PCIX (1 << 15) #define IOP34X_PFR 0x82180 /* Processor Frequency Register */ #define IOP34X_FREQ_MASK ((1 << 16) | (1 << 17) | (1 << 18)) #define IOP34X_FREQ_600 (0) #define IOP34X_FREQ_667 (1 << 16) #define IOP34X_FREQ_800 (1 << 17) #define IOP34X_FREQ_833 ((1 << 17) | (1 << 16)) #define IOP34X_FREQ_1000 (1 << 18) #define IOP34X_FREQ_1200 ((1 << 16) | (1 << 18)) #define IOP34X_UART0_VADDR IOP34X_VADDR + 0x82300 #define IOP34X_UART0_HWADDR IOP34X_HWADDR + 0x82300 #define IOP34X_UART1_VADDR IOP34X_VADDR + 0x82340 #define IOP34X_UART1_HWADDR IOP34X_HWADDR + 0x82340 #define IOP34X_PBI_HWADDR 0xffd81580 /* SDRAM Memory Controller */ #define SMC_SDBR 0x8180c /* Base Register */ #define SMC_SDBR_BASEADDR (1 << 27) #define SMC_SDBR_BASEADDR_MASK ((1 << 27) | (1 << 28) | (1 << 29) | (1 << 30) \ | (1U << 31)) #define SMC_SDUBR 0x81810 /* Upper Base Register */ #define SMC_SBSR 0x81814 /* SDRAM Bank Size Register */ #define SMC_SBSR_BANK_NB (1 << 2) /* Number of DDR Banks 0 => 2 Banks 1 => 1 Bank */ #define SMC_SBSR_BANK_SZ (1 << 27) /* SDRAM Bank Size : 0x00000 Empty 0x00001 128MB 0x00010 256MB 0x00100 512MB 0x01000 1GB */ #define SMC_SBSR_BANK_SZ_MASK ((1 << 27) | (1 << 28) | (1 << 29) | (1 << 30) \ | (1U << 31)) /* Two possible addresses for ATUe depending on configuration. */ #define IOP34X_ATUE_ADDR(esstrsr) ((((esstrsr) & (IOP34X_CONTROLLER_ONLY | \ IOP34X_INT_SEL_PCIX)) == (IOP34X_CONTROLLER_ONLY | IOP34X_INT_SEL_PCIX)) ? \ 0xffdc8000 : 0xffdcd000) /* Three possible addresses for ATU-X depending on configuration. */ #define IOP34X_ATUX_ADDR(esstrsr) (!((esstrsr) & IOP34X_CONTROLLER_ONLY) ? \ 0xffdcc000 : !((esstrsr) & IOP34X_INT_SEL_PCIX) ? 0xffdc8000 : 0xffdcd000) #define IOP34X_OIOBAR_SIZE 0x10000 #define IOP34X_PCIX_OIOBAR 0xfffb0000 #define IOP34X_PCIX_OIOBAR_VADDR 0xf01b0000 #define IOP34X_PCIX_OMBAR 0x100000000 #define IOP34X_PCIE_OIOBAR 0xfffd0000 #define IOP34X_PCIE_OIOBAR_VADDR 0xf01d0000 #define IOP34X_PCIE_OMBAR 0x200000000 /* ATU Registers */ /* Common for ATU-X and ATUe */ #define ATU_VID 0x0000 /* ATU Vendor ID */ #define ATU_DID 0x0002 /* ATU Device ID */ #define ATU_CMD 0x0004 /* ATU Command Register */ #define ATU_SR 0x0006 /* ATU Status Register */ #define ATU_RID 0x0008 /* ATU Revision ID */ #define ATU_CCR 0x0009 /* ATU Class Code */ #define ATU_CLSR 0x000c /* ATU Cacheline Size */ #define ATU_LT 0x000d /* ATU Latency Timer */ #define ATU_HTR 0x000e /* ATU Header Type */ #define ATU_BISTR 0x000f /* ATU BIST Register */ #define ATU_IABAR0 0x0010 /* Inbound ATU Base Address register 0 */ #define ATU_IAUBAR0 0x0014 /* Inbound ATU Upper Base Address Register 0 */ #define ATU_IABAR1 0x0018 /* Inbound ATU Base Address Register 1 */ #define ATU_IAUBAR1 0x001c /* Inbound ATU Upper Base Address Register 1 */ #define ATU_IABAR2 0x0020 /* Inbound ATU Base Address Register 2 */ #define ATU_IAUBAR2 0x0024 /* Inbound ATU Upper Base Address Register 2 */ #define ATU_VSIR 0x002c /* ATU Subsystem Vendor ID Register */ #define ATU_SIR 0x002e /* ATU Subsystem ID Register */ #define ATU_ERBAR 0x0030 /* Expansion ROM Base Address Register */ #define ATU_CAPPTR 0x0034 /* ATU Capabilities Pointer Register */ #define ATU_ILR 0x003c /* ATU Interrupt Line Register */ #define ATU_IPR 0x003d /* ATU Interrupt Pin Register */ #define ATU_MGNT 0x003e /* ATU Minimum Grand Register */ #define ATU_MLAT 0x003f /* ATU Maximum Latency Register */ #define ATU_IALR0 0x0040 /* Inbound ATU Limit Register 0 */ #define ATU_IATVR0 0x0044 /* Inbound ATU Translate Value Register 0 */ #define ATU_IAUTVR0 0x0048 /* Inbound ATU Upper Translate Value Register 0*/ #define ATU_IALR1 0x004c /* Inbound ATU Limit Register 1 */ #define ATU_IATVR1 0x0050 /* Inbound ATU Translate Value Register 1 */ #define ATU_IAUTVR1 0x0054 /* Inbound ATU Upper Translate Value Register 1*/ #define ATU_IALR2 0x0058 /* Inbound ATU Limit Register 2 */ #define ATU_IATVR2 0x005c /* Inbound ATU Translate Value Register 2 */ #define ATU_IAUTVR2 0x0060 /* Inbound ATU Upper Translate Value Register 2*/ #define ATU_ERLR 0x0064 /* Expansion ROM Limit Register */ -#define ATU_ERTVR 0x0068 /* Expansion ROM Translater Value Register */ +#define ATU_ERTVR 0x0068 /* Expansion ROM Translator Value Register */ #define ATU_ERUTVR 0x006c /* Expansion ROM Upper Translate Value Register*/ #define ATU_CR 0x0070 /* ATU Configuration Register */ #define ATU_CR_OUT_EN (1 << 1) #define ATU_PCSR 0x0074 /* PCI Configuration and Status Register */ #define PCIE_BUSNO(x) ((x & 0xff000000) >> 24) #define ATUX_CORE_RST ((1 << 30) | (1U << 31)) /* Core Processor Reset */ #define ATUX_P_RSTOUT (1 << 21) /* Central Resource PCI Bus Reset */ #define ATUE_CORE_RST ((1 << 9) | (1 << 8)) /* Core Processor Reset */ #define ATU_ISR 0x0078 /* ATU Interrupt Status Register */ #define ATUX_ISR_PIE (1 << 18) /* PCI Interface error */ #define ATUX_ISR_IBPR (1 << 16) /* Internal Bus Parity Error */ #define ATUX_ISR_DCE (1 << 14) /* Detected Correctable error */ #define ATUX_ISR_ISCE (1 << 13) /* Initiated Split Completion Error Msg */ #define ATUX_ISR_RSCE (1 << 12) /* Received Split Completion Error Msg */ #define ATUX_ISR_DPE (1 << 9) /* Detected Parity Error */ #define ATUX_ISR_IBMA (1 << 7) /* Internal Bus Master Abort */ #define ATUX_ISR_PMA (1 << 3) /* PCI Master Abort */ #define ATUX_ISR_PTAM (1 << 2) /* PCI Target Abort (Master) */ #define ATUX_ISR_PTAT (1 << 1) /* PCI Target Abort (Target) */ #define ATUX_ISR_PMPE (1 << 0) /* PCI Master Parity Error */ #define ATUX_ISR_ERRMSK (ATUX_ISR_PIE | ATUX_ISR_IBPR | ATUX_ISR_DCE | \ ATUX_ISR_ISCE | ATUX_ISR_RSCE | ATUX_ISR_DPE | ATUX_ISR_IBMA | ATUX_ISR_PMA\ | ATUX_ISR_PTAM | ATUX_ISR_PTAT | ATUX_ISR_PMPE) #define ATUE_ISR_HON (1 << 13) /* Halt on Error Interrupt */ #define ATUE_ISR_RSE (1 << 12) /* Root System Error Message */ #define ATUE_ISR_REM (1 << 11) /* Root Error Message */ #define ATUE_ISR_PIE (1 << 10) /* PCI Interface error */ #define ATUE_ISR_CEM (1 << 9) /* Correctable Error Message */ #define ATUE_ISR_UEM (1 << 8) /* Uncorrectable error message */ #define ATUE_ISR_CRS (1 << 7) /* Received Configuration Retry Status */ #define ATUE_ISR_IBMA (1 << 5) /* Internal Bus Master Abort */ #define ATUE_ISR_DPE (1 << 4) /* Detected Parity Error Interrupt */ #define ATUE_ISR_MAI (1 << 3) /* Received Master Abort Interrupt */ #define ATUE_ISR_STAI (1 << 2) /* Signaled Target Abort Interrupt */ #define ATUE_ISR_TAI (1 << 1) /* Received Target Abort Interrupt */ #define ATUE_ISR_MDPE (1 << 0) /* Master Data Parity Error Interrupt */ #define ATUE_ISR_ERRMSK (ATUE_ISR_HON | ATUE_ISR_RSE | ATUE_ISR_REM | \ ATUE_ISR_PIE | ATUE_ISR_CEM | ATUE_ISR_UEM | ATUE_ISR_CRS | ATUE_ISR_IBMA |\ ATUE_ISR_DPE | ATUE_ISR_MAI | ATUE_ISR_STAI | ATUE_ISR_TAI | ATUE_ISR_MDPE) #define ATU_IMR 0x007c /* ATU Interrupt Mask Register */ /* 0x0080 - 0x008f reserved */ #define ATU_VPDCID 0x0090 /* VPD Capability Identifier Register */ #define ATU_VPDNIP 0x0091 /* VPD Next Item Pointer Register */ #define ATU_VPDAR 0x0092 /* VPD Address Register */ #define ATU_VPDDR 0x0094 /* VPD Data Register */ #define ATU_PMCID 0x0098 /* PM Capability Identifier Register */ #define ATU_PMNIPR 0x0099 /* PM Next Item Pointer Register */ #define ATU_PMCR 0x009a /* ATU Power Management Capabilities Register */ #define ATU_PMCSR 0x009c /* ATU Power Management Control/Status Register*/ #define ATU_MSICIR 0x00a0 /* MSI Capability Identifier Register */ #define ATU_MSINIPR 0x00a1 /* MSI Next Item Pointer Register */ #define ATU_MCR 0x00a2 /* Message Control Register */ #define ATU_MAR 0x00a4 /* Message Address Register */ #define ATU_MUAR 0x00a8 /* Message Upper Address Register */ #define ATU_MDR 0x00ac /* Message Data Register */ #define ATU_PCIXSR 0x00d4 /* PCI-X Status Register */ #define PCIXSR_BUSNO(x) (((x) & 0xff00) >> 8) #define ATU_IABAR3 0x0200 /* Inbound ATU Base Address Register 3 */ #define ATU_IAUBAR3 0x0204 /* Inbound ATU Upper Base Address Register 3 */ #define ATU_IALR3 0x0208 /* Inbound ATU Limit Register 3 */ #define ATU_ITVR3 0x020c /* Inbound ATU Upper Translate Value Reg 3 */ #define ATU_OIOBAR 0x0300 /* Outbound I/O Base Address Register */ #define ATU_OIOWTVR 0x0304 /* Outbound I/O Window Translate Value Reg */ #define ATU_OUMBAR0 0x0308 /* Outbound Upper Memory Window base addr reg 0*/ #define ATU_OUMBAR_FUNC (28) #define ATU_OUMBAR_EN (1U << 31) #define ATU_OUMWTVR0 0x030c /* Outbound Upper 32bit Memory Window Translate Value Register 0 */ #define ATU_OUMBAR1 0x0310 /* Outbound Upper Memory Window base addr reg1*/ #define ATU_OUMWTVR1 0x0314 /* Outbound Upper 32bit Memory Window Translate Value Register 1 */ #define ATU_OUMBAR2 0x0318 /* Outbound Upper Memory Window base addr reg2*/ #define ATU_OUMWTVR2 0x031c /* Outbount Upper 32bit Memory Window Translate Value Register 2 */ #define ATU_OUMBAR3 0x0320 /* Outbound Upper Memory Window base addr reg3*/ #define ATU_OUMWTVR3 0x0324 /* Outbound Upper 32bit Memory Window Translate Value Register 3 */ /* ATU-X specific */ #define ATUX_OCCAR 0x0330 /* Outbound Configuration Cycle Address Reg */ #define ATUX_OCCDR 0x0334 /* Outbound Configuration Cycle Data Reg */ #define ATUX_OCCFN 0x0338 /* Outbound Configuration Cycle Function Number*/ /* ATUe specific */ #define ATUE_OCCAR 0x032c /* Outbound Configuration Cycle Address Reg */ #define ATUE_OCCDR 0x0330 /* Outbound Configuration Cycle Data Reg */ #define ATUE_OCCFN 0x0334 /* Outbound Configuration Cycle Function Number*/ /* Interrupts */ /* IINTRSRC0 */ #define ICU_INT_ADMA0_EOT (0) /* ADMA 0 End of transfer */ #define ICU_INT_ADMA0_EOC (1) /* ADMA 0 End of Chain */ #define ICU_INT_ADMA1_EOT (2) /* ADMA 1 End of transfer */ #define ICU_INT_ADMA1_EOC (3) /* ADMA 1 End of chain */ #define ICU_INT_ADMA2_EOT (4) /* ADMA 2 End of transfer */ #define ICU_INT_ADMA2_EOC (5) /* ADMA 2 end of chain */ #define ICU_INT_WDOG (6) /* Watchdog timer */ /* 7 Reserved */ #define ICU_INT_TIMER0 (8) /* Timer 0 */ #define ICU_INT_TIMER1 (9) /* Timer 1 */ #define ICU_INT_I2C0 (10) /* I2C bus interface 0 */ #define ICU_INT_I2C1 (11) /* I2C bus interface 1 */ #define ICU_INT_MU (12) /* Message Unit */ #define ICU_INT_MU_IPQ (13) /* Message unit inbound post queue */ #define ICU_INT_ATUE_IM (14) /* ATU-E inbound message */ #define ICU_INT_ATU_BIST (15) /* ATU/Start BIST */ #define ICU_INT_PMC (16) /* PMC */ #define ICU_INT_PMU (17) /* PMU */ #define ICU_INT_PC (18) /* Processor cache */ /* 19-23 Reserved */ #define ICU_INT_XINT0 (24) #define ICU_INT_XINT1 (25) #define ICU_INT_XINT2 (26) #define ICU_INT_XINT3 (27) #define ICU_INT_XINT4 (28) #define ICU_INT_XINT5 (29) #define ICU_INT_XINT6 (30) #define ICU_INT_XINT7 (31) /* IINTSRC1 */ #define ICU_INT_XINT8 (32) #define ICU_INT_XINT9 (33) #define ICU_INT_XINT10 (34) #define ICU_INT_XINT11 (35) #define ICU_INT_XINT12 (36) #define ICU_INT_XINT13 (37) #define ICU_INT_XINT14 (38) #define ICU_INT_XINT15 (39) /* 40-50 reserved */ #define ICU_INT_UART0 (51) /* UART 0 */ #define ICU_INT_UART1 (52) /* UART 1 */ #define ICU_INT_PBIUE (53) /* Peripheral bus interface unit error */ #define ICU_INT_ATUCRW (54) /* ATU Configuration register write */ #define ICU_INT_ATUE (55) /* ATU error */ #define ICU_INT_MCUE (56) /* Memory controller unit error */ #define ICU_INT_ADMA0E (57) /* ADMA Channel 0 error */ #define ICU_INT_ADMA1E (58) /* ADMA Channel 1 error */ #define ICU_INT_ADMA2E (59) /* ADMA Channel 2 error */ /* 60-61 reserved */ #define ICU_INT_MUE (62) /* Messaging Unit Error */ /* 63 reserved */ /* IINTSRC2 */ #define ICU_INT_IP (64) /* Inter-processor */ /* 65-93 reserved */ #define ICU_INT_SIBBE (94) /* South internal bus bridge error */ /* 95 reserved */ /* IINTSRC3 */ #define ICU_INT_I2C2 (96) /* I2C bus interface 2 */ #define ICU_INT_ATUE_BIST (97) /* ATU-E/Start BIST */ #define ICU_INT_ATUE_CRW (98) /* ATU-E Configuration register write */ #define ICU_INT_ATUEE (99) /* ATU-E Error */ #define ICU_INT_IMU (100) /* IMU */ /* 101-106 reserved */ #define ICU_INT_ATUE_MA (107) /* ATUE Interrupt message A */ #define ICU_INT_ATUE_MB (108) /* ATUE Interrupt message B */ #define ICU_INT_ATUE_MC (109) /* ATUE Interrupt message C */ #define ICU_INT_ATUE_MD (110) /* ATUE Interrupt message D */ #define ICU_INT_MU_MSIX_TW (111) /* MU MSI-X Table write */ /* 112 reserved */ #define ICU_INT_IMSI (113) /* Inbound MSI */ /* 114-126 reserved */ #define ICU_INT_HPI (127) /* HPI */ #endif /* I81342_REG_H_ */ Index: head/sys/arm/xscale/ixp425/avila_machdep.c =================================================================== --- head/sys/arm/xscale/ixp425/avila_machdep.c (revision 299068) +++ head/sys/arm/xscale/ixp425/avila_machdep.c (revision 299069) @@ -1,417 +1,417 @@ /* $NetBSD: hpc_machdep.c,v 1.70 2003/09/16 08:18:22 agc Exp $ */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * machdep.c * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * This file needs a lot of work. * * Created : 17/09/94 */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define KERNEL_PT_SYS 0 /* Page table for mapping proc0 zero page */ #define KERNEL_PT_IO 1 #define KERNEL_PT_IO_NUM 3 #define KERNEL_PT_BEFOREKERN KERNEL_PT_IO + KERNEL_PT_IO_NUM #define KERNEL_PT_AFKERNEL KERNEL_PT_BEFOREKERN + 1 /* L2 table for mapping after kernel */ #define KERNEL_PT_AFKERNEL_NUM 9 /* this should be evenly divisable by PAGE_SIZE / L2_TABLE_SIZE_REAL (or 4) */ #define NUM_KERNEL_PTS (KERNEL_PT_AFKERNEL + KERNEL_PT_AFKERNEL_NUM) struct pv_addr kernel_pt_table[NUM_KERNEL_PTS]; /* Physical and virtual addresses for some global pages */ struct pv_addr systempage; struct pv_addr msgbufpv; struct pv_addr irqstack; struct pv_addr undstack; struct pv_addr abtstack; struct pv_addr kernelstack; struct pv_addr minidataclean; /* Static device mappings. */ static const struct devmap_entry ixp425_devmap[] = { /* Physical/Virtual address for I/O space */ { IXP425_IO_VBASE, IXP425_IO_HWBASE, IXP425_IO_SIZE, }, /* Expansion Bus */ { IXP425_EXP_VBASE, IXP425_EXP_HWBASE, IXP425_EXP_SIZE, }, /* CFI Flash on the Expansion Bus */ { IXP425_EXP_BUS_CS0_VBASE, IXP425_EXP_BUS_CS0_HWBASE, IXP425_EXP_BUS_CS0_SIZE, }, /* IXP425 PCI Configuration */ { IXP425_PCI_VBASE, IXP425_PCI_HWBASE, IXP425_PCI_SIZE, }, /* SDRAM Controller */ { IXP425_MCU_VBASE, IXP425_MCU_HWBASE, IXP425_MCU_SIZE, }, /* PCI Memory Space */ { IXP425_PCI_MEM_VBASE, IXP425_PCI_MEM_HWBASE, IXP425_PCI_MEM_SIZE, }, /* Q-Mgr Memory Space */ { IXP425_QMGR_VBASE, IXP425_QMGR_HWBASE, IXP425_QMGR_SIZE, }, { 0 }, }; /* Static device mappings. */ static const struct devmap_entry ixp435_devmap[] = { /* Physical/Virtual address for I/O space */ { IXP425_IO_VBASE, IXP425_IO_HWBASE, IXP425_IO_SIZE, }, { IXP425_EXP_VBASE, IXP425_EXP_HWBASE, IXP425_EXP_SIZE, }, /* IXP425 PCI Configuration */ { IXP425_PCI_VBASE, IXP425_PCI_HWBASE, IXP425_PCI_SIZE, }, /* DDRII Controller NB: mapped same place as IXP425 */ { IXP425_MCU_VBASE, IXP435_MCU_HWBASE, IXP425_MCU_SIZE, }, /* PCI Memory Space */ { IXP425_PCI_MEM_VBASE, IXP425_PCI_MEM_HWBASE, IXP425_PCI_MEM_SIZE, }, /* Q-Mgr Memory Space */ { IXP425_QMGR_VBASE, IXP425_QMGR_HWBASE, IXP425_QMGR_SIZE, }, /* CFI Flash on the Expansion Bus */ { IXP425_EXP_BUS_CS0_VBASE, IXP425_EXP_BUS_CS0_HWBASE, IXP425_EXP_BUS_CS0_SIZE, }, /* USB1 Memory Space */ { IXP435_USB1_VBASE, IXP435_USB1_HWBASE, IXP435_USB1_SIZE, }, /* USB2 Memory Space */ { IXP435_USB2_VBASE, IXP435_USB2_HWBASE, IXP435_USB2_SIZE, }, /* GPS Memory Space */ { CAMBRIA_GPS_VBASE, CAMBRIA_GPS_HWBASE, CAMBRIA_GPS_SIZE, }, /* RS485 Memory Space */ { CAMBRIA_RS485_VBASE, CAMBRIA_RS485_HWBASE, CAMBRIA_RS485_SIZE, }, { 0 } }; extern vm_offset_t xscale_cache_clean_addr; void * initarm(struct arm_boot_params *abp) { #define next_chunk2(a,b) (((a) + (b)) &~ ((b)-1)) #define next_page(a) next_chunk2(a,PAGE_SIZE) struct pv_addr kernel_l1pt; struct pv_addr dpcpu; int loop, i; u_int l1pagetable; vm_offset_t freemempos; vm_offset_t freemem_pt; vm_offset_t afterkern; vm_offset_t freemem_after; vm_offset_t lastaddr; uint32_t memsize; /* kernel text starts where we were loaded at boot */ #define KERNEL_TEXT_OFF (abp->abp_physaddr - PHYSADDR) #define KERNEL_TEXT_BASE (KERNBASE + KERNEL_TEXT_OFF) #define KERNEL_TEXT_PHYS (PHYSADDR + KERNEL_TEXT_OFF) lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; set_cpufuncs(); /* NB: sets cputype */ pcpu_init(pcpup, 0, sizeof(struct pcpu)); PCPU_SET(curthread, &thread0); init_static_kenv(NULL, 0); /* Do basic tuning, hz etc */ init_param1(); /* * We allocate memory downwards from where we were loaded * by RedBoot; first the L1 page table, then NUM_KERNEL_PTS * entries in the L2 page table. Past that we re-align the * allocation boundary so later data structures (stacks, etc) * can be mapped with different attributes (write-back vs * write-through). Note this leaves a gap for expansion * (or might be repurposed). */ freemempos = abp->abp_physaddr; /* macros to simplify initial memory allocation */ #define alloc_pages(var, np) do { \ freemempos -= (np * PAGE_SIZE); \ (var) = freemempos; \ /* NB: this works because locore maps PA=VA */ \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); \ } while (0) #define valloc_pages(var, np) do { \ alloc_pages((var).pv_pa, (np)); \ (var).pv_va = (var).pv_pa + (KERNVIRTADDR - abp->abp_physaddr); \ } while (0) /* force L1 page table alignment */ while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos -= PAGE_SIZE; /* allocate contiguous L1 page table */ valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); /* now allocate L2 page tables; they are linked to L1 below */ for (loop = 0; loop < NUM_KERNEL_PTS; ++loop) { if (!(loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[loop], L2_TABLE_SIZE / PAGE_SIZE); } else { kernel_pt_table[loop].pv_pa = freemempos + (loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL)) * L2_TABLE_SIZE_REAL; kernel_pt_table[loop].pv_va = kernel_pt_table[loop].pv_pa + (KERNVIRTADDR - abp->abp_physaddr); } } freemem_pt = freemempos; /* base of allocated pt's */ /* * Re-align allocation boundary so we can map the area * write-back instead of write-through for the stacks and * related structures allocated below. */ freemempos = PHYSADDR + 0x100000; /* * Allocate a page for the system page mapped to V0x00000000 * This page will just contain the system vectors and can be * shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE); valloc_pages(abtstack, ABT_STACK_SIZE); valloc_pages(undstack, UND_STACK_SIZE); valloc_pages(kernelstack, kstack_pages); alloc_pages(minidataclean.pv_pa, 1); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Now construct the L1 page table. First map the L2 * page tables into the L1 so we can replace L1 mappings * later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* Map the L2 pages tables in the L1 page table */ pmap_link_l2pt(l1pagetable, rounddown2(ARM_VECTORS_HIGH, 0x00100000), &kernel_pt_table[KERNEL_PT_SYS]); pmap_link_l2pt(l1pagetable, IXP425_IO_VBASE, &kernel_pt_table[KERNEL_PT_IO]); pmap_link_l2pt(l1pagetable, IXP425_MCU_VBASE, &kernel_pt_table[KERNEL_PT_IO + 1]); pmap_link_l2pt(l1pagetable, IXP425_PCI_MEM_VBASE, &kernel_pt_table[KERNEL_PT_IO + 2]); pmap_link_l2pt(l1pagetable, KERNBASE, &kernel_pt_table[KERNEL_PT_BEFOREKERN]); pmap_map_chunk(l1pagetable, KERNBASE, PHYSADDR, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, KERNBASE + 0x100000, PHYSADDR + 0x100000, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, KERNEL_TEXT_BASE, KERNEL_TEXT_PHYS, next_chunk2(((uint32_t)lastaddr) - KERNEL_TEXT_BASE, L1_S_SIZE), VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); freemem_after = next_page((int)lastaddr); afterkern = round_page(next_chunk2((vm_offset_t)lastaddr, L1_S_SIZE)); for (i = 0; i < KERNEL_PT_AFKERNEL_NUM; i++) { pmap_link_l2pt(l1pagetable, afterkern + i * 0x00100000, &kernel_pt_table[KERNEL_PT_AFKERNEL + i]); } pmap_map_entry(l1pagetable, afterkern, minidataclean.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map the Mini-Data cache clean area. */ xscale_setup_minidata(l1pagetable, afterkern, minidataclean.pv_pa); /* Map the vector page. */ pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); if (cpu_is_ixp43x()) devmap_bootstrap(l1pagetable, ixp435_devmap); else devmap_bootstrap(l1pagetable, ixp425_devmap); /* * Give the XScale global cache clean code an appropriately * sized chunk of unmapped VA space starting at 0xff000000 * (our device mappings end before this address). */ xscale_cache_clean_addr = 0xff000000U; cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); cpu_setup(); /* ready to setup the console (XXX move earlier if possible) */ cninit(); /* * Fetch the RAM size from the MCU registers. The * expansion bus was mapped above so we can now read 'em. */ if (cpu_is_ixp43x()) memsize = ixp435_ddram_size(); else memsize = ixp425_sdram_size(); undefined_init(); init_proc0(kernelstack.pv_va); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_curmaxkvaddr = afterkern + PAGE_SIZE; vm_max_kernel_address = 0xe0000000; pmap_bootstrap(pmap_curmaxkvaddr, &kernel_l1pt); msgbufp = (void*)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Add the physical ram we have available. * * Exclude the kernel, and all the things we allocated which immediately * follow the kernel, from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_hardware_region(PHYSADDR, memsize); arm_physmem_exclude_region(freemem_pt, abp->abp_physaddr - freemem_pt, EXFLAG_NOALLOC); arm_physmem_exclude_region(freemempos, abp->abp_physaddr - 0x100000 - freemempos, EXFLAG_NOALLOC); arm_physmem_exclude_region(abp->abp_physaddr, virtual_avail - KERNVIRTADDR, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); #undef next_page #undef next_chunk2 } Index: head/sys/arm/xscale/ixp425/cambria_gpio.c =================================================================== --- head/sys/arm/xscale/ixp425/cambria_gpio.c (revision 299068) +++ head/sys/arm/xscale/ixp425/cambria_gpio.c (revision 299069) @@ -1,503 +1,503 @@ /*- * Copyright (c) 2010, Andrew Thompson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * GPIO driver for Gateworks Cambria * * Note: * The Cambria PLD does not set the i2c ack bit after each write, if we used the * regular iicbus interface it would abort the xfer after the address byte * times out and not write our latch. To get around this we grab the iicbus and - * then do our own bit banging. This is a comprimise to changing all the iicbb + * then do our own bit banging. This is a compromise to changing all the iicbb * device methods to allow a flag to be passed down and is similir to how Linux * does it. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "iicbb_if.h" #include "gpio_if.h" #define IIC_M_WR 0 /* write operation */ #define PLD_ADDR 0xac /* slave address */ #define I2C_DELAY 10 #define GPIO_CONF_CLR(sc, reg, mask) \ GPIO_CONF_WRITE_4(sc, reg, GPIO_CONF_READ_4(sc, reg) &~ (mask)) #define GPIO_CONF_SET(sc, reg, mask) \ GPIO_CONF_WRITE_4(sc, reg, GPIO_CONF_READ_4(sc, reg) | (mask)) #define GPIO_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define GPIO_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define GPIO_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED) #define GPIO_PINS 5 struct cambria_gpio_softc { device_t sc_dev; device_t sc_busdev; bus_space_tag_t sc_iot; bus_space_handle_t sc_gpio_ioh; struct mtx sc_mtx; struct gpio_pin sc_pins[GPIO_PINS]; uint8_t sc_latch; uint8_t sc_val; }; struct cambria_gpio_pin { const char *name; int pin; int flags; }; extern struct ixp425_softc *ixp425_softc; static struct cambria_gpio_pin cambria_gpio_pins[GPIO_PINS] = { { "PLD0", 0, GPIO_PIN_OUTPUT }, { "PLD1", 1, GPIO_PIN_OUTPUT }, { "PLD2", 2, GPIO_PIN_OUTPUT }, { "PLD3", 3, GPIO_PIN_OUTPUT }, { "PLD4", 4, GPIO_PIN_OUTPUT }, }; /* * Helpers */ static int cambria_gpio_read(struct cambria_gpio_softc *, uint32_t, unsigned int *); static int cambria_gpio_write(struct cambria_gpio_softc *); /* * Driver stuff */ static int cambria_gpio_probe(device_t dev); static int cambria_gpio_attach(device_t dev); static int cambria_gpio_detach(device_t dev); /* * GPIO interface */ static device_t cambria_gpio_get_bus(device_t); static int cambria_gpio_pin_max(device_t dev, int *maxpin); static int cambria_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps); static int cambria_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags); static int cambria_gpio_pin_getname(device_t dev, uint32_t pin, char *name); static int cambria_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags); static int cambria_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value); static int cambria_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val); static int cambria_gpio_pin_toggle(device_t dev, uint32_t pin); static int i2c_getsda(struct cambria_gpio_softc *sc) { uint32_t reg; IXP4XX_GPIO_LOCK(); GPIO_CONF_SET(sc, IXP425_GPIO_GPOER, GPIO_I2C_SDA_BIT); reg = GPIO_CONF_READ_4(sc, IXP425_GPIO_GPINR); IXP4XX_GPIO_UNLOCK(); return (reg & GPIO_I2C_SDA_BIT); } static void i2c_setsda(struct cambria_gpio_softc *sc, int val) { IXP4XX_GPIO_LOCK(); GPIO_CONF_CLR(sc, IXP425_GPIO_GPOUTR, GPIO_I2C_SDA_BIT); if (val) GPIO_CONF_SET(sc, IXP425_GPIO_GPOER, GPIO_I2C_SDA_BIT); else GPIO_CONF_CLR(sc, IXP425_GPIO_GPOER, GPIO_I2C_SDA_BIT); IXP4XX_GPIO_UNLOCK(); DELAY(I2C_DELAY); } static void i2c_setscl(struct cambria_gpio_softc *sc, int val) { IXP4XX_GPIO_LOCK(); GPIO_CONF_CLR(sc, IXP425_GPIO_GPOUTR, GPIO_I2C_SCL_BIT); if (val) GPIO_CONF_SET(sc, IXP425_GPIO_GPOER, GPIO_I2C_SCL_BIT); else GPIO_CONF_CLR(sc, IXP425_GPIO_GPOER, GPIO_I2C_SCL_BIT); IXP4XX_GPIO_UNLOCK(); DELAY(I2C_DELAY); } static void i2c_sendstart(struct cambria_gpio_softc *sc) { i2c_setsda(sc, 1); i2c_setscl(sc, 1); i2c_setsda(sc, 0); i2c_setscl(sc, 0); } static void i2c_sendstop(struct cambria_gpio_softc *sc) { i2c_setscl(sc, 1); i2c_setsda(sc, 1); i2c_setscl(sc, 0); i2c_setsda(sc, 0); } static void i2c_sendbyte(struct cambria_gpio_softc *sc, u_char data) { int i; for (i=7; i>=0; i--) { i2c_setsda(sc, data & (1<=0; i--) { i2c_setscl(sc, 1); if (i2c_getsda(sc)) data |= (1<sc_dev; int error; error = iicbus_request_bus(device_get_parent(dev), dev, IIC_DONTWAIT); if (error) return (error); i2c_sendstart(sc); i2c_sendbyte(sc, PLD_ADDR | LSB); *val = (i2c_readbyte(sc) & (1 << pin)) != 0; i2c_sendstop(sc); iicbus_release_bus(device_get_parent(dev), dev); return (0); } static int cambria_gpio_write(struct cambria_gpio_softc *sc) { device_t dev = sc->sc_dev; int error; error = iicbus_request_bus(device_get_parent(dev), dev, IIC_DONTWAIT); if (error) return (error); i2c_sendstart(sc); i2c_sendbyte(sc, PLD_ADDR & ~LSB); i2c_sendbyte(sc, sc->sc_latch); i2c_sendstop(sc); iicbus_release_bus(device_get_parent(dev), dev); return (0); } static device_t cambria_gpio_get_bus(device_t dev) { struct cambria_gpio_softc *sc; sc = device_get_softc(dev); return (sc->sc_busdev); } static int cambria_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = GPIO_PINS - 1; return (0); } static int cambria_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct cambria_gpio_softc *sc = device_get_softc(dev); if (pin >= GPIO_PINS) return (EINVAL); *caps = sc->sc_pins[pin].gp_caps; return (0); } static int cambria_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct cambria_gpio_softc *sc = device_get_softc(dev); if (pin >= GPIO_PINS) return (EINVAL); *flags = sc->sc_pins[pin].gp_flags; return (0); } static int cambria_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct cambria_gpio_softc *sc = device_get_softc(dev); if (pin >= GPIO_PINS) return (EINVAL); memcpy(name, sc->sc_pins[pin].gp_name, GPIOMAXNAME); return (0); } static int cambria_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct cambria_gpio_softc *sc = device_get_softc(dev); int error; uint8_t mask; mask = 1 << pin; if (pin >= GPIO_PINS) return (EINVAL); GPIO_LOCK(sc); sc->sc_pins[pin].gp_flags = flags; /* * Writing a logical one sets the signal high and writing a logical * zero sets the signal low. To configure a digital I/O signal as an * input, a logical one must first be written to the data bit to * three-state the associated output. */ if (flags & GPIO_PIN_INPUT || sc->sc_val & mask) sc->sc_latch |= mask; /* input or output & high */ else sc->sc_latch &= ~mask; error = cambria_gpio_write(sc); GPIO_UNLOCK(sc); return (error); } static int cambria_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct cambria_gpio_softc *sc = device_get_softc(dev); int error; uint8_t mask; mask = 1 << pin; if (pin >= GPIO_PINS) return (EINVAL); GPIO_LOCK(sc); if (value) sc->sc_val |= mask; else sc->sc_val &= ~mask; if (sc->sc_pins[pin].gp_flags != GPIO_PIN_OUTPUT) { /* just save, altering the latch will disable input */ GPIO_UNLOCK(sc); return (0); } if (value) sc->sc_latch |= mask; else sc->sc_latch &= ~mask; error = cambria_gpio_write(sc); GPIO_UNLOCK(sc); return (error); } static int cambria_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct cambria_gpio_softc *sc = device_get_softc(dev); int error = 0; if (pin >= GPIO_PINS) return (EINVAL); GPIO_LOCK(sc); if (sc->sc_pins[pin].gp_flags == GPIO_PIN_OUTPUT) *val = (sc->sc_latch & (1 << pin)) ? 1 : 0; else error = cambria_gpio_read(sc, pin, val); GPIO_UNLOCK(sc); return (error); } static int cambria_gpio_pin_toggle(device_t dev, uint32_t pin) { struct cambria_gpio_softc *sc = device_get_softc(dev); int error = 0; if (pin >= GPIO_PINS) return (EINVAL); GPIO_LOCK(sc); sc->sc_val ^= (1 << pin); if (sc->sc_pins[pin].gp_flags == GPIO_PIN_OUTPUT) { sc->sc_latch ^= (1 << pin); error = cambria_gpio_write(sc); } GPIO_UNLOCK(sc); return (error); } static int cambria_gpio_probe(device_t dev) { device_set_desc(dev, "Gateworks Cambria GPIO driver"); return (0); } static int cambria_gpio_attach(device_t dev) { struct cambria_gpio_softc *sc = device_get_softc(dev); int pin; sc->sc_dev = dev; sc->sc_iot = ixp425_softc->sc_iot; sc->sc_gpio_ioh = ixp425_softc->sc_gpio_ioh; mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF); for (pin = 0; pin < GPIO_PINS; pin++) { struct cambria_gpio_pin *p = &cambria_gpio_pins[pin]; strncpy(sc->sc_pins[pin].gp_name, p->name, GPIOMAXNAME); sc->sc_pins[pin].gp_pin = pin; sc->sc_pins[pin].gp_caps = GPIO_PIN_INPUT|GPIO_PIN_OUTPUT; sc->sc_pins[pin].gp_flags = 0; cambria_gpio_pin_setflags(dev, pin, p->flags); } sc->sc_busdev = gpiobus_attach_bus(dev); if (sc->sc_busdev == NULL) { mtx_destroy(&sc->sc_mtx); return (ENXIO); } return (0); } static int cambria_gpio_detach(device_t dev) { struct cambria_gpio_softc *sc = device_get_softc(dev); KASSERT(mtx_initialized(&sc->sc_mtx), ("gpio mutex not initialized")); gpiobus_detach_bus(dev); mtx_destroy(&sc->sc_mtx); return(0); } static device_method_t cambria_gpio_methods[] = { DEVMETHOD(device_probe, cambria_gpio_probe), DEVMETHOD(device_attach, cambria_gpio_attach), DEVMETHOD(device_detach, cambria_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_get_bus, cambria_gpio_get_bus), DEVMETHOD(gpio_pin_max, cambria_gpio_pin_max), DEVMETHOD(gpio_pin_getname, cambria_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, cambria_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, cambria_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, cambria_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, cambria_gpio_pin_get), DEVMETHOD(gpio_pin_set, cambria_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, cambria_gpio_pin_toggle), {0, 0}, }; static driver_t cambria_gpio_driver = { "gpio", cambria_gpio_methods, sizeof(struct cambria_gpio_softc), }; static devclass_t cambria_gpio_devclass; DRIVER_MODULE(gpio_cambria, iicbus, cambria_gpio_driver, cambria_gpio_devclass, 0, 0); MODULE_VERSION(gpio_cambria, 1); MODULE_DEPEND(gpio_cambria, iicbus, 1, 1, 1); Index: head/sys/arm/xscale/ixp425/ixp425_npe.c =================================================================== --- head/sys/arm/xscale/ixp425/ixp425_npe.c (revision 299068) +++ head/sys/arm/xscale/ixp425/ixp425_npe.c (revision 299069) @@ -1,1574 +1,1574 @@ /*- * Copyright (c) 2006-2008 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. */ /*- * Copyright (c) 2001-2005, Intel Corporation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the Intel Corporation nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Intel XScale Network Processing Engine (NPE) support. * * Each NPE has an ixpnpeX device associated with it that is * attached at boot. Depending on the microcode loaded into * an NPE there may be an Ethernet interface (npeX) or some * other network interface (e.g. for ATM). This file has support * for loading microcode images and the associated NPE CPU * manipulations (start, stop, reset). * * The code here basically replaces the npeDl and npeMh classes * in the Intel Access Library (IAL). * * NB: Microcode images are loaded with firmware(9). To * include microcode in a static kernel include the * ixpnpe_fw device. Otherwise the firmware will be * automatically loaded from the filesystem. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct ixpnpe_softc { device_t sc_dev; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; bus_size_t sc_size; /* size of mapped register window */ struct resource *sc_irq; /* IRQ resource */ void *sc_ih; /* interrupt handler */ struct mtx sc_mtx; /* mailbox lock */ uint32_t sc_msg[2]; /* reply msg collected in ixpnpe_intr */ int sc_msgwaiting; /* sc_msg holds valid data */ int sc_npeid; int sc_nrefs; /* # of references */ int validImage; /* valid ucode image loaded */ int started; /* NPE is started */ uint8_t functionalityId;/* ucode functionality ID */ int insMemSize; /* size of instruction memory */ int dataMemSize; /* size of data memory */ uint32_t savedExecCount; uint32_t savedEcsDbgCtxtReg2; }; static struct ixpnpe_softc *npes[NPE_MAX]; #define IX_NPEDL_NPEIMAGE_FIELD_MASK 0xff /* used to read download map from version in microcode image */ #define IX_NPEDL_BLOCK_TYPE_INSTRUCTION 0x00000000 #define IX_NPEDL_BLOCK_TYPE_DATA 0x00000001 #define IX_NPEDL_BLOCK_TYPE_STATE 0x00000002 #define IX_NPEDL_END_OF_DOWNLOAD_MAP 0x0000000F /* * masks used to extract address info from State information context * register addresses as read from microcode image */ #define IX_NPEDL_MASK_STATE_ADDR_CTXT_REG 0x0000000F #define IX_NPEDL_MASK_STATE_ADDR_CTXT_NUM 0x000000F0 /* LSB offset of Context Number field in State-Info Context Address */ #define IX_NPEDL_OFFSET_STATE_ADDR_CTXT_NUM 4 /* size (in words) of single State Information entry (ctxt reg address|data) */ #define IX_NPEDL_STATE_INFO_ENTRY_SIZE 2 typedef struct { uint32_t type; uint32_t offset; } IxNpeDlNpeMgrDownloadMapBlockEntry; typedef union { IxNpeDlNpeMgrDownloadMapBlockEntry block; uint32_t eodmMarker; } IxNpeDlNpeMgrDownloadMapEntry; typedef struct { /* 1st entry in the download map (there may be more than one) */ IxNpeDlNpeMgrDownloadMapEntry entry[1]; } IxNpeDlNpeMgrDownloadMap; /* used to access an instruction or data block in a microcode image */ typedef struct { uint32_t npeMemAddress; uint32_t size; uint32_t data[1]; } IxNpeDlNpeMgrCodeBlock; /* used to access each Context Reg entry state-information block */ typedef struct { uint32_t addressInfo; uint32_t value; } IxNpeDlNpeMgrStateInfoCtxtRegEntry; /* used to access a state-information block in a microcode image */ typedef struct { uint32_t size; IxNpeDlNpeMgrStateInfoCtxtRegEntry ctxtRegEntry[1]; } IxNpeDlNpeMgrStateInfoBlock; static int npe_debug = 0; SYSCTL_INT(_debug, OID_AUTO, ixp425npe, CTLFLAG_RWTUN, &npe_debug, 0, "IXP4XX NPE debug msgs"); #define DPRINTF(dev, fmt, ...) do { \ if (npe_debug) device_printf(dev, fmt, __VA_ARGS__); \ } while (0) #define DPRINTFn(n, dev, fmt, ...) do { \ if (npe_debug >= n) printf(fmt, __VA_ARGS__); \ } while (0) static int npe_checkbits(struct ixpnpe_softc *, uint32_t reg, uint32_t); static int npe_isstopped(struct ixpnpe_softc *); static int npe_load_ins(struct ixpnpe_softc *, const IxNpeDlNpeMgrCodeBlock *bp, int verify); static int npe_load_data(struct ixpnpe_softc *, const IxNpeDlNpeMgrCodeBlock *bp, int verify); static int npe_load_stateinfo(struct ixpnpe_softc *, const IxNpeDlNpeMgrStateInfoBlock *bp, int verify); static int npe_load_image(struct ixpnpe_softc *, const uint32_t *imageCodePtr, int verify); static int npe_cpu_reset(struct ixpnpe_softc *); static int npe_cpu_start(struct ixpnpe_softc *); static int npe_cpu_stop(struct ixpnpe_softc *); static void npe_cmd_issue_write(struct ixpnpe_softc *, uint32_t cmd, uint32_t addr, uint32_t data); static uint32_t npe_cmd_issue_read(struct ixpnpe_softc *, uint32_t cmd, uint32_t addr); static int npe_ins_write(struct ixpnpe_softc *, uint32_t addr, uint32_t data, int verify); static int npe_data_write(struct ixpnpe_softc *, uint32_t addr, uint32_t data, int verify); static void npe_ecs_reg_write(struct ixpnpe_softc *, uint32_t reg, uint32_t data); static uint32_t npe_ecs_reg_read(struct ixpnpe_softc *, uint32_t reg); static void npe_issue_cmd(struct ixpnpe_softc *, uint32_t command); static void npe_cpu_step_save(struct ixpnpe_softc *); static int npe_cpu_step(struct ixpnpe_softc *, uint32_t npeInstruction, uint32_t ctxtNum, uint32_t ldur); static void npe_cpu_step_restore(struct ixpnpe_softc *); static int npe_logical_reg_read(struct ixpnpe_softc *, uint32_t regAddr, uint32_t regSize, uint32_t ctxtNum, uint32_t *regVal); static int npe_logical_reg_write(struct ixpnpe_softc *, uint32_t regAddr, uint32_t regVal, uint32_t regSize, uint32_t ctxtNum, int verify); static int npe_physical_reg_write(struct ixpnpe_softc *, uint32_t regAddr, uint32_t regValue, int verify); static int npe_ctx_reg_write(struct ixpnpe_softc *, uint32_t ctxtNum, uint32_t ctxtReg, uint32_t ctxtRegVal, int verify); static void ixpnpe_intr(void *arg); static uint32_t npe_reg_read(struct ixpnpe_softc *sc, bus_size_t off) { uint32_t v = bus_space_read_4(sc->sc_iot, sc->sc_ioh, off); DPRINTFn(9, sc->sc_dev, "%s(0x%lx) => 0x%x\n", __func__, off, v); return v; } static void npe_reg_write(struct ixpnpe_softc *sc, bus_size_t off, uint32_t val) { DPRINTFn(9, sc->sc_dev, "%s(0x%lx, 0x%x)\n", __func__, off, val); bus_space_write_4(sc->sc_iot, sc->sc_ioh, off, val); } struct ixpnpe_softc * ixpnpe_attach(device_t dev, int npeid) { struct npeconfig { uint32_t base; uint32_t size; int irq; uint32_t ins_memsize; uint32_t data_memsize; }; static const struct npeconfig npeconfigs[NPE_MAX] = { [NPE_A] = { .base = IXP425_NPE_A_HWBASE, .size = IXP425_NPE_A_SIZE, .irq = IXP425_INT_NPE_A, .ins_memsize = IX_NPEDL_INS_MEMSIZE_WORDS_NPEA, .data_memsize = IX_NPEDL_DATA_MEMSIZE_WORDS_NPEA }, [NPE_B] = { .base = IXP425_NPE_B_HWBASE, .size = IXP425_NPE_B_SIZE, .irq = IXP425_INT_NPE_B, .ins_memsize = IX_NPEDL_INS_MEMSIZE_WORDS_NPEB, .data_memsize = IX_NPEDL_DATA_MEMSIZE_WORDS_NPEB }, [NPE_C] = { .base = IXP425_NPE_C_HWBASE, .size = IXP425_NPE_C_SIZE, .irq = IXP425_INT_NPE_C, .ins_memsize = IX_NPEDL_INS_MEMSIZE_WORDS_NPEC, .data_memsize = IX_NPEDL_DATA_MEMSIZE_WORDS_NPEC }, }; struct ixp425_softc *sa = device_get_softc(device_get_parent(dev)); struct ixpnpe_softc *sc; const struct npeconfig *config; int rid; if (npeid >= NPE_MAX) { device_printf(dev, "%s: bad npeid %d\n", __func__, npeid); return NULL; } sc = npes[npeid]; if (sc != NULL) { sc->sc_nrefs++; return sc; } config = &npeconfigs[npeid]; /* XXX M_BUS */ sc = malloc(sizeof(struct ixpnpe_softc), M_TEMP, M_WAITOK | M_ZERO); sc->sc_dev = dev; sc->sc_iot = sa->sc_iot; mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "npe driver", MTX_DEF); sc->sc_npeid = npeid; sc->sc_nrefs = 1; sc->sc_size = config->size; if (cpu_is_ixp42x()) { /* NB: instruction/data memory sizes are NPE-dependent */ sc->insMemSize = config->ins_memsize; sc->dataMemSize = config->data_memsize; } else { sc->insMemSize = IXP46X_NPEDL_INS_MEMSIZE_WORDS; sc->dataMemSize = IXP46X_NPEDL_DATA_MEMSIZE_WORDS; } if (bus_space_map(sc->sc_iot, config->base, sc->sc_size, 0, &sc->sc_ioh)) panic("%s: Cannot map registers", device_get_name(dev)); /* * Setup IRQ and handler for NPE message support. */ rid = 0; sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, config->irq, config->irq, 1, RF_ACTIVE); if (sc->sc_irq == NULL) panic("%s: Unable to allocate irq %u", device_get_name(dev), config->irq); /* XXX could be a source of entropy */ bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixpnpe_intr, sc, &sc->sc_ih); /* * Enable output fifo interrupts (NB: must also set OFIFO Write Enable) */ npe_reg_write(sc, IX_NPECTL, npe_reg_read(sc, IX_NPECTL) | (IX_NPECTL_OFE | IX_NPECTL_OFWE)); npes[npeid] = sc; return sc; } void ixpnpe_detach(struct ixpnpe_softc *sc) { if (--sc->sc_nrefs == 0) { npes[sc->sc_npeid] = NULL; /* disable output fifo interrupts */ npe_reg_write(sc, IX_NPECTL, npe_reg_read(sc, IX_NPECTL) &~ (IX_NPECTL_OFE | IX_NPECTL_OFWE)); bus_teardown_intr(sc->sc_dev, sc->sc_irq, sc->sc_ih); bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_size); mtx_destroy(&sc->sc_mtx); free(sc, M_TEMP); } } int ixpnpe_stopandreset(struct ixpnpe_softc *sc) { int error; mtx_lock(&sc->sc_mtx); error = npe_cpu_stop(sc); /* stop NPE */ if (error == 0) error = npe_cpu_reset(sc); /* reset it */ if (error == 0) sc->started = 0; /* mark stopped */ mtx_unlock(&sc->sc_mtx); DPRINTF(sc->sc_dev, "%s: error %d\n", __func__, error); return error; } static int ixpnpe_start_locked(struct ixpnpe_softc *sc) { int error; if (!sc->started) { error = npe_cpu_start(sc); if (error == 0) sc->started = 1; } else error = 0; DPRINTF(sc->sc_dev, "%s: error %d\n", __func__, error); return error; } int ixpnpe_start(struct ixpnpe_softc *sc) { int ret; mtx_lock(&sc->sc_mtx); ret = ixpnpe_start_locked(sc); mtx_unlock(&sc->sc_mtx); return (ret); } int ixpnpe_stop(struct ixpnpe_softc *sc) { int error; mtx_lock(&sc->sc_mtx); error = npe_cpu_stop(sc); if (error == 0) sc->started = 0; mtx_unlock(&sc->sc_mtx); DPRINTF(sc->sc_dev, "%s: error %d\n", __func__, error); return error; } /* * Indicates the start of an NPE Image, in new NPE Image Library format. * 2 consecutive occurrences indicates the end of the NPE Image Library */ #define NPE_IMAGE_MARKER 0xfeedf00d /* * NPE Image Header definition, used in new NPE Image Library format */ typedef struct { uint32_t marker; uint32_t id; uint32_t size; } IxNpeDlImageMgrImageHeader; static int npe_findimage(struct ixpnpe_softc *sc, const uint32_t *imageLibrary, uint32_t imageId, const uint32_t **imagePtr, uint32_t *imageSize) { const IxNpeDlImageMgrImageHeader *image; uint32_t offset = 0; while (imageLibrary[offset] == NPE_IMAGE_MARKER) { image = (const IxNpeDlImageMgrImageHeader *) &imageLibrary[offset]; offset += sizeof(IxNpeDlImageMgrImageHeader)/sizeof(uint32_t); DPRINTF(sc->sc_dev, "%s: off %u mark 0x%x id 0x%x size %u\n", __func__, offset, image->marker, image->id, image->size); if (image->id == imageId) { *imagePtr = imageLibrary + offset; *imageSize = image->size; return 0; } /* 2 consecutive NPE_IMAGE_MARKER's indicates end of library */ if (image->id == NPE_IMAGE_MARKER) { DPRINTF(sc->sc_dev, "imageId 0x%08x not found in " "image library header\n", imageId); /* reached end of library, image not found */ return ESRCH; } offset += image->size; } return ESRCH; } static int ixpnpe_load_firmware(struct ixpnpe_softc *sc, const char *imageName, uint32_t imageId) { static const char *devname[4] = { "IXP425", "IXP435/IXP465", "DeviceID#2", "DeviceID#3" }; uint32_t imageSize; const uint32_t *imageCodePtr; const struct firmware *fw; int error; DPRINTF(sc->sc_dev, "load %s, imageId 0x%08x\n", imageName, imageId); #if 0 IxFeatureCtrlDeviceId devid = IX_NPEDL_DEVICEID_FROM_IMAGEID_GET(imageId); /* * Checking if image being loaded is meant for device that is running. * Image is forward compatible. i.e Image built for IXP42X should run * on IXP46X but not vice versa. */ if (devid > (ixFeatureCtrlDeviceRead() & IX_FEATURE_CTRL_DEVICE_TYPE_MASK)) return EINVAL; #endif error = ixpnpe_stopandreset(sc); /* stop and reset the NPE */ if (error != 0) return error; fw = firmware_get(imageName); if (fw == NULL) return ENOENT; /* Locate desired image in files w/ combined images */ error = npe_findimage(sc, fw->data, imageId, &imageCodePtr, &imageSize); if (error != 0) goto done; device_printf(sc->sc_dev, "load fw image %s.NPE-%c Func 0x%x Rev %u.%u\n", devname[NPEIMAGE_DEVID(imageId)], 'A' + NPEIMAGE_NPEID(imageId), NPEIMAGE_FUNCID(imageId), NPEIMAGE_MAJOR(imageId), NPEIMAGE_MINOR(imageId)); /* * If download was successful, store image Id in list of - * currently loaded images. If a critical error occured + * currently loaded images. If a critical error occurred * during download, record that the NPE has an invalid image */ mtx_lock(&sc->sc_mtx); error = npe_load_image(sc, imageCodePtr, 1 /*VERIFY*/); if (error == 0) { sc->validImage = 1; error = ixpnpe_start_locked(sc); } else { sc->validImage = 0; } sc->functionalityId = IX_NPEDL_FUNCTIONID_FROM_IMAGEID_GET(imageId); mtx_unlock(&sc->sc_mtx); done: firmware_put(fw, FIRMWARE_UNLOAD); DPRINTF(sc->sc_dev, "%s: error %d\n", __func__, error); return error; } static int override_imageid(device_t dev, const char *resname, uint32_t *val) { int unit = device_get_unit(dev); int resval; if (resource_int_value("npe", unit, resname, &resval) != 0) return 0; /* XXX validate */ if (bootverbose) device_printf(dev, "using npe.%d.%s=0x%x override\n", unit, resname, resval); *val = resval; return 1; } int ixpnpe_init(struct ixpnpe_softc *sc) { static const uint32_t npeconfig[NPE_MAX] = { [NPE_A] = IXP425_NPE_A_IMAGEID, [NPE_B] = IXP425_NPE_B_IMAGEID, [NPE_C] = IXP425_NPE_C_IMAGEID, }; uint32_t imageid, msg[2]; int error; if (sc->started) return 0; /* * Load NPE firmware and start it running. We assume * that minor version bumps remain compatible so probe * the firmware image starting with the expected version * and then bump the minor version up to the max. */ if (!override_imageid(sc->sc_dev, "imageid", &imageid)) imageid = npeconfig[sc->sc_npeid]; for (;;) { error = ixpnpe_load_firmware(sc, "npe_fw", imageid); if (error == 0) break; /* * ESRCH is returned when the requested image * is not present */ if (error != ESRCH) { device_printf(sc->sc_dev, "cannot init NPE (error %d)\n", error); return error; } /* bump the minor version up to the max possible */ if (NPEIMAGE_MINOR(imageid) == 0xff) { device_printf(sc->sc_dev, "cannot locate firmware " "(imageid 0x%08x)\n", imageid); return error; } imageid++; } /* NB: firmware should respond with a status msg */ if (ixpnpe_recvmsg_sync(sc, msg) != 0) { device_printf(sc->sc_dev, "firmware did not respond as expected\n"); return EIO; } return 0; } int ixpnpe_getfunctionality(struct ixpnpe_softc *sc) { return (sc->validImage ? sc->functionalityId : 0); } static int npe_checkbits(struct ixpnpe_softc *sc, uint32_t reg, uint32_t expectedBitsSet) { uint32_t val; val = npe_reg_read(sc, reg); DPRINTFn(5, sc->sc_dev, "%s(0x%x, 0x%x) => 0x%x (%u)\n", __func__, reg, expectedBitsSet, val, (val & expectedBitsSet) == expectedBitsSet); return ((val & expectedBitsSet) == expectedBitsSet); } static int npe_isstopped(struct ixpnpe_softc *sc) { return npe_checkbits(sc, IX_NPEDL_REG_OFFSET_EXCTL, IX_NPEDL_EXCTL_STATUS_STOP); } static int npe_load_ins(struct ixpnpe_softc *sc, const IxNpeDlNpeMgrCodeBlock *bp, int verify) { uint32_t npeMemAddress; int i, blockSize; npeMemAddress = bp->npeMemAddress; blockSize = bp->size; /* NB: instruction/data count */ if (npeMemAddress + blockSize > sc->insMemSize) { device_printf(sc->sc_dev, "Block size %u too big for NPE memory\n", blockSize); return EINVAL; /* XXX */ } for (i = 0; i < blockSize; i++, npeMemAddress++) { if (npe_ins_write(sc, npeMemAddress, bp->data[i], verify) != 0) { device_printf(sc->sc_dev, "NPE instruction write failed"); return EIO; } } return 0; } static int npe_load_data(struct ixpnpe_softc *sc, const IxNpeDlNpeMgrCodeBlock *bp, int verify) { uint32_t npeMemAddress; int i, blockSize; npeMemAddress = bp->npeMemAddress; blockSize = bp->size; /* NB: instruction/data count */ if (npeMemAddress + blockSize > sc->dataMemSize) { device_printf(sc->sc_dev, "Block size %u too big for NPE memory\n", blockSize); return EINVAL; } for (i = 0; i < blockSize; i++, npeMemAddress++) { if (npe_data_write(sc, npeMemAddress, bp->data[i], verify) != 0) { device_printf(sc->sc_dev, "NPE data write failed\n"); return EIO; } } return 0; } static int npe_load_stateinfo(struct ixpnpe_softc *sc, const IxNpeDlNpeMgrStateInfoBlock *bp, int verify) { int i, nentries, error; npe_cpu_step_save(sc); /* for each state-info context register entry in block */ nentries = bp->size / IX_NPEDL_STATE_INFO_ENTRY_SIZE; error = 0; for (i = 0; i < nentries; i++) { /* each state-info entry is 2 words (address, value) */ uint32_t regVal = bp->ctxtRegEntry[i].value; uint32_t addrInfo = bp->ctxtRegEntry[i].addressInfo; uint32_t reg = (addrInfo & IX_NPEDL_MASK_STATE_ADDR_CTXT_REG); uint32_t cNum = (addrInfo & IX_NPEDL_MASK_STATE_ADDR_CTXT_NUM) >> IX_NPEDL_OFFSET_STATE_ADDR_CTXT_NUM; /* error-check Context Register No. and Context Number values */ if (!(0 <= reg && reg < IX_NPEDL_CTXT_REG_MAX)) { device_printf(sc->sc_dev, "invalid Context Register %u\n", reg); error = EINVAL; break; } if (!(0 <= cNum && cNum < IX_NPEDL_CTXT_NUM_MAX)) { device_printf(sc->sc_dev, "invalid Context Number %u\n", cNum); error = EINVAL; break; } /* NOTE that there is no STEVT register for Context 0 */ if (cNum == 0 && reg == IX_NPEDL_CTXT_REG_STEVT) { device_printf(sc->sc_dev, "no STEVT for Context 0\n"); error = EINVAL; break; } if (npe_ctx_reg_write(sc, cNum, reg, regVal, verify) != 0) { device_printf(sc->sc_dev, "write of state-info to NPE failed\n"); error = EIO; break; } } npe_cpu_step_restore(sc); return error; } static int npe_load_image(struct ixpnpe_softc *sc, const uint32_t *imageCodePtr, int verify) { #define EOM(marker) ((marker) == IX_NPEDL_END_OF_DOWNLOAD_MAP) const IxNpeDlNpeMgrDownloadMap *downloadMap; int i, error; if (!npe_isstopped(sc)) { /* verify NPE is stopped */ device_printf(sc->sc_dev, "cannot load image, NPE not stopped\n"); return EIO; } /* * Read Download Map, checking each block type and calling * appropriate function to perform download */ error = 0; downloadMap = (const IxNpeDlNpeMgrDownloadMap *) imageCodePtr; for (i = 0; !EOM(downloadMap->entry[i].eodmMarker); i++) { /* calculate pointer to block to be downloaded */ const uint32_t *bp = imageCodePtr + downloadMap->entry[i].block.offset; switch (downloadMap->entry[i].block.type) { case IX_NPEDL_BLOCK_TYPE_INSTRUCTION: error = npe_load_ins(sc, (const IxNpeDlNpeMgrCodeBlock *) bp, verify); DPRINTF(sc->sc_dev, "%s: inst, error %d\n", __func__, error); break; case IX_NPEDL_BLOCK_TYPE_DATA: error = npe_load_data(sc, (const IxNpeDlNpeMgrCodeBlock *) bp, verify); DPRINTF(sc->sc_dev, "%s: data, error %d\n", __func__, error); break; case IX_NPEDL_BLOCK_TYPE_STATE: error = npe_load_stateinfo(sc, (const IxNpeDlNpeMgrStateInfoBlock *) bp, verify); DPRINTF(sc->sc_dev, "%s: state, error %d\n", __func__, error); break; default: device_printf(sc->sc_dev, "unknown block type 0x%x in download map\n", downloadMap->entry[i].block.type); error = EIO; /* XXX */ break; } if (error != 0) break; } return error; #undef EOM } /* contains Reset values for Context Store Registers */ static const struct { uint32_t regAddr; uint32_t regResetVal; } ixNpeDlEcsRegResetValues[] = { { IX_NPEDL_ECS_BG_CTXT_REG_0, IX_NPEDL_ECS_BG_CTXT_REG_0_RESET }, { IX_NPEDL_ECS_BG_CTXT_REG_1, IX_NPEDL_ECS_BG_CTXT_REG_1_RESET }, { IX_NPEDL_ECS_BG_CTXT_REG_2, IX_NPEDL_ECS_BG_CTXT_REG_2_RESET }, { IX_NPEDL_ECS_PRI_1_CTXT_REG_0, IX_NPEDL_ECS_PRI_1_CTXT_REG_0_RESET }, { IX_NPEDL_ECS_PRI_1_CTXT_REG_1, IX_NPEDL_ECS_PRI_1_CTXT_REG_1_RESET }, { IX_NPEDL_ECS_PRI_1_CTXT_REG_2, IX_NPEDL_ECS_PRI_1_CTXT_REG_2_RESET }, { IX_NPEDL_ECS_PRI_2_CTXT_REG_0, IX_NPEDL_ECS_PRI_2_CTXT_REG_0_RESET }, { IX_NPEDL_ECS_PRI_2_CTXT_REG_1, IX_NPEDL_ECS_PRI_2_CTXT_REG_1_RESET }, { IX_NPEDL_ECS_PRI_2_CTXT_REG_2, IX_NPEDL_ECS_PRI_2_CTXT_REG_2_RESET }, { IX_NPEDL_ECS_DBG_CTXT_REG_0, IX_NPEDL_ECS_DBG_CTXT_REG_0_RESET }, { IX_NPEDL_ECS_DBG_CTXT_REG_1, IX_NPEDL_ECS_DBG_CTXT_REG_1_RESET }, { IX_NPEDL_ECS_DBG_CTXT_REG_2, IX_NPEDL_ECS_DBG_CTXT_REG_2_RESET }, { IX_NPEDL_ECS_INSTRUCT_REG, IX_NPEDL_ECS_INSTRUCT_REG_RESET } }; /* contains Reset values for Context Store Registers */ static const uint32_t ixNpeDlCtxtRegResetValues[] = { IX_NPEDL_CTXT_REG_RESET_STEVT, IX_NPEDL_CTXT_REG_RESET_STARTPC, IX_NPEDL_CTXT_REG_RESET_REGMAP, IX_NPEDL_CTXT_REG_RESET_CINDEX, }; #define IX_NPEDL_PARITY_BIT_MASK 0x3F00FFFF #define IX_NPEDL_CONFIG_CTRL_REG_MASK 0x3F3FFFFF #if 0 /* * Reset the NPE and its coprocessor using the * fuse bits in the feature control register. */ static void npe_reset(int npeid) { uint32_t mask = EXP_FCTRL_NPEA << npeid; uint32_t v; v = ixp4xx_read_feature_bits(); ixp4xx_write_feature_bits(v &~ mask); /* un-fuse and un-reset the NPE & coprocessor */ ixp4xx_write_feature_bits(v | mask); } #endif static int npe_cpu_reset(struct ixpnpe_softc *sc) { #define N(a) (sizeof(a) / sizeof(a[0])) uint32_t ctxtReg; /* identifies Context Store reg (0-3) */ uint32_t regAddr; uint32_t regVal; uint32_t ixNpeConfigCtrlRegVal; int i, error = 0; /* pre-store the NPE Config Control Register Value */ ixNpeConfigCtrlRegVal = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_CTL); ixNpeConfigCtrlRegVal |= 0x3F000000; /* disable the parity interrupt */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_CTL, (ixNpeConfigCtrlRegVal & IX_NPEDL_PARITY_BIT_MASK)); DPRINTFn(2, sc->sc_dev, "%s: dis parity int, CTL => 0x%x\n", __func__, ixNpeConfigCtrlRegVal & IX_NPEDL_PARITY_BIT_MASK); npe_cpu_step_save(sc); /* * Clear the FIFOs. */ while (npe_checkbits(sc, IX_NPEDL_REG_OFFSET_WFIFO, IX_NPEDL_MASK_WFIFO_VALID)) { /* read from the Watch-point FIFO until empty */ (void) npe_reg_read(sc, IX_NPEDL_REG_OFFSET_WFIFO); } while (npe_checkbits(sc, IX_NPEDL_REG_OFFSET_STAT, IX_NPEDL_MASK_STAT_OFNE)) { /* read from the outFIFO until empty */ (void) npe_reg_read(sc, IX_NPEDL_REG_OFFSET_FIFO); } while (npe_checkbits(sc, IX_NPEDL_REG_OFFSET_STAT, IX_NPEDL_MASK_STAT_IFNE)) { /* - * Step execution of the NPE intruction to read inFIFO using + * Step execution of the NPE instruction to read inFIFO using * the Debug Executing Context stack. */ error = npe_cpu_step(sc, IX_NPEDL_INSTR_RD_FIFO, 0, 0); if (error != 0) { DPRINTF(sc->sc_dev, "%s: cannot step (1), error %u\n", __func__, error); npe_cpu_step_restore(sc); return error; } } /* * Reset the mailbox reg */ /* ...from XScale side */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_MBST, IX_NPEDL_REG_RESET_MBST); /* ...from NPE side */ error = npe_cpu_step(sc, IX_NPEDL_INSTR_RESET_MBOX, 0, 0); if (error != 0) { DPRINTF(sc->sc_dev, "%s: cannot step (2), error %u\n", __func__, error); npe_cpu_step_restore(sc); return error; } /* * Reset the physical registers in the NPE register file: * Note: no need to save/restore REGMAP for Context 0 here * since all Context Store regs are reset in subsequent code. */ for (regAddr = 0; regAddr < IX_NPEDL_TOTAL_NUM_PHYS_REG && error == 0; regAddr++) { /* for each physical register in the NPE reg file, write 0 : */ error = npe_physical_reg_write(sc, regAddr, 0, TRUE); if (error != 0) { DPRINTF(sc->sc_dev, "%s: cannot write phy reg," "error %u\n", __func__, error); npe_cpu_step_restore(sc); return error; /* abort reset */ } } /* * Reset the context store: */ for (i = IX_NPEDL_CTXT_NUM_MIN; i <= IX_NPEDL_CTXT_NUM_MAX; i++) { /* set each context's Context Store registers to reset values */ for (ctxtReg = 0; ctxtReg < IX_NPEDL_CTXT_REG_MAX; ctxtReg++) { /* NOTE that there is no STEVT register for Context 0 */ if (i == 0 && ctxtReg == IX_NPEDL_CTXT_REG_STEVT) continue; regVal = ixNpeDlCtxtRegResetValues[ctxtReg]; error = npe_ctx_reg_write(sc, i, ctxtReg, regVal, TRUE); if (error != 0) { DPRINTF(sc->sc_dev, "%s: cannot write ctx reg," "error %u\n", __func__, error); npe_cpu_step_restore(sc); return error; /* abort reset */ } } } npe_cpu_step_restore(sc); /* write Reset values to Execution Context Stack registers */ for (i = 0; i < N(ixNpeDlEcsRegResetValues); i++) npe_ecs_reg_write(sc, ixNpeDlEcsRegResetValues[i].regAddr, ixNpeDlEcsRegResetValues[i].regResetVal); /* clear the profile counter */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_CLR_PROFILE_CNT); /* clear registers EXCT, AP0, AP1, AP2 and AP3 */ for (regAddr = IX_NPEDL_REG_OFFSET_EXCT; regAddr <= IX_NPEDL_REG_OFFSET_AP3; regAddr += sizeof(uint32_t)) npe_reg_write(sc, regAddr, 0); /* Reset the Watch-count register */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_WC, 0); #if 0 /* * WR IXA00055043 - Remove IMEM Parity Introduced by NPE Reset Operation * XXX Removed because it breaks IXP435 operation; e.g. on Gateworks * XXX 2358 boards reseting NPE-A after NPE-C is running causes both * XXX npe's to stop working */ npe_reset(sc->sc_npeid); #endif /* * Call NpeMgr function to stop the NPE again after the Feature Control * has unfused and Un-Reset the NPE and its associated Coprocessors. */ error = npe_cpu_stop(sc); /* restore NPE configuration bus Control Register - Parity Settings */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_CTL, (ixNpeConfigCtrlRegVal & IX_NPEDL_CONFIG_CTRL_REG_MASK)); DPRINTFn(2, sc->sc_dev, "%s: restore CTL => 0x%x\n", __func__, npe_reg_read(sc, IX_NPEDL_REG_OFFSET_CTL)); return error; #undef N } static int npe_cpu_start(struct ixpnpe_softc *sc) { uint32_t ecsRegVal; /* * Ensure only Background Context Stack Level is Active by turning off * the Active bit in each of the other Executing Context Stack levels. */ ecsRegVal = npe_ecs_reg_read(sc, IX_NPEDL_ECS_PRI_1_CTXT_REG_0); ecsRegVal &= ~IX_NPEDL_MASK_ECS_REG_0_ACTIVE; npe_ecs_reg_write(sc, IX_NPEDL_ECS_PRI_1_CTXT_REG_0, ecsRegVal); ecsRegVal = npe_ecs_reg_read(sc, IX_NPEDL_ECS_PRI_2_CTXT_REG_0); ecsRegVal &= ~IX_NPEDL_MASK_ECS_REG_0_ACTIVE; npe_ecs_reg_write(sc, IX_NPEDL_ECS_PRI_2_CTXT_REG_0, ecsRegVal); ecsRegVal = npe_ecs_reg_read(sc, IX_NPEDL_ECS_DBG_CTXT_REG_0); ecsRegVal &= ~IX_NPEDL_MASK_ECS_REG_0_ACTIVE; npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_0, ecsRegVal); /* clear the pipeline */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_CLR_PIPE); /* start NPE execution by issuing cmd through EXCTL register on NPE */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_START); /* * Check execution status of NPE to verify operation was successful. */ return npe_checkbits(sc, IX_NPEDL_REG_OFFSET_EXCTL, IX_NPEDL_EXCTL_STATUS_RUN) ? 0 : EIO; } static int npe_cpu_stop(struct ixpnpe_softc *sc) { /* stop NPE execution by issuing cmd through EXCTL register on NPE */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_STOP); /* verify that NPE Stop was successful */ return npe_checkbits(sc, IX_NPEDL_REG_OFFSET_EXCTL, IX_NPEDL_EXCTL_STATUS_STOP) ? 0 : EIO; } #define IX_NPEDL_REG_SIZE_BYTE 8 #define IX_NPEDL_REG_SIZE_SHORT 16 #define IX_NPEDL_REG_SIZE_WORD 32 /* * Introduce extra read cycles after issuing read command to NPE * so that we read the register after the NPE has updated it * This is to overcome race condition between XScale and NPE */ #define IX_NPEDL_DELAY_READ_CYCLES 2 /* * To mask top three MSBs of 32bit word to download into NPE IMEM */ #define IX_NPEDL_MASK_UNUSED_IMEM_BITS 0x1FFFFFFF; static void npe_cmd_issue_write(struct ixpnpe_softc *sc, uint32_t cmd, uint32_t addr, uint32_t data) { npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXDATA, data); npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXAD, addr); npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCTL, cmd); } static uint32_t npe_cmd_issue_read(struct ixpnpe_softc *sc, uint32_t cmd, uint32_t addr) { uint32_t data; int i; npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXAD, addr); npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCTL, cmd); for (i = 0; i <= IX_NPEDL_DELAY_READ_CYCLES; i++) data = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_EXDATA); return data; } static int npe_ins_write(struct ixpnpe_softc *sc, uint32_t addr, uint32_t data, int verify) { DPRINTFn(4, sc->sc_dev, "%s(0x%x, 0x%x)\n", __func__, addr, data); npe_cmd_issue_write(sc, IX_NPEDL_EXCTL_CMD_WR_INS_MEM, addr, data); if (verify) { uint32_t rdata; /* * Write invalid data to this reg, so we can see if we're * reading the EXDATA register too early. */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXDATA, ~data); /* * Disabled since top 3 MSB are not used for Azusa * hardware Refer WR:IXA00053900 */ data &= IX_NPEDL_MASK_UNUSED_IMEM_BITS; rdata = npe_cmd_issue_read(sc, IX_NPEDL_EXCTL_CMD_RD_INS_MEM, addr); rdata &= IX_NPEDL_MASK_UNUSED_IMEM_BITS; if (data != rdata) return EIO; } return 0; } static int npe_data_write(struct ixpnpe_softc *sc, uint32_t addr, uint32_t data, int verify) { DPRINTFn(4, sc->sc_dev, "%s(0x%x, 0x%x)\n", __func__, addr, data); npe_cmd_issue_write(sc, IX_NPEDL_EXCTL_CMD_WR_DATA_MEM, addr, data); if (verify) { /* * Write invalid data to this reg, so we can see if we're * reading the EXDATA register too early. */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXDATA, ~data); if (data != npe_cmd_issue_read(sc, IX_NPEDL_EXCTL_CMD_RD_DATA_MEM, addr)) return EIO; } return 0; } static void npe_ecs_reg_write(struct ixpnpe_softc *sc, uint32_t reg, uint32_t data) { npe_cmd_issue_write(sc, IX_NPEDL_EXCTL_CMD_WR_ECS_REG, reg, data); } static uint32_t npe_ecs_reg_read(struct ixpnpe_softc *sc, uint32_t reg) { return npe_cmd_issue_read(sc, IX_NPEDL_EXCTL_CMD_RD_ECS_REG, reg); } static void npe_issue_cmd(struct ixpnpe_softc *sc, uint32_t command) { npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCTL, command); } static void npe_cpu_step_save(struct ixpnpe_softc *sc) { /* turn off the halt bit by clearing Execution Count register. */ /* save reg contents 1st and restore later */ sc->savedExecCount = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_EXCT); npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCT, 0); /* ensure that IF and IE are on (temporarily), so that we don't end up * stepping forever */ sc->savedEcsDbgCtxtReg2 = npe_ecs_reg_read(sc, IX_NPEDL_ECS_DBG_CTXT_REG_2); npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_2, (sc->savedEcsDbgCtxtReg2 | IX_NPEDL_MASK_ECS_DBG_REG_2_IF | IX_NPEDL_MASK_ECS_DBG_REG_2_IE)); } static int npe_cpu_step(struct ixpnpe_softc *sc, uint32_t npeInstruction, uint32_t ctxtNum, uint32_t ldur) { #define IX_NPE_DL_MAX_NUM_OF_RETRIES 1000000 uint32_t ecsDbgRegVal; uint32_t oldWatchcount, newWatchcount; int tries; /* set the Active bit, and the LDUR, in the debug level */ ecsDbgRegVal = IX_NPEDL_MASK_ECS_REG_0_ACTIVE | (ldur << IX_NPEDL_OFFSET_ECS_REG_0_LDUR); npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_0, ecsDbgRegVal); /* * Set CCTXT at ECS DEBUG L3 to specify in which context to execute the * instruction, and set SELCTXT at ECS DEBUG Level to specify which * context store to access. * Debug ECS Level Reg 1 has form 0x000n000n, where n = context number */ ecsDbgRegVal = (ctxtNum << IX_NPEDL_OFFSET_ECS_REG_1_CCTXT) | (ctxtNum << IX_NPEDL_OFFSET_ECS_REG_1_SELCTXT); npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_1, ecsDbgRegVal); /* clear the pipeline */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_CLR_PIPE); /* load NPE instruction into the instruction register */ npe_ecs_reg_write(sc, IX_NPEDL_ECS_INSTRUCT_REG, npeInstruction); /* need this value later to wait for completion of NPE execution step */ oldWatchcount = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_WC); /* issue a Step One command via the Execution Control register */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_STEP); /* * Force the XScale to wait until the NPE has finished execution step * NOTE that this delay will be very small, just long enough to allow a * single NPE instruction to complete execution; if instruction * execution is not completed before timeout retries, exit the while * loop. */ newWatchcount = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_WC); for (tries = 0; tries < IX_NPE_DL_MAX_NUM_OF_RETRIES && newWatchcount == oldWatchcount; tries++) { /* Watch Count register incr's when NPE completes an inst */ newWatchcount = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_WC); } return (tries < IX_NPE_DL_MAX_NUM_OF_RETRIES) ? 0 : EIO; #undef IX_NPE_DL_MAX_NUM_OF_RETRIES } static void npe_cpu_step_restore(struct ixpnpe_softc *sc) { /* clear active bit in debug level */ npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_0, 0); /* clear the pipeline */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_CLR_PIPE); /* restore Execution Count register contents. */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCT, sc->savedExecCount); /* restore IF and IE bits to original values */ npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_2, sc->savedEcsDbgCtxtReg2); } static int npe_logical_reg_read(struct ixpnpe_softc *sc, uint32_t regAddr, uint32_t regSize, uint32_t ctxtNum, uint32_t *regVal) { uint32_t npeInstruction, mask; int error; switch (regSize) { case IX_NPEDL_REG_SIZE_BYTE: npeInstruction = IX_NPEDL_INSTR_RD_REG_BYTE; mask = 0xff; break; case IX_NPEDL_REG_SIZE_SHORT: npeInstruction = IX_NPEDL_INSTR_RD_REG_SHORT; mask = 0xffff; break; case IX_NPEDL_REG_SIZE_WORD: npeInstruction = IX_NPEDL_INSTR_RD_REG_WORD; mask = 0xffffffff; break; default: return EINVAL; } /* make regAddr be the SRC and DEST operands (e.g. movX d0, d0) */ npeInstruction |= (regAddr << IX_NPEDL_OFFSET_INSTR_SRC) | (regAddr << IX_NPEDL_OFFSET_INSTR_DEST); /* step execution of NPE inst using Debug Executing Context stack */ error = npe_cpu_step(sc, npeInstruction, ctxtNum, IX_NPEDL_RD_INSTR_LDUR); if (error != 0) { DPRINTF(sc->sc_dev, "%s(0x%x, %u, %u), cannot step, error %d\n", __func__, regAddr, regSize, ctxtNum, error); return error; } /* read value of register from Execution Data register */ *regVal = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_EXDATA); /* align value from left to right */ *regVal = (*regVal >> (IX_NPEDL_REG_SIZE_WORD - regSize)) & mask; return 0; } static int npe_logical_reg_write(struct ixpnpe_softc *sc, uint32_t regAddr, uint32_t regVal, uint32_t regSize, uint32_t ctxtNum, int verify) { int error; DPRINTFn(4, sc->sc_dev, "%s(0x%x, 0x%x, %u, %u)\n", __func__, regAddr, regVal, regSize, ctxtNum); if (regSize == IX_NPEDL_REG_SIZE_WORD) { /* * NPE register addressing is left-to-right: e.g. |d0|d1|d2|d3| * Write upper half-word (short) to |d0|d1| */ error = npe_logical_reg_write(sc, regAddr, regVal >> IX_NPEDL_REG_SIZE_SHORT, IX_NPEDL_REG_SIZE_SHORT, ctxtNum, verify); if (error != 0) return error; /* Write lower half-word (short) to |d2|d3| */ error = npe_logical_reg_write(sc, regAddr + sizeof(uint16_t), regVal & 0xffff, IX_NPEDL_REG_SIZE_SHORT, ctxtNum, verify); } else { uint32_t npeInstruction; switch (regSize) { case IX_NPEDL_REG_SIZE_BYTE: npeInstruction = IX_NPEDL_INSTR_WR_REG_BYTE; regVal &= 0xff; break; case IX_NPEDL_REG_SIZE_SHORT: npeInstruction = IX_NPEDL_INSTR_WR_REG_SHORT; regVal &= 0xffff; break; default: return EINVAL; } /* fill dest operand field of inst with dest reg addr */ npeInstruction |= (regAddr << IX_NPEDL_OFFSET_INSTR_DEST); /* fill src operand field of inst with least-sig 5 bits of val*/ npeInstruction |= ((regVal & IX_NPEDL_MASK_IMMED_INSTR_SRC_DATA) << IX_NPEDL_OFFSET_INSTR_SRC); /* fill coprocessor field of inst with most-sig 11 bits of val*/ npeInstruction |= ((regVal & IX_NPEDL_MASK_IMMED_INSTR_COPROC_DATA) << IX_NPEDL_DISPLACE_IMMED_INSTR_COPROC_DATA); - /* step execution of NPE intruction using Debug ECS */ + /* step execution of NPE instruction using Debug ECS */ error = npe_cpu_step(sc, npeInstruction, ctxtNum, IX_NPEDL_WR_INSTR_LDUR); } if (error != 0) { DPRINTF(sc->sc_dev, "%s(0x%x, 0x%x, %u, %u), error %u " "writing reg\n", __func__, regAddr, regVal, regSize, ctxtNum, error); return error; } if (verify) { uint32_t retRegVal; error = npe_logical_reg_read(sc, regAddr, regSize, ctxtNum, &retRegVal); if (error == 0 && regVal != retRegVal) error = EIO; /* XXX ambiguous */ } return error; } /* * There are 32 physical registers used in an NPE. These are * treated as 16 pairs of 32-bit registers. To write one of the pair, * write the pair number (0-16) to the REGMAP for Context 0. Then write * the value to register 0 or 4 in the regfile, depending on which * register of the pair is to be written */ static int npe_physical_reg_write(struct ixpnpe_softc *sc, uint32_t regAddr, uint32_t regValue, int verify) { int error; /* * Set REGMAP for context 0 to (regAddr >> 1) to choose which pair * (0-16) of physical registers to write . */ error = npe_logical_reg_write(sc, IX_NPEDL_CTXT_REG_ADDR_REGMAP, (regAddr >> IX_NPEDL_OFFSET_PHYS_REG_ADDR_REGMAP), IX_NPEDL_REG_SIZE_SHORT, 0, verify); if (error == 0) { /* regAddr = 0 or 4 */ regAddr = (regAddr & IX_NPEDL_MASK_PHYS_REG_ADDR_LOGICAL_ADDR) * sizeof(uint32_t); error = npe_logical_reg_write(sc, regAddr, regValue, IX_NPEDL_REG_SIZE_WORD, 0, verify); } return error; } static int npe_ctx_reg_write(struct ixpnpe_softc *sc, uint32_t ctxtNum, uint32_t ctxtReg, uint32_t ctxtRegVal, int verify) { DPRINTFn(4, sc->sc_dev, "%s(%u, %u, %u)\n", __func__, ctxtNum, ctxtReg, ctxtRegVal); /* * Context 0 has no STARTPC. Instead, this value is used to set * NextPC for Background ECS, to set where NPE starts executing code */ if (ctxtNum == 0 && ctxtReg == IX_NPEDL_CTXT_REG_STARTPC) { /* read BG_CTXT_REG_0, update NEXTPC bits, & write back to reg*/ uint32_t v = npe_ecs_reg_read(sc, IX_NPEDL_ECS_BG_CTXT_REG_0); v &= ~IX_NPEDL_MASK_ECS_REG_0_NEXTPC; v |= (ctxtRegVal << IX_NPEDL_OFFSET_ECS_REG_0_NEXTPC) & IX_NPEDL_MASK_ECS_REG_0_NEXTPC; npe_ecs_reg_write(sc, IX_NPEDL_ECS_BG_CTXT_REG_0, v); return 0; } else { static const struct { uint32_t regAddress; uint32_t regSize; } regAccInfo[IX_NPEDL_CTXT_REG_MAX] = { { IX_NPEDL_CTXT_REG_ADDR_STEVT, IX_NPEDL_REG_SIZE_BYTE }, { IX_NPEDL_CTXT_REG_ADDR_STARTPC, IX_NPEDL_REG_SIZE_SHORT }, { IX_NPEDL_CTXT_REG_ADDR_REGMAP, IX_NPEDL_REG_SIZE_SHORT }, { IX_NPEDL_CTXT_REG_ADDR_CINDEX, IX_NPEDL_REG_SIZE_BYTE } }; return npe_logical_reg_write(sc, regAccInfo[ctxtReg].regAddress, ctxtRegVal, regAccInfo[ctxtReg].regSize, ctxtNum, verify); } } /* * NPE Mailbox support. */ #define IX_NPEMH_MAXTRIES 100000 static int ofifo_wait(struct ixpnpe_softc *sc) { int i; for (i = 0; i < IX_NPEMH_MAXTRIES; i++) { if (npe_reg_read(sc, IX_NPESTAT) & IX_NPESTAT_OFNE) return 1; DELAY(10); } device_printf(sc->sc_dev, "%s: timeout, last status 0x%x\n", __func__, npe_reg_read(sc, IX_NPESTAT)); return 0; } static int getmsg(struct ixpnpe_softc *sc, uint32_t msg[2]) { mtx_assert(&sc->sc_mtx, MA_OWNED); if (!ofifo_wait(sc)) return EAGAIN; msg[0] = npe_reg_read(sc, IX_NPEFIFO); DPRINTF(sc->sc_dev, "%s: msg0 0x%x\n", __func__, msg[0]); if (!ofifo_wait(sc)) return EAGAIN; msg[1] = npe_reg_read(sc, IX_NPEFIFO); DPRINTF(sc->sc_dev, "%s: msg1 0x%x\n", __func__, msg[1]); return 0; } static void ixpnpe_intr(void *arg) { struct ixpnpe_softc *sc = arg; uint32_t status; mtx_lock(&sc->sc_mtx); status = npe_reg_read(sc, IX_NPESTAT); DPRINTF(sc->sc_dev, "%s: status 0x%x\n", __func__, status); if ((status & IX_NPESTAT_OFINT) == 0) { /* NB: should not happen */ device_printf(sc->sc_dev, "%s: status 0x%x\n", __func__, status); /* XXX must silence interrupt? */ mtx_unlock(&sc->sc_mtx); return; } /* * A message is waiting in the output FIFO, copy it so * the interrupt will be silenced. */ if (getmsg(sc, sc->sc_msg) == 0) sc->sc_msgwaiting = 1; mtx_unlock(&sc->sc_mtx); } static int ififo_wait(struct ixpnpe_softc *sc) { int i; for (i = 0; i < IX_NPEMH_MAXTRIES; i++) { if (npe_reg_read(sc, IX_NPESTAT) & IX_NPESTAT_IFNF) return 1; DELAY(10); } device_printf(sc->sc_dev, "%s: timeout, last status 0x%x\n", __func__, npe_reg_read(sc, IX_NPESTAT)); return 0; } static int putmsg(struct ixpnpe_softc *sc, const uint32_t msg[2]) { mtx_assert(&sc->sc_mtx, MA_OWNED); DPRINTF(sc->sc_dev, "%s: msg 0x%x:0x%x\n", __func__, msg[0], msg[1]); if (!ififo_wait(sc)) return EIO; npe_reg_write(sc, IX_NPEFIFO, msg[0]); if (!ififo_wait(sc)) return EIO; npe_reg_write(sc, IX_NPEFIFO, msg[1]); return 0; } /* * Send a msg to the NPE and wait for a reply. We spin as * we may be called early with interrupts not properly setup. */ int ixpnpe_sendandrecvmsg_sync(struct ixpnpe_softc *sc, const uint32_t send[2], uint32_t recv[2]) { int error; mtx_lock(&sc->sc_mtx); error = putmsg(sc, send); if (error == 0) error = getmsg(sc, recv); mtx_unlock(&sc->sc_mtx); return error; } /* * Send a msg to the NPE w/o waiting for a reply. */ int ixpnpe_sendmsg_async(struct ixpnpe_softc *sc, const uint32_t msg[2]) { int error; mtx_lock(&sc->sc_mtx); error = putmsg(sc, msg); mtx_unlock(&sc->sc_mtx); return error; } static int recvmsg_locked(struct ixpnpe_softc *sc, uint32_t msg[2]) { mtx_assert(&sc->sc_mtx, MA_OWNED); DPRINTF(sc->sc_dev, "%s: msgwaiting %d\n", __func__, sc->sc_msgwaiting); if (sc->sc_msgwaiting) { msg[0] = sc->sc_msg[0]; msg[1] = sc->sc_msg[1]; sc->sc_msgwaiting = 0; return 0; } return EAGAIN; } /* * Receive any msg previously received from the NPE. If nothing * is available we return EAGAIN and the caller is required to * do a synchronous receive or try again later. */ int ixpnpe_recvmsg_async(struct ixpnpe_softc *sc, uint32_t msg[2]) { int error; mtx_lock(&sc->sc_mtx); error = recvmsg_locked(sc, msg); mtx_unlock(&sc->sc_mtx); return error; } /* * Receive a msg from the NPE. If one was received asynchronously * then it's returned; otherwise we poll synchronously. */ int ixpnpe_recvmsg_sync(struct ixpnpe_softc *sc, uint32_t msg[2]) { int error; mtx_lock(&sc->sc_mtx); error = recvmsg_locked(sc, msg); if (error == EAGAIN) error = getmsg(sc, msg); mtx_unlock(&sc->sc_mtx); return error; } Index: head/sys/arm/xscale/ixp425/ixp425_npereg.h =================================================================== --- head/sys/arm/xscale/ixp425/ixp425_npereg.h (revision 299068) +++ head/sys/arm/xscale/ixp425/ixp425_npereg.h (revision 299069) @@ -1,427 +1,427 @@ /*- * Copyright (c) 2006 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD$ */ /*- * Copyright (c) 2001-2005, Intel Corporation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the Intel Corporation nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _IXP425_NPEREG_H_ #define _IXP425_NPEREG_H_ /* signature found as 1st word in a microcode image library */ #define IX_NPEDL_IMAGEMGR_SIGNATURE 0xDEADBEEF /* marks end of header in a microcode image library */ #define IX_NPEDL_IMAGEMGR_END_OF_HEADER 0xFFFFFFFF /* * Intel (R) IXP400 Software NPE Image ID Definition * * Definition of NPE Image ID to be passed to ixNpeDlNpeInitAndStart() * as input of type uint32_t which has the following fields format: * * Field [Bit Location] * ----------------------------------- * Device ID [31 - 28] * NPE ID [27 - 24] * NPE Functionality ID [23 - 16] * Major Release Number [15 - 8] * Minor Release Number [7 - 0] */ #define IX_NPEDL_NPEID_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 24) & 0xf) #define IX_NPEDL_DEVICEID_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 28) & 0xf) #define IX_NPEDL_FUNCTIONID_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 16) & 0xff) #define IX_NPEDL_MAJOR_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 8) & 0xff) #define IX_NPEDL_MINOR_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 0) & 0xff) /* * Instruction and Data Memory Size (in words) for each NPE */ #define IX_NPEDL_INS_MEMSIZE_WORDS_NPEA 4096 #define IX_NPEDL_INS_MEMSIZE_WORDS_NPEB 2048 #define IX_NPEDL_INS_MEMSIZE_WORDS_NPEC 2048 #define IX_NPEDL_DATA_MEMSIZE_WORDS_NPEA 2048 #define IX_NPEDL_DATA_MEMSIZE_WORDS_NPEB 2048 #define IX_NPEDL_DATA_MEMSIZE_WORDS_NPEC 2048 #define IXP46X_NPEDL_INS_MEMSIZE_WORDS 4096 #define IXP46X_NPEDL_DATA_MEMSIZE_WORDS 4096 /* BAR offsets */ #define IX_NPEDL_REG_OFFSET_EXAD 0x00000000 /* Execution Address */ #define IX_NPEDL_REG_OFFSET_EXDATA 0x00000004 /* Execution Data */ #define IX_NPEDL_REG_OFFSET_EXCTL 0x00000008 /* Execution Control */ #define IX_NPEDL_REG_OFFSET_EXCT 0x0000000C /* Execution Count */ #define IX_NPEDL_REG_OFFSET_AP0 0x00000010 /* Action Point 0 */ #define IX_NPEDL_REG_OFFSET_AP1 0x00000014 /* Action Point 1 */ #define IX_NPEDL_REG_OFFSET_AP2 0x00000018 /* Action Point 2 */ #define IX_NPEDL_REG_OFFSET_AP3 0x0000001C /* Action Point 3 */ #define IX_NPEDL_REG_OFFSET_WFIFO 0x00000020 /* Watchpoint FIFO */ #define IX_NPEDL_REG_OFFSET_WC 0x00000024 /* Watch Count */ #define IX_NPEDL_REG_OFFSET_PROFCT 0x00000028 /* Profile Count */ #define IX_NPEDL_REG_OFFSET_STAT 0x0000002C /* Messaging Status */ #define IX_NPEDL_REG_OFFSET_CTL 0x00000030 /* Messaging Control */ #define IX_NPEDL_REG_OFFSET_MBST 0x00000034 /* Mailbox Status */ #define IX_NPEDL_REG_OFFSET_FIFO 0x00000038 /* Message FIFO */ /* * Reset value for Mailbox (MBST) register - * NOTE that if used, it should be complemented with an NPE intruction + * NOTE that if used, it should be complemented with an NPE instruction * to clear the Mailbox at the NPE side as well */ #define IX_NPEDL_REG_RESET_MBST 0x0000F0F0 #define IX_NPEDL_MASK_WFIFO_VALID 0x80000000 /* VALID bit */ #define IX_NPEDL_MASK_STAT_OFNE 0x00010000 /* OFNE bit */ #define IX_NPEDL_MASK_STAT_IFNE 0x00080000 /* IFNE bit */ /* * EXCTL (Execution Control) Register commands */ #define IX_NPEDL_EXCTL_CMD_NPE_STEP 0x01 /* Step 1 instruction */ #define IX_NPEDL_EXCTL_CMD_NPE_START 0x02 /* Start execution */ #define IX_NPEDL_EXCTL_CMD_NPE_STOP 0x03 /* Stop execution */ #define IX_NPEDL_EXCTL_CMD_NPE_CLR_PIPE 0x04 /* Clear ins pipeline */ /* * Read/write operations use address in EXAD and data in EXDATA. */ #define IX_NPEDL_EXCTL_CMD_RD_INS_MEM 0x10 /* Read ins memory */ #define IX_NPEDL_EXCTL_CMD_WR_INS_MEM 0x11 /* Write ins memory */ #define IX_NPEDL_EXCTL_CMD_RD_DATA_MEM 0x12 /* Read data memory */ #define IX_NPEDL_EXCTL_CMD_WR_DATA_MEM 0x13 /* Write data memory */ #define IX_NPEDL_EXCTL_CMD_RD_ECS_REG 0x14 /* Read ECS register */ #define IX_NPEDL_EXCTL_CMD_WR_ECS_REG 0x15 /* Write ECS register */ #define IX_NPEDL_EXCTL_CMD_CLR_PROFILE_CNT 0x0C /* Clear Profile Count register */ /* * EXCTL (Execution Control) Register status bit masks */ #define IX_NPEDL_EXCTL_STATUS_RUN 0x80000000 #define IX_NPEDL_EXCTL_STATUS_STOP 0x40000000 #define IX_NPEDL_EXCTL_STATUS_CLEAR 0x20000000 #define IX_NPEDL_EXCTL_STATUS_ECS_K 0x00800000 /* pipeline Klean */ /* * Executing Context Stack (ECS) level registers */ #define IX_NPEDL_ECS_BG_CTXT_REG_0 0x00 /* reg 0 @ bg ctx */ #define IX_NPEDL_ECS_BG_CTXT_REG_1 0x01 /* reg 1 @ bg ctx */ #define IX_NPEDL_ECS_BG_CTXT_REG_2 0x02 /* reg 2 @ bg ctx */ #define IX_NPEDL_ECS_PRI_1_CTXT_REG_0 0x04 /* reg 0 @ pri 1 ctx */ #define IX_NPEDL_ECS_PRI_1_CTXT_REG_1 0x05 /* reg 1 @ pri 1 ctx */ #define IX_NPEDL_ECS_PRI_1_CTXT_REG_2 0x06 /* reg 2 @ pri 1 ctx */ #define IX_NPEDL_ECS_PRI_2_CTXT_REG_0 0x08 /* reg 0 @ pri 2 ctx */ #define IX_NPEDL_ECS_PRI_2_CTXT_REG_1 0x09 /* reg 1 @ pri 2 ctx */ #define IX_NPEDL_ECS_PRI_2_CTXT_REG_2 0x0A /* reg 2 @ pri 2 ctx */ #define IX_NPEDL_ECS_DBG_CTXT_REG_0 0x0C /* reg 0 @ debug ctx */ #define IX_NPEDL_ECS_DBG_CTXT_REG_1 0x0D /* reg 1 @ debug ctx */ #define IX_NPEDL_ECS_DBG_CTXT_REG_2 0x0E /* reg 2 @ debug ctx */ #define IX_NPEDL_ECS_INSTRUCT_REG 0x11 /* Instruction reg */ /* * Execution Access register reset values */ #define IX_NPEDL_ECS_BG_CTXT_REG_0_RESET 0xA0000000 #define IX_NPEDL_ECS_BG_CTXT_REG_1_RESET 0x01000000 #define IX_NPEDL_ECS_BG_CTXT_REG_2_RESET 0x00008000 #define IX_NPEDL_ECS_PRI_1_CTXT_REG_0_RESET 0x20000080 #define IX_NPEDL_ECS_PRI_1_CTXT_REG_1_RESET 0x01000000 #define IX_NPEDL_ECS_PRI_1_CTXT_REG_2_RESET 0x00008000 #define IX_NPEDL_ECS_PRI_2_CTXT_REG_0_RESET 0x20000080 #define IX_NPEDL_ECS_PRI_2_CTXT_REG_1_RESET 0x01000000 #define IX_NPEDL_ECS_PRI_2_CTXT_REG_2_RESET 0x00008000 #define IX_NPEDL_ECS_DBG_CTXT_REG_0_RESET 0x20000000 #define IX_NPEDL_ECS_DBG_CTXT_REG_1_RESET 0x00000000 #define IX_NPEDL_ECS_DBG_CTXT_REG_2_RESET 0x001E0000 #define IX_NPEDL_ECS_INSTRUCT_REG_RESET 0x1003C00F /* * Masks used to read/write particular bits in Execution Access registers */ #define IX_NPEDL_MASK_ECS_REG_0_ACTIVE 0x80000000 /* Active bit */ #define IX_NPEDL_MASK_ECS_REG_0_NEXTPC 0x1FFF0000 /* NextPC bits */ #define IX_NPEDL_MASK_ECS_REG_0_LDUR 0x00000700 /* LDUR bits */ #define IX_NPEDL_MASK_ECS_REG_1_CCTXT 0x000F0000 /* NextPC bits */ #define IX_NPEDL_MASK_ECS_REG_1_SELCTXT 0x0000000F #define IX_NPEDL_MASK_ECS_DBG_REG_2_IF 0x00100000 /* IF bit */ #define IX_NPEDL_MASK_ECS_DBG_REG_2_IE 0x00080000 /* IE bit */ /* * Bit-Offsets from LSB of particular bit-fields in Execution Access registers. */ #define IX_NPEDL_OFFSET_ECS_REG_0_NEXTPC 16 #define IX_NPEDL_OFFSET_ECS_REG_0_LDUR 8 #define IX_NPEDL_OFFSET_ECS_REG_1_CCTXT 16 #define IX_NPEDL_OFFSET_ECS_REG_1_SELCTXT 0 /* * NPE core & co-processor instruction templates to load into NPE Instruction * Register, for read/write of NPE register file registers. */ /* * Read an 8-bit NPE internal logical register * and return the value in the EXDATA register (aligned to MSB). * NPE Assembler instruction: "mov8 d0, d0 &&& DBG_WrExec" */ #define IX_NPEDL_INSTR_RD_REG_BYTE 0x0FC00000 /* * Read a 16-bit NPE internal logical register * and return the value in the EXDATA register (aligned to MSB). * NPE Assembler instruction: "mov16 d0, d0 &&& DBG_WrExec" */ #define IX_NPEDL_INSTR_RD_REG_SHORT 0x0FC08010 /* * Read a 16-bit NPE internal logical register * and return the value in the EXDATA register. * NPE Assembler instruction: "mov32 d0, d0 &&& DBG_WrExec" */ #define IX_NPEDL_INSTR_RD_REG_WORD 0x0FC08210 /* * Write an 8-bit NPE internal logical register. * NPE Assembler instruction: "mov8 d0, #0" */ #define IX_NPEDL_INSTR_WR_REG_BYTE 0x00004000 /* * Write a 16-bit NPE internal logical register. * NPE Assembler instruction: "mov16 d0, #0" */ #define IX_NPEDL_INSTR_WR_REG_SHORT 0x0000C000 /* * Write a 16-bit NPE internal logical register. * NPE Assembler instruction: "cprd32 d0 &&& DBG_RdInFIFO" */ #define IX_NPEDL_INSTR_RD_FIFO 0x0F888220 /* * Reset Mailbox (MBST) register * NPE Assembler instruction: "mov32 d0, d0 &&& DBG_ClearM" */ #define IX_NPEDL_INSTR_RESET_MBOX 0x0FAC8210 /* * Bit-offsets from LSB, of particular bit-fields in an NPE instruction */ #define IX_NPEDL_OFFSET_INSTR_SRC 4 /* src operand */ #define IX_NPEDL_OFFSET_INSTR_DEST 9 /* dest operand */ #define IX_NPEDL_OFFSET_INSTR_COPROC 18 /* coprocessor ins */ /* * Masks used to read/write particular bits of an NPE Instruction */ /** * Mask the bits of 16-bit data value (least-sig 5 bits) to be used in * SRC field of immediate-mode NPE instruction */ #define IX_NPEDL_MASK_IMMED_INSTR_SRC_DATA 0x1F /** * Mask the bits of 16-bit data value (most-sig 11 bits) to be used in * COPROC field of immediate-mode NPE instruction */ #define IX_NPEDL_MASK_IMMED_INSTR_COPROC_DATA 0xFFE0 /** * LSB offset of the bit-field of 16-bit data value (most-sig 11 bits) * to be used in COPROC field of immediate-mode NPE instruction */ #define IX_NPEDL_OFFSET_IMMED_INSTR_COPROC_DATA 5 /** * Number of left-shifts required to align most-sig 11 bits of 16-bit * data value into COPROC field of immediate-mode NPE instruction */ #define IX_NPEDL_DISPLACE_IMMED_INSTR_COPROC_DATA \ (IX_NPEDL_OFFSET_INSTR_COPROC - IX_NPEDL_OFFSET_IMMED_INSTR_COPROC_DATA) /** * LDUR value used with immediate-mode NPE Instructions by the NpeDl * for writing to NPE internal logical registers */ #define IX_NPEDL_WR_INSTR_LDUR 1 /** * LDUR value used with NON-immediate-mode NPE Instructions by the NpeDl * for reading from NPE internal logical registers */ #define IX_NPEDL_RD_INSTR_LDUR 0 /** * NPE internal Context Store registers. */ typedef enum { IX_NPEDL_CTXT_REG_STEVT = 0, /**< identifies STEVT */ IX_NPEDL_CTXT_REG_STARTPC, /**< identifies STARTPC */ IX_NPEDL_CTXT_REG_REGMAP, /**< identifies REGMAP */ IX_NPEDL_CTXT_REG_CINDEX, /**< identifies CINDEX */ IX_NPEDL_CTXT_REG_MAX /**< Total number of Context Store registers */ } IxNpeDlCtxtRegNum; /* * NPE Context Store register logical addresses */ #define IX_NPEDL_CTXT_REG_ADDR_STEVT 0x0000001B #define IX_NPEDL_CTXT_REG_ADDR_STARTPC 0x0000001C #define IX_NPEDL_CTXT_REG_ADDR_REGMAP 0x0000001E #define IX_NPEDL_CTXT_REG_ADDR_CINDEX 0x0000001F /* * NPE Context Store register reset values */ /** * Reset value of STEVT NPE internal Context Store register * (STEVT = off, 0x80) */ #define IX_NPEDL_CTXT_REG_RESET_STEVT 0x80 /** * Reset value of STARTPC NPE internal Context Store register * (STARTPC = 0x0000) */ #define IX_NPEDL_CTXT_REG_RESET_STARTPC 0x0000 /** * Reset value of REGMAP NPE internal Context Store register * (REGMAP = d0->p0, d8->p2, d16->p4) */ #define IX_NPEDL_CTXT_REG_RESET_REGMAP 0x0820 /** * Reset value of CINDEX NPE internal Context Store register * (CINDEX = 0) */ #define IX_NPEDL_CTXT_REG_RESET_CINDEX 0x00 /* * Numeric range of context levels available on an NPE */ #define IX_NPEDL_CTXT_NUM_MIN 0 #define IX_NPEDL_CTXT_NUM_MAX 15 /** * Number of Physical registers currently supported * Initial NPE implementations will have a 32-word register file. * Later implementations may have a 64-word register file. */ #define IX_NPEDL_TOTAL_NUM_PHYS_REG 32 /** * LSB-offset of Regmap number in Physical NPE register address, used * for Physical To Logical register address mapping in the NPE */ #define IX_NPEDL_OFFSET_PHYS_REG_ADDR_REGMAP 1 /** * Mask to extract a logical NPE register address from a physical * register address, used for Physical To Logical address mapping */ #define IX_NPEDL_MASK_PHYS_REG_ADDR_LOGICAL_ADDR 0x1 /* * NPE Message/Mailbox interface. */ #define IX_NPESTAT IX_NPEDL_REG_OFFSET_STAT /* status register */ #define IX_NPECTL IX_NPEDL_REG_OFFSET_CTL /* control register */ #define IX_NPEFIFO IX_NPEDL_REG_OFFSET_FIFO /* FIFO register */ /* control register */ #define IX_NPECTL_OFE 0x00010000 /* output fifo enable */ #define IX_NPECTL_IFE 0x00020000 /* input fifo enable */ #define IX_NPECTL_OFWE 0x01000000 /* output fifo write enable */ #define IX_NPECTL_IFWE 0x02000000 /* input fifo write enable */ /* status register */ #define IX_NPESTAT_OFNE 0x00010000 /* output fifo not empty */ #define IX_NPESTAT_IFNF 0x00020000 /* input fifo not full */ #define IX_NPESTAT_OFNF 0x00040000 /* output fifo not full */ #define IX_NPESTAT_IFNE 0x00080000 /* input fifo not empty */ #define IX_NPESTAT_MBINT 0x00100000 /* Mailbox interrupt */ #define IX_NPESTAT_IFINT 0x00200000 /* input fifo interrupt */ #define IX_NPESTAT_OFINT 0x00400000 /* output fifo interrupt */ #define IX_NPESTAT_WFINT 0x00800000 /* watch fifo interrupt */ #endif /* _IXP425_NPEREG_H_ */ Index: head/sys/arm/xscale/ixp425/ixp425_qmgr.c =================================================================== --- head/sys/arm/xscale/ixp425/ixp425_qmgr.c (revision 299068) +++ head/sys/arm/xscale/ixp425/ixp425_qmgr.c (revision 299069) @@ -1,1104 +1,1104 @@ /*- * Copyright (c) 2006 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. */ /*- * Copyright (c) 2001-2005, Intel Corporation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the Intel Corporation nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Intel XScale Queue Manager support. * * Each IXP4XXX device has a hardware block that implements a priority * queue manager that is shared between the XScale cpu and the backend * devices (such as the NPE). Queues are accessed by reading/writing * special memory locations. The queue contents are mapped into a shared * SRAM region with entries managed in a circular buffer. The XScale * processor can receive interrupts based on queue contents (a condition * code determines when interrupts should be delivered). * * The code here basically replaces the qmgr class in the Intel Access * Library (IAL). */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * State per AQM hw queue. * This structure holds q configuration and dispatch state. */ struct qmgrInfo { int qSizeInWords; /* queue size in words */ uint32_t qOflowStatBitMask; /* overflow status mask */ int qWriteCount; /* queue write count */ bus_size_t qAccRegAddr; /* access register */ bus_size_t qUOStatRegAddr; /* status register */ bus_size_t qConfigRegAddr; /* config register */ int qSizeInEntries; /* queue size in entries */ uint32_t qUflowStatBitMask; /* underflow status mask */ int qReadCount; /* queue read count */ /* XXX union */ uint32_t qStatRegAddr; uint32_t qStatBitsOffset; uint32_t qStat0BitMask; uint32_t qStat1BitMask; uint32_t intRegCheckMask; /* interrupt reg check mask */ void (*cb)(int, void *); /* callback function */ void *cbarg; /* callback argument */ int priority; /* dispatch priority */ #if 0 /* NB: needed only for A0 parts */ u_int statusWordOffset; /* status word offset */ uint32_t statusMask; /* status mask */ uint32_t statusCheckValue; /* status check value */ #endif }; struct ixpqmgr_softc { device_t sc_dev; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; struct resource *sc_irq1; /* IRQ resource */ void *sc_ih1; /* interrupt handler */ int sc_rid1; /* resource id for irq */ struct resource *sc_irq2; void *sc_ih2; int sc_rid2; struct qmgrInfo qinfo[IX_QMGR_MAX_NUM_QUEUES]; /* * This array contains a list of queue identifiers ordered by * priority. The table is split logically between queue * identifiers 0-31 and 32-63. To optimize lookups bit masks * are kept for the first-32 and last-32 q's. When the * table needs to be rebuilt mark rebuildTable and it'll * happen after the next interrupt. */ int priorityTable[IX_QMGR_MAX_NUM_QUEUES]; uint32_t lowPriorityTableFirstHalfMask; uint32_t uppPriorityTableFirstHalfMask; int rebuildTable; /* rebuild priorityTable */ uint32_t aqmFreeSramAddress; /* SRAM free space */ }; static int qmgr_debug; SYSCTL_INT(_debug, OID_AUTO, qmgr, CTLFLAG_RWTUN, &qmgr_debug, 0, "IXP4XX Q-Manager debug msgs"); #define DPRINTF(dev, fmt, ...) do { \ if (qmgr_debug) printf(fmt, __VA_ARGS__); \ } while (0) #define DPRINTFn(n, dev, fmt, ...) do { \ if (qmgr_debug >= n) printf(fmt, __VA_ARGS__); \ } while (0) static struct ixpqmgr_softc *ixpqmgr_sc = NULL; static void ixpqmgr_rebuild(struct ixpqmgr_softc *); static void ixpqmgr_intr(void *); static void aqm_int_enable(struct ixpqmgr_softc *sc, int qId); static void aqm_int_disable(struct ixpqmgr_softc *sc, int qId); static void aqm_qcfg(struct ixpqmgr_softc *sc, int qId, u_int ne, u_int nf); static void aqm_srcsel_write(struct ixpqmgr_softc *sc, int qId, int sourceId); static void aqm_reset(struct ixpqmgr_softc *sc); static void dummyCallback(int qId, void *arg) { /* XXX complain */ } static uint32_t aqm_reg_read(struct ixpqmgr_softc *sc, bus_size_t off) { DPRINTFn(9, sc->sc_dev, "%s(0x%x)\n", __func__, (int)off); return bus_space_read_4(sc->sc_iot, sc->sc_ioh, off); } static void aqm_reg_write(struct ixpqmgr_softc *sc, bus_size_t off, uint32_t val) { DPRINTFn(9, sc->sc_dev, "%s(0x%x, 0x%x)\n", __func__, (int)off, val); bus_space_write_4(sc->sc_iot, sc->sc_ioh, off, val); } static int ixpqmgr_probe(device_t dev) { device_set_desc(dev, "IXP4XX Q-Manager"); return 0; } static int ixpqmgr_attach(device_t dev) { struct ixpqmgr_softc *sc = device_get_softc(dev); struct ixp425_softc *sa = device_get_softc(device_get_parent(dev)); int i, err; ixpqmgr_sc = sc; sc->sc_dev = dev; sc->sc_iot = sa->sc_iot; if (bus_space_map(sc->sc_iot, IXP425_QMGR_HWBASE, IXP425_QMGR_SIZE, 0, &sc->sc_ioh)) panic("%s: Cannot map registers", device_get_name(dev)); /* NB: we only use the lower 32 q's */ /* Set up QMGR interrupts */ sc->sc_rid1 = 0; sc->sc_irq1 = bus_alloc_resource(dev, SYS_RES_IRQ, &sc->sc_rid1, IXP425_INT_QUE1_32, IXP425_INT_QUE1_32, 1, RF_ACTIVE); sc->sc_rid2 = 1; sc->sc_irq2 = bus_alloc_resource(dev, SYS_RES_IRQ, &sc->sc_rid2, IXP425_INT_QUE33_64, IXP425_INT_QUE33_64, 1, RF_ACTIVE); if (sc->sc_irq1 == NULL || sc->sc_irq2 == NULL) panic("Unable to allocate the qmgr irqs.\n"); err = bus_setup_intr(dev, sc->sc_irq1, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixpqmgr_intr, NULL, &sc->sc_ih1); if (err) { device_printf(dev, "failed to set up qmgr irq=%d\n", IXP425_INT_QUE1_32); return (ENXIO); } err = bus_setup_intr(dev, sc->sc_irq2, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixpqmgr_intr, NULL, &sc->sc_ih2); if (err) { device_printf(dev, "failed to set up qmgr irq=%d\n", IXP425_INT_QUE33_64); return (ENXIO); } /* NB: softc is pre-zero'd */ for (i = 0; i < IX_QMGR_MAX_NUM_QUEUES; i++) { struct qmgrInfo *qi = &sc->qinfo[i]; qi->cb = dummyCallback; qi->priority = IX_QMGR_Q_PRIORITY_0; /* default priority */ /* * There are two interrupt registers, 32 bits each. One * for the lower queues(0-31) and one for the upper * queues(32-63). Therefore need to mod by 32 i.e the * min upper queue identifier. */ qi->intRegCheckMask = (1<<(i%(IX_QMGR_MIN_QUEUPP_QID))); /* * Register addresses and bit masks are calculated and * stored here to optimize QRead, QWrite and QStatusGet * functions. */ /* AQM Queue access reg addresses, per queue */ qi->qAccRegAddr = IX_QMGR_Q_ACCESS_ADDR_GET(i); qi->qAccRegAddr = IX_QMGR_Q_ACCESS_ADDR_GET(i); qi->qConfigRegAddr = IX_QMGR_Q_CONFIG_ADDR_GET(i); /* AQM Queue lower-group (0-31), only */ if (i < IX_QMGR_MIN_QUEUPP_QID) { /* AQM Q underflow/overflow status reg address, per queue */ qi->qUOStatRegAddr = IX_QMGR_QUEUOSTAT0_OFFSET + ((i / IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD) * sizeof(uint32_t)); /* AQM Q underflow status bit masks for status reg per queue */ qi->qUflowStatBitMask = (IX_QMGR_UNDERFLOW_BIT_OFFSET + 1) << ((i & (IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD - 1)) * (32 / IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD)); /* AQM Q overflow status bit masks for status reg, per queue */ qi->qOflowStatBitMask = (IX_QMGR_OVERFLOW_BIT_OFFSET + 1) << ((i & (IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD - 1)) * (32 / IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD)); /* AQM Q lower-group (0-31) status reg addresses, per queue */ qi->qStatRegAddr = IX_QMGR_QUELOWSTAT0_OFFSET + ((i / IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD) * sizeof(uint32_t)); /* AQM Q lower-group (0-31) status register bit offset */ qi->qStatBitsOffset = (i & (IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD - 1)) * (32 / IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD); } else { /* AQM Q upper-group (32-63), only */ qi->qUOStatRegAddr = 0; /* XXX */ /* AQM Q upper-group (32-63) Nearly Empty status reg bitmasks */ qi->qStat0BitMask = (1 << (i - IX_QMGR_MIN_QUEUPP_QID)); /* AQM Q upper-group (32-63) Full status register bitmasks */ qi->qStat1BitMask = (1 << (i - IX_QMGR_MIN_QUEUPP_QID)); } } sc->aqmFreeSramAddress = 0x100; /* Q buffer space starts at 0x2100 */ ixpqmgr_rebuild(sc); /* build initial priority table */ aqm_reset(sc); /* reset h/w */ return (0); } static int ixpqmgr_detach(device_t dev) { struct ixpqmgr_softc *sc = device_get_softc(dev); aqm_reset(sc); /* disable interrupts */ bus_teardown_intr(dev, sc->sc_irq1, sc->sc_ih1); bus_teardown_intr(dev, sc->sc_irq2, sc->sc_ih2); bus_release_resource(dev, SYS_RES_IRQ, sc->sc_rid1, sc->sc_irq1); bus_release_resource(dev, SYS_RES_IRQ, sc->sc_rid2, sc->sc_irq2); bus_space_unmap(sc->sc_iot, sc->sc_ioh, IXP425_QMGR_SIZE); return (0); } int ixpqmgr_qconfig(int qId, int qEntries, int ne, int nf, int srcSel, qconfig_hand_t *cb, void *cbarg) { struct ixpqmgr_softc *sc = ixpqmgr_sc; struct qmgrInfo *qi = &sc->qinfo[qId]; DPRINTF(sc->sc_dev, "%s(%u, %u, %u, %u, %u, %p, %p)\n", __func__, qId, qEntries, ne, nf, srcSel, cb, cbarg); /* NB: entry size is always 1 */ qi->qSizeInWords = qEntries; qi->qReadCount = 0; qi->qWriteCount = 0; qi->qSizeInEntries = qEntries; /* XXX kept for code clarity */ if (cb == NULL) { /* Reset to dummy callback */ qi->cb = dummyCallback; qi->cbarg = NULL; } else { qi->cb = cb; qi->cbarg = cbarg; } /* Write the config register; NB must be AFTER qinfo setup */ aqm_qcfg(sc, qId, ne, nf); /* * Account for space just allocated to queue. */ sc->aqmFreeSramAddress += (qi->qSizeInWords * sizeof(uint32_t)); /* Set the interrupt source if this queue is in the range 0-31 */ if (qId < IX_QMGR_MIN_QUEUPP_QID) aqm_srcsel_write(sc, qId, srcSel); if (cb != NULL) /* Enable the interrupt */ aqm_int_enable(sc, qId); sc->rebuildTable = TRUE; return 0; /* XXX */ } int ixpqmgr_qwrite(int qId, uint32_t entry) { struct ixpqmgr_softc *sc = ixpqmgr_sc; struct qmgrInfo *qi = &sc->qinfo[qId]; DPRINTFn(3, sc->sc_dev, "%s(%u, 0x%x) writeCount %u size %u\n", __func__, qId, entry, qi->qWriteCount, qi->qSizeInEntries); /* write the entry */ aqm_reg_write(sc, qi->qAccRegAddr, entry); /* NB: overflow is available for lower queues only */ if (qId < IX_QMGR_MIN_QUEUPP_QID) { int qSize = qi->qSizeInEntries; /* * Increment the current number of entries in the queue * and check for overflow . */ if (qi->qWriteCount++ == qSize) { /* check for overflow */ uint32_t status = aqm_reg_read(sc, qi->qUOStatRegAddr); int qPtrs; /* * Read the status twice because the status may * not be immediately ready after the write operation */ if ((status & qi->qOflowStatBitMask) || ((status = aqm_reg_read(sc, qi->qUOStatRegAddr)) & qi->qOflowStatBitMask)) { /* * The queue is full, clear the overflow status bit if set. */ aqm_reg_write(sc, qi->qUOStatRegAddr, status & ~qi->qOflowStatBitMask); qi->qWriteCount = qSize; DPRINTFn(5, sc->sc_dev, "%s(%u, 0x%x) Q full, overflow status cleared\n", __func__, qId, entry); return ENOSPC; } /* - * No overflow occured : someone is draining the queue + * No overflow occurred : someone is draining the queue * and the current counter needs to be * updated from the current number of entries in the queue */ /* calculate number of words in q */ qPtrs = aqm_reg_read(sc, qi->qConfigRegAddr); DPRINTFn(2, sc->sc_dev, "%s(%u, 0x%x) Q full, no overflow status, qConfig 0x%x\n", __func__, qId, entry, qPtrs); qPtrs = (qPtrs - (qPtrs >> 7)) & 0x7f; if (qPtrs == 0) { /* * The queue may be full at the time of the * snapshot. Next access will check * the overflow status again. */ qi->qWriteCount = qSize; } else { /* convert the number of words to a number of entries */ qi->qWriteCount = qPtrs & (qSize - 1); } } } return 0; } int ixpqmgr_qread(int qId, uint32_t *entry) { struct ixpqmgr_softc *sc = ixpqmgr_sc; struct qmgrInfo *qi = &sc->qinfo[qId]; bus_size_t off = qi->qAccRegAddr; *entry = aqm_reg_read(sc, off); /* * Reset the current read count : next access to the read function * will force a underflow status check. */ qi->qReadCount = 0; /* Check if underflow occurred on the read */ if (*entry == 0 && qId < IX_QMGR_MIN_QUEUPP_QID) { /* get the queue status */ uint32_t status = aqm_reg_read(sc, qi->qUOStatRegAddr); if (status & qi->qUflowStatBitMask) { /* clear underflow status */ aqm_reg_write(sc, qi->qUOStatRegAddr, status &~ qi->qUflowStatBitMask); return ENOSPC; } } return 0; } int ixpqmgr_qreadm(int qId, uint32_t n, uint32_t *p) { struct ixpqmgr_softc *sc = ixpqmgr_sc; struct qmgrInfo *qi = &sc->qinfo[qId]; uint32_t entry; bus_size_t off = qi->qAccRegAddr; entry = aqm_reg_read(sc, off); while (--n) { if (entry == 0) { /* if we read a NULL entry, stop. We have underflowed */ break; } *p++ = entry; /* store */ entry = aqm_reg_read(sc, off); } *p = entry; /* * Reset the current read count : next access to the read function * will force a underflow status check. */ qi->qReadCount = 0; /* Check if underflow occurred on the read */ if (entry == 0 && qId < IX_QMGR_MIN_QUEUPP_QID) { /* get the queue status */ uint32_t status = aqm_reg_read(sc, qi->qUOStatRegAddr); if (status & qi->qUflowStatBitMask) { /* clear underflow status */ aqm_reg_write(sc, qi->qUOStatRegAddr, status &~ qi->qUflowStatBitMask); return ENOSPC; } } return 0; } uint32_t ixpqmgr_getqstatus(int qId) { #define QLOWSTATMASK \ ((1 << (32 / IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD)) - 1) struct ixpqmgr_softc *sc = ixpqmgr_sc; const struct qmgrInfo *qi = &sc->qinfo[qId]; uint32_t status; if (qId < IX_QMGR_MIN_QUEUPP_QID) { /* read the status of a queue in the range 0-31 */ status = aqm_reg_read(sc, qi->qStatRegAddr); /* mask out the status bits relevant only to this queue */ status = (status >> qi->qStatBitsOffset) & QLOWSTATMASK; } else { /* read status of a queue in the range 32-63 */ status = 0; if (aqm_reg_read(sc, IX_QMGR_QUEUPPSTAT0_OFFSET)&qi->qStat0BitMask) status |= IX_QMGR_Q_STATUS_NE_BIT_MASK; /* nearly empty */ if (aqm_reg_read(sc, IX_QMGR_QUEUPPSTAT1_OFFSET)&qi->qStat1BitMask) status |= IX_QMGR_Q_STATUS_F_BIT_MASK; /* full */ } return status; #undef QLOWSTATMASK } uint32_t ixpqmgr_getqconfig(int qId) { struct ixpqmgr_softc *sc = ixpqmgr_sc; return aqm_reg_read(sc, IX_QMGR_Q_CONFIG_ADDR_GET(qId)); } void ixpqmgr_dump(void) { struct ixpqmgr_softc *sc = ixpqmgr_sc; int i, a; /* status registers */ printf("0x%04x: %08x %08x %08x %08x\n" , 0x400 , aqm_reg_read(sc, 0x400) , aqm_reg_read(sc, 0x400+4) , aqm_reg_read(sc, 0x400+8) , aqm_reg_read(sc, 0x400+12) ); printf("0x%04x: %08x %08x %08x %08x\n" , 0x410 , aqm_reg_read(sc, 0x410) , aqm_reg_read(sc, 0x410+4) , aqm_reg_read(sc, 0x410+8) , aqm_reg_read(sc, 0x410+12) ); printf("0x%04x: %08x %08x %08x %08x\n" , 0x420 , aqm_reg_read(sc, 0x420) , aqm_reg_read(sc, 0x420+4) , aqm_reg_read(sc, 0x420+8) , aqm_reg_read(sc, 0x420+12) ); printf("0x%04x: %08x %08x %08x %08x\n" , 0x430 , aqm_reg_read(sc, 0x430) , aqm_reg_read(sc, 0x430+4) , aqm_reg_read(sc, 0x430+8) , aqm_reg_read(sc, 0x430+12) ); /* q configuration registers */ for (a = 0x2000; a < 0x20ff; a += 32) printf("0x%04x: %08x %08x %08x %08x %08x %08x %08x %08x\n" , a , aqm_reg_read(sc, a) , aqm_reg_read(sc, a+4) , aqm_reg_read(sc, a+8) , aqm_reg_read(sc, a+12) , aqm_reg_read(sc, a+16) , aqm_reg_read(sc, a+20) , aqm_reg_read(sc, a+24) , aqm_reg_read(sc, a+28) ); /* allocated SRAM */ for (i = 0x100; i < sc->aqmFreeSramAddress; i += 32) { a = 0x2000 + i; printf("0x%04x: %08x %08x %08x %08x %08x %08x %08x %08x\n" , a , aqm_reg_read(sc, a) , aqm_reg_read(sc, a+4) , aqm_reg_read(sc, a+8) , aqm_reg_read(sc, a+12) , aqm_reg_read(sc, a+16) , aqm_reg_read(sc, a+20) , aqm_reg_read(sc, a+24) , aqm_reg_read(sc, a+28) ); } for (i = 0; i < 16; i++) { printf("Q[%2d] config 0x%08x status 0x%02x " "Q[%2d] config 0x%08x status 0x%02x\n" , i, ixpqmgr_getqconfig(i), ixpqmgr_getqstatus(i) , i+16, ixpqmgr_getqconfig(i+16), ixpqmgr_getqstatus(i+16) ); } } void ixpqmgr_notify_enable(int qId, int srcSel) { struct ixpqmgr_softc *sc = ixpqmgr_sc; #if 0 /* Calculate the checkMask and checkValue for this q */ aqm_calc_statuscheck(sc, qId, srcSel); #endif /* Set the interrupt source if this queue is in the range 0-31 */ if (qId < IX_QMGR_MIN_QUEUPP_QID) aqm_srcsel_write(sc, qId, srcSel); /* Enable the interrupt */ aqm_int_enable(sc, qId); } void ixpqmgr_notify_disable(int qId) { struct ixpqmgr_softc *sc = ixpqmgr_sc; aqm_int_disable(sc, qId); } /* * Rebuild the priority table used by the dispatcher. */ static void ixpqmgr_rebuild(struct ixpqmgr_softc *sc) { int q, pri; int lowQuePriorityTableIndex, uppQuePriorityTableIndex; struct qmgrInfo *qi; sc->lowPriorityTableFirstHalfMask = 0; sc->uppPriorityTableFirstHalfMask = 0; lowQuePriorityTableIndex = 0; uppQuePriorityTableIndex = 32; for (pri = 0; pri < IX_QMGR_NUM_PRIORITY_LEVELS; pri++) { /* low priority q's */ for (q = 0; q < IX_QMGR_MIN_QUEUPP_QID; q++) { qi = &sc->qinfo[q]; if (qi->priority == pri) { /* * Build the priority table bitmask which match the * queues of the first half of the priority table. */ if (lowQuePriorityTableIndex < 16) { sc->lowPriorityTableFirstHalfMask |= qi->intRegCheckMask; } sc->priorityTable[lowQuePriorityTableIndex++] = q; } } /* high priority q's */ for (; q < IX_QMGR_MAX_NUM_QUEUES; q++) { qi = &sc->qinfo[q]; if (qi->priority == pri) { /* * Build the priority table bitmask which match the * queues of the first half of the priority table . */ if (uppQuePriorityTableIndex < 48) { sc->uppPriorityTableFirstHalfMask |= qi->intRegCheckMask; } sc->priorityTable[uppQuePriorityTableIndex++] = q; } } } sc->rebuildTable = FALSE; } /* * Count the number of leading zero bits in a word, * and return the same value than the CLZ instruction. * Note this is similar to the standard ffs function but * it counts zero's from the MSB instead of the LSB. * * word (in) return value (out) * 0x80000000 0 * 0x40000000 1 * ,,, ,,, * 0x00000002 30 * 0x00000001 31 * 0x00000000 32 * * The C version of this function is used as a replacement * for system not providing the equivalent of the CLZ * assembly language instruction. * * Note that this version is big-endian */ static unsigned int _lzcount(uint32_t word) { unsigned int lzcount = 0; if (word == 0) return 32; while ((word & 0x80000000) == 0) { word <<= 1; lzcount++; } return lzcount; } static void ixpqmgr_intr(void *arg) { struct ixpqmgr_softc *sc = ixpqmgr_sc; uint32_t intRegVal; /* Interrupt reg val */ struct qmgrInfo *qi; int priorityTableIndex; /* Priority table index */ int qIndex; /* Current queue being processed */ /* Read the interrupt register */ intRegVal = aqm_reg_read(sc, IX_QMGR_QINTREG0_OFFSET); /* Write back to clear interrupt */ aqm_reg_write(sc, IX_QMGR_QINTREG0_OFFSET, intRegVal); DPRINTFn(5, sc->sc_dev, "%s: ISR0 0x%x ISR1 0x%x\n", __func__, intRegVal, aqm_reg_read(sc, IX_QMGR_QINTREG1_OFFSET)); /* No queue has interrupt register set */ if (intRegVal != 0) { /* get the first queue Id from the interrupt register value */ qIndex = (32 - 1) - _lzcount(intRegVal); DPRINTFn(2, sc->sc_dev, "%s: ISR0 0x%x qIndex %u\n", __func__, intRegVal, qIndex); /* * Optimize for single callback case. */ qi = &sc->qinfo[qIndex]; if (intRegVal == qi->intRegCheckMask) { /* * Only 1 queue event triggered a notification. * Call the callback function for this queue */ qi->cb(qIndex, qi->cbarg); } else { /* * The event is triggered by more than 1 queue, * the queue search will start from the beginning * or the middle of the priority table. * * The search will end when all the bits of the interrupt * register are cleared. There is no need to maintain * a separate value and test it at each iteration. */ if (intRegVal & sc->lowPriorityTableFirstHalfMask) { priorityTableIndex = 0; } else { priorityTableIndex = 16; } /* * Iterate over the priority table until all the bits * of the interrupt register are cleared. */ do { qIndex = sc->priorityTable[priorityTableIndex++]; qi = &sc->qinfo[qIndex]; /* If this queue caused this interrupt to be raised */ if (intRegVal & qi->intRegCheckMask) { /* Call the callback function for this queue */ qi->cb(qIndex, qi->cbarg); /* Clear the interrupt register bit */ intRegVal &= ~qi->intRegCheckMask; } } while (intRegVal); } } /* Rebuild the priority table if needed */ if (sc->rebuildTable) ixpqmgr_rebuild(sc); } #if 0 /* * Generate the parameters used to check if a Q's status matches * the specified source select. We calculate which status word * to check (statusWordOffset), the value to check the status * against (statusCheckValue) and the mask (statusMask) to mask * out all but the bits to check in the status word. */ static void aqm_calc_statuscheck(int qId, IxQMgrSourceId srcSel) { struct qmgrInfo *qi = &qinfo[qId]; uint32_t shiftVal; if (qId < IX_QMGR_MIN_QUEUPP_QID) { switch (srcSel) { case IX_QMGR_Q_SOURCE_ID_E: qi->statusCheckValue = IX_QMGR_Q_STATUS_E_BIT_MASK; qi->statusMask = IX_QMGR_Q_STATUS_E_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NE: qi->statusCheckValue = IX_QMGR_Q_STATUS_NE_BIT_MASK; qi->statusMask = IX_QMGR_Q_STATUS_NE_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NF: qi->statusCheckValue = IX_QMGR_Q_STATUS_NF_BIT_MASK; qi->statusMask = IX_QMGR_Q_STATUS_NF_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_F: qi->statusCheckValue = IX_QMGR_Q_STATUS_F_BIT_MASK; qi->statusMask = IX_QMGR_Q_STATUS_F_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NOT_E: qi->statusCheckValue = 0; qi->statusMask = IX_QMGR_Q_STATUS_E_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NOT_NE: qi->statusCheckValue = 0; qi->statusMask = IX_QMGR_Q_STATUS_NE_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NOT_NF: qi->statusCheckValue = 0; qi->statusMask = IX_QMGR_Q_STATUS_NF_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NOT_F: qi->statusCheckValue = 0; qi->statusMask = IX_QMGR_Q_STATUS_F_BIT_MASK; break; default: /* Should never hit */ IX_OSAL_ASSERT(0); break; } /* One nibble of status per queue so need to shift the * check value and mask out to the correct position. */ shiftVal = (qId % IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD) * IX_QMGR_QUELOWSTAT_BITS_PER_Q; /* Calculate the which status word to check from the qId, * 8 Qs status per word */ qi->statusWordOffset = qId / IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD; qi->statusCheckValue <<= shiftVal; qi->statusMask <<= shiftVal; } else { /* One status word */ qi->statusWordOffset = 0; /* Single bits per queue and int source bit hardwired NE, * Qs start at 32. */ qi->statusMask = 1 << (qId - IX_QMGR_MIN_QUEUPP_QID); qi->statusCheckValue = qi->statusMask; } } #endif static void aqm_int_enable(struct ixpqmgr_softc *sc, int qId) { bus_size_t reg; uint32_t v; if (qId < IX_QMGR_MIN_QUEUPP_QID) reg = IX_QMGR_QUEIEREG0_OFFSET; else reg = IX_QMGR_QUEIEREG1_OFFSET; v = aqm_reg_read(sc, reg); aqm_reg_write(sc, reg, v | (1 << (qId % IX_QMGR_MIN_QUEUPP_QID))); DPRINTF(sc->sc_dev, "%s(%u) 0x%lx: 0x%x => 0x%x\n", __func__, qId, reg, v, aqm_reg_read(sc, reg)); } static void aqm_int_disable(struct ixpqmgr_softc *sc, int qId) { bus_size_t reg; uint32_t v; if (qId < IX_QMGR_MIN_QUEUPP_QID) reg = IX_QMGR_QUEIEREG0_OFFSET; else reg = IX_QMGR_QUEIEREG1_OFFSET; v = aqm_reg_read(sc, reg); aqm_reg_write(sc, reg, v &~ (1 << (qId % IX_QMGR_MIN_QUEUPP_QID))); DPRINTF(sc->sc_dev, "%s(%u) 0x%lx: 0x%x => 0x%x\n", __func__, qId, reg, v, aqm_reg_read(sc, reg)); } static unsigned log2(unsigned n) { unsigned count; /* * N.B. this function will return 0 if supplied 0. */ for (count = 0; n/2; count++) n /= 2; return count; } static __inline unsigned toAqmEntrySize(int entrySize) { /* entrySize 1("00"),2("01"),4("10") */ return log2(entrySize); } static __inline unsigned toAqmBufferSize(unsigned bufferSizeInWords) { /* bufferSize 16("00"),32("01),64("10"),128("11") */ return log2(bufferSizeInWords / IX_QMGR_MIN_BUFFER_SIZE); } static __inline unsigned toAqmWatermark(int watermark) { /* * Watermarks 0("000"),1("001"),2("010"),4("011"), * 8("100"),16("101"),32("110"),64("111") */ return log2(2 * watermark); } static void aqm_qcfg(struct ixpqmgr_softc *sc, int qId, u_int ne, u_int nf) { const struct qmgrInfo *qi = &sc->qinfo[qId]; uint32_t qCfg; uint32_t baseAddress; /* Build config register */ qCfg = ((toAqmEntrySize(1) & IX_QMGR_ENTRY_SIZE_MASK) << IX_QMGR_Q_CONFIG_ESIZE_OFFSET) | ((toAqmBufferSize(qi->qSizeInWords) & IX_QMGR_SIZE_MASK) << IX_QMGR_Q_CONFIG_BSIZE_OFFSET); /* baseAddress, calculated relative to start address */ baseAddress = sc->aqmFreeSramAddress; /* base address must be word-aligned */ KASSERT((baseAddress % IX_QMGR_BASE_ADDR_16_WORD_ALIGN) == 0, ("address not word-aligned")); /* Now convert to a 16 word pointer as required by QUECONFIG register */ baseAddress >>= IX_QMGR_BASE_ADDR_16_WORD_SHIFT; qCfg |= baseAddress << IX_QMGR_Q_CONFIG_BADDR_OFFSET; /* set watermarks */ qCfg |= (toAqmWatermark(ne) << IX_QMGR_Q_CONFIG_NE_OFFSET) | (toAqmWatermark(nf) << IX_QMGR_Q_CONFIG_NF_OFFSET); DPRINTF(sc->sc_dev, "%s(%u, %u, %u) 0x%x => 0x%x @ 0x%x\n", __func__, qId, ne, nf, aqm_reg_read(sc, IX_QMGR_Q_CONFIG_ADDR_GET(qId)), qCfg, IX_QMGR_Q_CONFIG_ADDR_GET(qId)); aqm_reg_write(sc, IX_QMGR_Q_CONFIG_ADDR_GET(qId), qCfg); } static void aqm_srcsel_write(struct ixpqmgr_softc *sc, int qId, int sourceId) { bus_size_t off; uint32_t v; /* * Calculate the register offset; multiple queues split across registers */ off = IX_QMGR_INT0SRCSELREG0_OFFSET + ((qId / IX_QMGR_INTSRC_NUM_QUE_PER_WORD) * sizeof(uint32_t)); v = aqm_reg_read(sc, off); if (off == IX_QMGR_INT0SRCSELREG0_OFFSET && qId == 0) { /* Queue 0 at INT0SRCSELREG should not corrupt the value bit-3 */ v |= 0x7; } else { const uint32_t bpq = 32 / IX_QMGR_INTSRC_NUM_QUE_PER_WORD; uint32_t mask; int qshift; qshift = (qId & (IX_QMGR_INTSRC_NUM_QUE_PER_WORD-1)) * bpq; mask = ((1 << bpq) - 1) << qshift; /* q's status mask */ /* merge sourceId */ v = (v &~ mask) | ((sourceId << qshift) & mask); } DPRINTF(sc->sc_dev, "%s(%u, %u) 0x%x => 0x%x @ 0x%lx\n", __func__, qId, sourceId, aqm_reg_read(sc, off), v, off); aqm_reg_write(sc, off, v); } /* * Reset AQM registers to default values. */ static void aqm_reset(struct ixpqmgr_softc *sc) { int i; /* Reset queues 0..31 status registers 0..3 */ aqm_reg_write(sc, IX_QMGR_QUELOWSTAT0_OFFSET, IX_QMGR_QUELOWSTAT_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUELOWSTAT1_OFFSET, IX_QMGR_QUELOWSTAT_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUELOWSTAT2_OFFSET, IX_QMGR_QUELOWSTAT_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUELOWSTAT3_OFFSET, IX_QMGR_QUELOWSTAT_RESET_VALUE); /* Reset underflow/overflow status registers 0..1 */ aqm_reg_write(sc, IX_QMGR_QUEUOSTAT0_OFFSET, IX_QMGR_QUEUOSTAT_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUEUOSTAT1_OFFSET, IX_QMGR_QUEUOSTAT_RESET_VALUE); /* Reset queues 32..63 nearly empty status registers */ aqm_reg_write(sc, IX_QMGR_QUEUPPSTAT0_OFFSET, IX_QMGR_QUEUPPSTAT0_RESET_VALUE); /* Reset queues 32..63 full status registers */ aqm_reg_write(sc, IX_QMGR_QUEUPPSTAT1_OFFSET, IX_QMGR_QUEUPPSTAT1_RESET_VALUE); /* Reset int0 status flag source select registers 0..3 */ aqm_reg_write(sc, IX_QMGR_INT0SRCSELREG0_OFFSET, IX_QMGR_INT0SRCSELREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_INT0SRCSELREG1_OFFSET, IX_QMGR_INT0SRCSELREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_INT0SRCSELREG2_OFFSET, IX_QMGR_INT0SRCSELREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_INT0SRCSELREG3_OFFSET, IX_QMGR_INT0SRCSELREG_RESET_VALUE); /* Reset queue interrupt enable register 0..1 */ aqm_reg_write(sc, IX_QMGR_QUEIEREG0_OFFSET, IX_QMGR_QUEIEREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUEIEREG1_OFFSET, IX_QMGR_QUEIEREG_RESET_VALUE); /* Reset queue interrupt register 0..1 */ aqm_reg_write(sc, IX_QMGR_QINTREG0_OFFSET, IX_QMGR_QINTREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QINTREG1_OFFSET, IX_QMGR_QINTREG_RESET_VALUE); /* Reset queue configuration words 0..63 */ for (i = 0; i < IX_QMGR_MAX_NUM_QUEUES; i++) aqm_reg_write(sc, sc->qinfo[i].qConfigRegAddr, IX_QMGR_QUECONFIG_RESET_VALUE); /* XXX zero SRAM to simplify debugging */ for (i = IX_QMGR_QUEBUFFER_SPACE_OFFSET; i < IX_QMGR_AQM_SRAM_SIZE_IN_BYTES; i += sizeof(uint32_t)) aqm_reg_write(sc, i, 0); } static device_method_t ixpqmgr_methods[] = { DEVMETHOD(device_probe, ixpqmgr_probe), DEVMETHOD(device_attach, ixpqmgr_attach), DEVMETHOD(device_detach, ixpqmgr_detach), { 0, 0 } }; static driver_t ixpqmgr_driver = { "ixpqmgr", ixpqmgr_methods, sizeof(struct ixpqmgr_softc), }; static devclass_t ixpqmgr_devclass; DRIVER_MODULE(ixpqmgr, ixp, ixpqmgr_driver, ixpqmgr_devclass, 0, 0); Index: head/sys/arm/xscale/pxa/pxa_machdep.c =================================================================== --- head/sys/arm/xscale/pxa/pxa_machdep.c (revision 299068) +++ head/sys/arm/xscale/pxa/pxa_machdep.c (revision 299069) @@ -1,437 +1,437 @@ /* $NetBSD: hpc_machdep.c,v 1.70 2003/09/16 08:18:22 agc Exp $ */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * machdep.c * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * This file needs a lot of work. * * Created : 17/09/94 */ #include "opt_ddb.h" #include "opt_kstack_pages.h" #include __FBSDID("$FreeBSD$"); #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define KERNEL_PT_SYS 0 /* Page table for mapping proc0 zero page */ #define KERNEL_PT_IOPXS 1 #define KERNEL_PT_BEFOREKERN 2 #define KERNEL_PT_AFKERNEL 3 /* L2 table for mapping after kernel */ #define KERNEL_PT_AFKERNEL_NUM 9 /* this should be evenly divisable by PAGE_SIZE / L2_TABLE_SIZE_REAL (or 4) */ #define NUM_KERNEL_PTS (KERNEL_PT_AFKERNEL + KERNEL_PT_AFKERNEL_NUM) struct pv_addr kernel_pt_table[NUM_KERNEL_PTS]; /* Physical and virtual addresses for some global pages */ struct pv_addr systempage; struct pv_addr msgbufpv; struct pv_addr irqstack; struct pv_addr undstack; struct pv_addr abtstack; struct pv_addr kernelstack; struct pv_addr minidataclean; static void pxa_probe_sdram(bus_space_tag_t, bus_space_handle_t, uint32_t *, uint32_t *); /* Static device mappings. */ static const struct devmap_entry pxa_devmap[] = { /* * Map the on-board devices up into the KVA region so we don't muck * up user-space. */ { PXA2X0_PERIPH_START + PXA2X0_PERIPH_OFFSET, PXA2X0_PERIPH_START, PXA250_PERIPH_END - PXA2X0_PERIPH_START, }, { 0, 0, 0, } }; #define SDRAM_START 0xa0000000 extern vm_offset_t xscale_cache_clean_addr; void * initarm(struct arm_boot_params *abp) { struct pv_addr kernel_l1pt; struct pv_addr dpcpu; int loop; u_int l1pagetable; vm_offset_t freemempos; vm_offset_t freemem_pt; vm_offset_t afterkern; vm_offset_t freemem_after; vm_offset_t lastaddr; int i, j; uint32_t memsize[PXA2X0_SDRAM_BANKS], memstart[PXA2X0_SDRAM_BANKS]; lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; set_cpufuncs(); pcpu_init(pcpup, 0, sizeof(struct pcpu)); PCPU_SET(curthread, &thread0); /* Do basic tuning, hz etc */ init_param1(); freemempos = 0xa0200000; /* Define a macro to simplify memory allocation */ #define valloc_pages(var, np) \ alloc_pages((var).pv_pa, (np)); \ (var).pv_va = (var).pv_pa + 0x20000000; #define alloc_pages(var, np) \ freemempos -= (np * PAGE_SIZE); \ (var) = freemempos; \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos -= PAGE_SIZE; valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); for (loop = 0; loop < NUM_KERNEL_PTS; ++loop) { if (!(loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[loop], L2_TABLE_SIZE / PAGE_SIZE); } else { kernel_pt_table[loop].pv_pa = freemempos + (loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL)) * L2_TABLE_SIZE_REAL; kernel_pt_table[loop].pv_va = kernel_pt_table[loop].pv_pa + 0x20000000; } } freemem_pt = freemempos; freemempos = 0xa0100000; /* * Allocate a page for the system page mapped to V0x00000000 * This page will just contain the system vectors and can be * shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE); valloc_pages(abtstack, ABT_STACK_SIZE); valloc_pages(undstack, UND_STACK_SIZE); valloc_pages(kernelstack, kstack_pages); alloc_pages(minidataclean.pv_pa, 1); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Allocate memory for the l1 and l2 page tables. The scheme to avoid * wasting memory by allocating the l1pt on the first 16k memory was * taken from NetBSD rpc_machdep.c. NKPT should be greater than 12 for * this to work (which is supposed to be the case). */ /* * Now we start construction of the L1 page table * We start by mapping the L2 page tables into the L1. * This means that we can replace L1 mappings later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* Map the L2 pages tables in the L1 page table */ pmap_link_l2pt(l1pagetable, rounddown2(ARM_VECTORS_HIGH, 0x00100000), &kernel_pt_table[KERNEL_PT_SYS]); #if 0 /* XXXBJR: What is this? Don't know if there's an analogue. */ pmap_link_l2pt(l1pagetable, IQ80321_IOPXS_VBASE, &kernel_pt_table[KERNEL_PT_IOPXS]); #endif pmap_link_l2pt(l1pagetable, KERNBASE, &kernel_pt_table[KERNEL_PT_BEFOREKERN]); pmap_map_chunk(l1pagetable, KERNBASE, SDRAM_START, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, KERNBASE + 0x100000, SDRAM_START + 0x100000, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, KERNBASE + 0x200000, SDRAM_START + 0x200000, rounddown2(((uint32_t)(lastaddr) - KERNBASE - 0x200000) + L1_S_SIZE, L1_S_SIZE), VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); freemem_after = rounddown2((int)lastaddr + PAGE_SIZE, PAGE_SIZE); afterkern = round_page(rounddown2((vm_offset_t)lastaddr + L1_S_SIZE, L1_S_SIZE)); for (i = 0; i < KERNEL_PT_AFKERNEL_NUM; i++) { pmap_link_l2pt(l1pagetable, afterkern + i * 0x00100000, &kernel_pt_table[KERNEL_PT_AFKERNEL + i]); } pmap_map_entry(l1pagetable, afterkern, minidataclean.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map the Mini-Data cache clean area. */ xscale_setup_minidata(l1pagetable, afterkern, minidataclean.pv_pa); /* Map the vector page. */ pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); devmap_bootstrap(l1pagetable, pxa_devmap); /* * Give the XScale global cache clean code an appropriately * sized chunk of unmapped VA space starting at 0xff000000 * (our device mappings end before this address). */ xscale_cache_clean_addr = 0xff000000U; cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); cpu_setup(); /* * Sort out bus_space for on-board devices. */ pxa_obio_tag_init(); /* * Fetch the SDRAM start/size from the PXA2X0 SDRAM configration * registers. */ pxa_probe_sdram(obio_tag, PXA2X0_MEMCTL_BASE, memstart, memsize); /* Fire up consoles. */ cninit(); undefined_init(); init_proc0(kernelstack.pv_va); /* Enable MMU, I-cache, D-cache, write buffer. */ arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_curmaxkvaddr = afterkern + PAGE_SIZE; vm_max_kernel_address = 0xe0000000; pmap_bootstrap(pmap_curmaxkvaddr, &kernel_l1pt); msgbufp = (void*)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Add the physical ram we have available. * * Exclude the kernel (and all the things we allocated which immediately * follow the kernel) from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ for (j = 0; j < PXA2X0_SDRAM_BANKS; j++) { if (memsize[j] > 0) arm_physmem_hardware_region(memstart[j], memsize[j]); } arm_physmem_exclude_region(freemem_pt, abp->abp_physaddr - freemem_pt, EXFLAG_NOALLOC); arm_physmem_exclude_region(freemempos, abp->abp_physaddr - 0x100000 - freemempos, EXFLAG_NOALLOC); arm_physmem_exclude_region(abp->abp_physaddr, virtual_avail - KERNVIRTADDR, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); } static void pxa_probe_sdram(bus_space_tag_t bst, bus_space_handle_t bsh, uint32_t *memstart, uint32_t *memsize) { uint32_t mdcnfg, dwid, dcac, drac, dnb; int i; mdcnfg = bus_space_read_4(bst, bsh, MEMCTL_MDCNFG); /* * Scan all 4 SDRAM banks */ for (i = 0; i < PXA2X0_SDRAM_BANKS; i++) { memstart[i] = 0; memsize[i] = 0; switch (i) { case 0: case 1: if ((i == 0 && (mdcnfg & MDCNFG_DE0) == 0) || (i == 1 && (mdcnfg & MDCNFG_DE1) == 0)) continue; dwid = mdcnfg >> MDCNFD_DWID01_SHIFT; dcac = mdcnfg >> MDCNFD_DCAC01_SHIFT; drac = mdcnfg >> MDCNFD_DRAC01_SHIFT; dnb = mdcnfg >> MDCNFD_DNB01_SHIFT; break; case 2: case 3: if ((i == 2 && (mdcnfg & MDCNFG_DE2) == 0) || (i == 3 && (mdcnfg & MDCNFG_DE3) == 0)) continue; dwid = mdcnfg >> MDCNFD_DWID23_SHIFT; dcac = mdcnfg >> MDCNFD_DCAC23_SHIFT; drac = mdcnfg >> MDCNFD_DRAC23_SHIFT; dnb = mdcnfg >> MDCNFD_DNB23_SHIFT; break; default: panic("pxa_probe_sdram: impossible"); } dwid = 2 << (1 - (dwid & MDCNFD_DWID_MASK)); /* 16/32 width */ dcac = 1 << ((dcac & MDCNFD_DCAC_MASK) + 8); /* 8-11 columns */ drac = 1 << ((drac & MDCNFD_DRAC_MASK) + 11); /* 11-13 rows */ dnb = 2 << (dnb & MDCNFD_DNB_MASK); /* # of banks */ memsize[i] = dwid * dcac * drac * dnb; memstart[i] = PXA2X0_SDRAM0_START + (i * PXA2X0_SDRAM_BANK_SIZE); } } #define TIMER_FREQUENCY 3686400 #define UNIMPLEMENTED panic("%s: unimplemented", __func__) /* XXXBJR: Belongs with DELAY in a timer.c of some sort. */ void cpu_startprofclock(void) { UNIMPLEMENTED; } void cpu_stopprofclock(void) { UNIMPLEMENTED; } static struct arm32_dma_range pxa_range = { .dr_sysbase = 0, .dr_busbase = 0, .dr_len = ~0u, }; struct arm32_dma_range * bus_dma_get_range(void) { return (&pxa_range); } int bus_dma_get_range_nb(void) { return (1); }