Index: head/sys/dev/drm2/i915/i915_dma.c =================================================================== --- head/sys/dev/drm2/i915/i915_dma.c (revision 287176) +++ head/sys/dev/drm2/i915/i915_dma.c (revision 287177) @@ -1,1727 +1,1714 @@ /* i915_dma.c -- DMA support for the I915 -*- linux-c -*- */ /* * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #define LP_RING(d) (&((struct drm_i915_private *)(d))->rings[RCS]) #define BEGIN_LP_RING(n) \ intel_ring_begin(LP_RING(dev_priv), (n)) #define OUT_RING(x) \ intel_ring_emit(LP_RING(dev_priv), x) #define ADVANCE_LP_RING() \ intel_ring_advance(LP_RING(dev_priv)) /** * Lock test for when it's just for synchronization of ring access. * * In that case, we don't need to do it when GEM is initialized as nobody else * has access to the ring. */ #define RING_LOCK_TEST_WITH_RETURN(dev, file) do { \ if (LP_RING(dev->dev_private)->obj == NULL) \ LOCK_TEST_WITH_RETURN(dev, file); \ } while (0) static inline u32 intel_read_legacy_status_page(struct drm_i915_private *dev_priv, int reg) { if (I915_NEED_GFX_HWS(dev_priv->dev)) return ((volatile u32*)(dev_priv->dri1.gfx_hws_cpu_addr))[reg]; else return intel_read_status_page(LP_RING(dev_priv), reg); } #define READ_HWSP(dev_priv, reg) intel_read_legacy_status_page(dev_priv, reg) #define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX) #define I915_BREADCRUMB_INDEX 0x21 void i915_update_dri1_breadcrumb(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv; if (dev->primary->master) { master_priv = dev->primary->master->driver_priv; if (master_priv->sarea_priv) master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); } } static void i915_write_hws_pga(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; u32 addr; addr = dev_priv->status_page_dmah->busaddr; if (INTEL_INFO(dev)->gen >= 4) addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; I915_WRITE(HWS_PGA, addr); } /** * Sets up the hardware status page for devices that need a physical address * in the register. */ static int i915_init_phys_hws(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = LP_RING(dev_priv); /* * Program Hardware Status Page * XXXKIB Keep 4GB limit for allocation for now. This method * of allocation is used on <= 965 hardware, that has several * erratas regarding the use of physical memory > 4 GB. */ dev_priv->status_page_dmah = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, BUS_SPACE_MAXADDR); if (!dev_priv->status_page_dmah) { DRM_ERROR("Can not allocate hardware status page\n"); return -ENOMEM; } ring->status_page.page_addr = dev_priv->hw_status_page = dev_priv->status_page_dmah->vaddr; dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr; memset(dev_priv->hw_status_page, 0, PAGE_SIZE); i915_write_hws_pga(dev); DRM_DEBUG("Enabled hardware status page, phys %jx\n", (uintmax_t)dev_priv->dma_status_page); return 0; } /** * Frees the hardware status page, whether it's a physical address or a virtual * address set up by the X Server. */ static void i915_free_hws(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = LP_RING(dev_priv); if (dev_priv->status_page_dmah) { drm_pci_free(dev, dev_priv->status_page_dmah); dev_priv->status_page_dmah = NULL; } if (dev_priv->status_gfx_addr) { dev_priv->status_gfx_addr = 0; ring->status_page.gfx_addr = 0; pmap_unmapdev((vm_offset_t)dev_priv->dri1.gfx_hws_cpu_addr, PAGE_SIZE); } /* Need to rewrite hardware status page */ I915_WRITE(HWS_PGA, 0x1ffff000); } void i915_kernel_lost_context(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv; struct intel_ring_buffer *ring = LP_RING(dev_priv); /* * We should never lose context on the ring with modesetting * as we don't expose it to userspace */ if (drm_core_check_feature(dev, DRIVER_MODESET)) return; ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; ring->space = ring->head - (ring->tail + 8); if (ring->space < 0) ring->space += ring->size; if (!dev->primary->master) return; master_priv = dev->primary->master->driver_priv; if (ring->head == ring->tail && master_priv->sarea_priv) master_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY; } static int i915_dma_cleanup(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; int i; /* Make sure interrupts are disabled here because the uninstall ioctl * may not have been called from userspace and after dev_private * is freed, it's too late. */ if (dev->irq_enabled) drm_irq_uninstall(dev); DRM_LOCK(dev); for (i = 0; i < I915_NUM_RINGS; i++) intel_cleanup_ring_buffer(&dev_priv->rings[i]); DRM_UNLOCK(dev); /* Clear the HWS virtual address at teardown */ if (I915_NEED_GFX_HWS(dev)) i915_free_hws(dev); return 0; } static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; int ret; master_priv->sarea = drm_getsarea(dev); if (master_priv->sarea) { master_priv->sarea_priv = (drm_i915_sarea_t *) ((u8 *)master_priv->sarea->handle + init->sarea_priv_offset); } else { DRM_DEBUG_DRIVER("sarea not found assuming DRI2 userspace\n"); } if (init->ring_size != 0) { if (LP_RING(dev_priv)->obj != NULL) { i915_dma_cleanup(dev); DRM_ERROR("Client tried to initialize ringbuffer in " "GEM mode\n"); return -EINVAL; } ret = intel_render_ring_init_dri(dev, init->ring_start, init->ring_size); if (ret) { i915_dma_cleanup(dev); return ret; } } dev_priv->cpp = init->cpp; dev_priv->back_offset = init->back_offset; dev_priv->front_offset = init->front_offset; dev_priv->current_page = 0; if (master_priv->sarea_priv) master_priv->sarea_priv->pf_current_page = 0; /* Allow hardware batchbuffers unless told otherwise. */ dev_priv->dri1.allow_batchbuffer = 1; return 0; } static int i915_dma_resume(struct drm_device * dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; struct intel_ring_buffer *ring = LP_RING(dev_priv); DRM_DEBUG("\n"); if (ring->virtual_start == NULL) { DRM_ERROR("can not ioremap virtual address for" " ring buffer\n"); return -ENOMEM; } /* Program Hardware Status Page */ if (!ring->status_page.page_addr) { DRM_ERROR("Can not find hardware status page\n"); return -EINVAL; } DRM_DEBUG("hw status page @ %p\n", ring->status_page.page_addr); if (ring->status_page.gfx_addr != 0) intel_ring_setup_status_page(ring); else i915_write_hws_pga(dev); DRM_DEBUG("Enabled hardware status page\n"); return 0; } static int i915_dma_init(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_init_t *init = data; int retcode = 0; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; switch (init->func) { case I915_INIT_DMA: retcode = i915_initialize(dev, init); break; case I915_CLEANUP_DMA: retcode = i915_dma_cleanup(dev); break; case I915_RESUME_DMA: retcode = i915_dma_resume(dev); break; default: retcode = -EINVAL; break; } return retcode; } /* Implement basically the same security restrictions as hardware does * for MI_BATCH_NON_SECURE. These can be made stricter at any time. * * Most of the calculations below involve calculating the size of a * particular instruction. It's important to get the size right as * that tells us where the next instruction to check is. Any illegal * instruction detected will be given a size of zero, which is a * signal to abort the rest of the buffer. */ static int validate_cmd(int cmd) { switch (((cmd >> 29) & 0x7)) { case 0x0: switch ((cmd >> 23) & 0x3f) { case 0x0: return 1; /* MI_NOOP */ case 0x4: return 1; /* MI_FLUSH */ default: return 0; /* disallow everything else */ } break; case 0x1: return 0; /* reserved */ case 0x2: return (cmd & 0xff) + 2; /* 2d commands */ case 0x3: if (((cmd >> 24) & 0x1f) <= 0x18) return 1; switch ((cmd >> 24) & 0x1f) { case 0x1c: return 1; case 0x1d: switch ((cmd >> 16) & 0xff) { case 0x3: return (cmd & 0x1f) + 2; case 0x4: return (cmd & 0xf) + 2; default: return (cmd & 0xffff) + 2; } case 0x1e: if (cmd & (1 << 23)) return (cmd & 0xffff) + 1; else return 1; case 0x1f: if ((cmd & (1 << 23)) == 0) /* inline vertices */ return (cmd & 0x1ffff) + 2; else if (cmd & (1 << 17)) /* indirect random */ if ((cmd & 0xffff) == 0) return 0; /* unknown length, too hard */ else return (((cmd & 0xffff) + 1) / 2) + 1; else return 2; /* indirect sequential */ default: return 0; } default: return 0; } return 0; } static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords) { drm_i915_private_t *dev_priv = dev->dev_private; int i; if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8) return -EINVAL; BEGIN_LP_RING((dwords+1)&~1); for (i = 0; i < dwords;) { int cmd, sz; if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd))) return -EINVAL; if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords) return -EINVAL; OUT_RING(cmd); while (++i, --sz) { if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd))) { return -EINVAL; } OUT_RING(cmd); } } if (dwords & 1) OUT_RING(0); ADVANCE_LP_RING(); return 0; } -int i915_emit_box(struct drm_device * dev, - struct drm_clip_rect *boxes, - int i, int DR1, int DR4) -{ - struct drm_clip_rect box; - - if (DRM_COPY_FROM_USER_UNCHECKED(&box, &boxes[i], sizeof(box))) { - return -EFAULT; - } - - return (i915_emit_box_p(dev, &box, DR1, DR4)); -} - int -i915_emit_box_p(struct drm_device *dev, +i915_emit_box(struct drm_device *dev, struct drm_clip_rect *box, int DR1, int DR4) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; if (box->y2 <= box->y1 || box->x2 <= box->x1 || box->y2 <= 0 || box->x2 <= 0) { DRM_ERROR("Bad box %d,%d..%d,%d\n", box->x1, box->y1, box->x2, box->y2); return -EINVAL; } if (INTEL_INFO(dev)->gen >= 4) { ret = BEGIN_LP_RING(4); if (ret) return ret; OUT_RING(GFX_OP_DRAWRECT_INFO_I965); OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); OUT_RING(DR4); } else { ret = BEGIN_LP_RING(6); if (ret) return ret; OUT_RING(GFX_OP_DRAWRECT_INFO); OUT_RING(DR1); OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); OUT_RING(DR4); OUT_RING(0); } ADVANCE_LP_RING(); return 0; } /* XXX: Emitting the counter should really be moved to part of the IRQ * emit. For now, do it in both places: */ static void i915_emit_breadcrumb(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; if (++dev_priv->counter > 0x7FFFFFFFUL) dev_priv->counter = 0; if (master_priv->sarea_priv) master_priv->sarea_priv->last_enqueue = dev_priv->counter; if (BEGIN_LP_RING(4) == 0) { OUT_RING(MI_STORE_DWORD_INDEX); OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); OUT_RING(dev_priv->counter); OUT_RING(0); ADVANCE_LP_RING(); } } static int i915_dispatch_cmdbuffer(struct drm_device * dev, drm_i915_cmdbuffer_t *cmd, struct drm_clip_rect *cliprects, void *cmdbuf) { int nbox = cmd->num_cliprects; int i = 0, count, ret; if (cmd->sz & 0x3) { DRM_ERROR("alignment\n"); return -EINVAL; } i915_kernel_lost_context(dev); count = nbox ? nbox : 1; for (i = 0; i < count; i++) { if (i < nbox) { - ret = i915_emit_box_p(dev, &cmd->cliprects[i], - cmd->DR1, cmd->DR4); + ret = i915_emit_box(dev, &cliprects[i], + cmd->DR1, cmd->DR4); if (ret) return ret; } ret = i915_emit_cmds(dev, cmdbuf, cmd->sz / 4); if (ret) return ret; } i915_emit_breadcrumb(dev); return 0; } static int i915_dispatch_batchbuffer(struct drm_device * dev, drm_i915_batchbuffer_t * batch, struct drm_clip_rect *cliprects) { struct drm_i915_private *dev_priv = dev->dev_private; int nbox = batch->num_cliprects; int i, count, ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; if ((batch->start | batch->used) & 0x7) { DRM_ERROR("alignment\n"); return -EINVAL; } i915_kernel_lost_context(dev); count = nbox ? nbox : 1; for (i = 0; i < count; i++) { if (i < nbox) { - int ret = i915_emit_box_p(dev, &cliprects[i], - batch->DR1, batch->DR4); + ret = i915_emit_box(dev, &cliprects[i], + batch->DR1, batch->DR4); if (ret) return ret; } if (!IS_I830(dev) && !IS_845G(dev)) { ret = BEGIN_LP_RING(2); if (ret) return ret; if (INTEL_INFO(dev)->gen >= 4) { OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965); OUT_RING(batch->start); } else { OUT_RING(MI_BATCH_BUFFER_START | (2 << 6)); OUT_RING(batch->start | MI_BATCH_NON_SECURE); } } else { ret = BEGIN_LP_RING(4); if (ret) return ret; OUT_RING(MI_BATCH_BUFFER); OUT_RING(batch->start | MI_BATCH_NON_SECURE); OUT_RING(batch->start + batch->used - 4); OUT_RING(0); } ADVANCE_LP_RING(); } i915_emit_breadcrumb(dev); return 0; } static int i915_dispatch_flip(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; int ret; if (!master_priv->sarea_priv) return -EINVAL; DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n", __func__, dev_priv->current_page, master_priv->sarea_priv->pf_current_page); i915_kernel_lost_context(dev); ret = BEGIN_LP_RING(10); if (ret) return ret; OUT_RING(MI_FLUSH | MI_READ_FLUSH); OUT_RING(0); OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP); OUT_RING(0); if (dev_priv->current_page == 0) { OUT_RING(dev_priv->back_offset); dev_priv->current_page = 1; } else { OUT_RING(dev_priv->front_offset); dev_priv->current_page = 0; } OUT_RING(0); OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP); OUT_RING(0); ADVANCE_LP_RING(); master_priv->sarea_priv->last_enqueue = dev_priv->counter++; if (BEGIN_LP_RING(4) == 0) { OUT_RING(MI_STORE_DWORD_INDEX); OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); OUT_RING(dev_priv->counter); OUT_RING(0); ADVANCE_LP_RING(); } master_priv->sarea_priv->pf_current_page = dev_priv->current_page; return 0; } static int i915_quiescent(struct drm_device *dev) { struct intel_ring_buffer *ring = LP_RING(dev->dev_private); i915_kernel_lost_context(dev); return (intel_wait_ring_idle(ring)); } static int i915_flush_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { int ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; RING_LOCK_TEST_WITH_RETURN(dev, file_priv); DRM_LOCK(dev); ret = i915_quiescent(dev); DRM_UNLOCK(dev); return (ret); } int i915_batchbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv; drm_i915_batchbuffer_t *batch = data; struct drm_clip_rect *cliprects; size_t cliplen; int ret; if (!dev_priv->dri1.allow_batchbuffer) { DRM_ERROR("Batchbuffer ioctl disabled\n"); return -EINVAL; } DRM_DEBUG("i915 batchbuffer, start %x used %d cliprects %d\n", batch->start, batch->used, batch->num_cliprects); cliplen = batch->num_cliprects * sizeof(struct drm_clip_rect); if (batch->num_cliprects < 0) return -EFAULT; if (batch->num_cliprects != 0) { cliprects = malloc(batch->num_cliprects * sizeof(struct drm_clip_rect), DRM_MEM_DMA, M_WAITOK | M_ZERO); ret = -copyin(batch->cliprects, cliprects, batch->num_cliprects * sizeof(struct drm_clip_rect)); if (ret != 0) goto fail_free; } else cliprects = NULL; DRM_LOCK(dev); RING_LOCK_TEST_WITH_RETURN(dev, file_priv); ret = i915_dispatch_batchbuffer(dev, batch, cliprects); DRM_UNLOCK(dev); if (sarea_priv) sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); fail_free: free(cliprects, DRM_MEM_DMA); return ret; } int i915_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv; drm_i915_cmdbuffer_t *cmdbuf = data; struct drm_clip_rect *cliprects = NULL; void *batch_data; int ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; DRM_DEBUG("i915 cmdbuffer, buf %p sz %d cliprects %d\n", cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects); if (cmdbuf->num_cliprects < 0) return -EINVAL; batch_data = malloc(cmdbuf->sz, DRM_MEM_DMA, M_WAITOK); ret = -copyin(cmdbuf->buf, batch_data, cmdbuf->sz); if (ret != 0) goto fail_batch_free; if (cmdbuf->num_cliprects) { cliprects = malloc(cmdbuf->num_cliprects * sizeof(struct drm_clip_rect), DRM_MEM_DMA, M_WAITOK | M_ZERO); ret = -copyin(cmdbuf->cliprects, cliprects, cmdbuf->num_cliprects * sizeof(struct drm_clip_rect)); if (ret != 0) goto fail_clip_free; } DRM_LOCK(dev); RING_LOCK_TEST_WITH_RETURN(dev, file_priv); ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects, batch_data); DRM_UNLOCK(dev); if (ret) { DRM_ERROR("i915_dispatch_cmdbuffer failed\n"); goto fail_clip_free; } if (sarea_priv) sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); fail_clip_free: free(cliprects, DRM_MEM_DMA); fail_batch_free: free(batch_data, DRM_MEM_DMA); return ret; } static int i915_emit_irq(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; i915_kernel_lost_context(dev); DRM_DEBUG("i915: emit_irq\n"); dev_priv->counter++; if (dev_priv->counter > 0x7FFFFFFFUL) dev_priv->counter = 1; if (master_priv->sarea_priv) master_priv->sarea_priv->last_enqueue = dev_priv->counter; if (BEGIN_LP_RING(4) == 0) { OUT_RING(MI_STORE_DWORD_INDEX); OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); OUT_RING(dev_priv->counter); OUT_RING(MI_USER_INTERRUPT); ADVANCE_LP_RING(); } return dev_priv->counter; } static int i915_wait_irq(struct drm_device * dev, int irq_nr) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; int ret; struct intel_ring_buffer *ring = LP_RING(dev_priv); DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr, READ_BREADCRUMB(dev_priv)); if (READ_BREADCRUMB(dev_priv) >= irq_nr) { if (master_priv->sarea_priv) master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); return 0; } if (master_priv->sarea_priv) master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; ret = 0; mtx_lock(&dev_priv->irq_lock); if (ring->irq_get(ring)) { while (ret == 0 && READ_BREADCRUMB(dev_priv) < irq_nr) { ret = -msleep(ring, &dev_priv->irq_lock, PCATCH, "915wtq", 3 * hz); if (ret == -ERESTART) ret = -ERESTARTSYS; } ring->irq_put(ring); mtx_unlock(&dev_priv->irq_lock); } else { mtx_unlock(&dev_priv->irq_lock); if (_intel_wait_for(dev, READ_BREADCRUMB(dev_priv) >= irq_nr, 3000, 1, "915wir")) ret = -EBUSY; } if (ret == -EBUSY) { DRM_ERROR("EBUSY -- rec: %d emitted: %d\n", READ_BREADCRUMB(dev_priv), (int)dev_priv->counter); } return ret; } /* Needs the lock as it touches the ring. */ int i915_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_irq_emit_t *emit = data; int result; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; if (!dev_priv || !LP_RING(dev_priv)->virtual_start) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } RING_LOCK_TEST_WITH_RETURN(dev, file_priv); DRM_LOCK(dev); result = i915_emit_irq(dev); DRM_UNLOCK(dev); if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) { DRM_ERROR("copy_to_user\n"); return -EFAULT; } return 0; } /* Doesn't need the hardware lock. */ static int i915_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_irq_wait_t *irqwait = data; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } return i915_wait_irq(dev, irqwait->irq_seq); } static int i915_vblank_pipe_get(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_vblank_pipe_t *pipe = data; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; return 0; } /** * Schedule buffer swap at given vertical blank. */ static int i915_vblank_swap(struct drm_device *dev, void *data, struct drm_file *file_priv) { /* The delayed swap mechanism was fundamentally racy, and has been * removed. The model was that the client requested a delayed flip/swap * from the kernel, then waited for vblank before continuing to perform * rendering. The problem was that the kernel might wake the client * up before it dispatched the vblank swap (since the lock has to be * held while touching the ringbuffer), in which case the client would * clear and start the next frame before the swap occurred, and * flicker would occur in addition to likely missing the vblank. * * In the absence of this ioctl, userland falls back to a correct path * of waiting for a vblank, then dispatching the swap on its own. * Context switching to userland and back is plenty fast enough for * meeting the requirements of vblank swapping. */ return -EINVAL; } static int i915_flip_bufs(struct drm_device *dev, void *data, struct drm_file *file_priv) { int ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; DRM_DEBUG("%s\n", __func__); RING_LOCK_TEST_WITH_RETURN(dev, file_priv); DRM_LOCK(dev); ret = i915_dispatch_flip(dev); DRM_UNLOCK(dev); return ret; } int i915_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_getparam_t *param = data; int value; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } switch (param->param) { case I915_PARAM_IRQ_ACTIVE: value = dev->irq_enabled ? 1 : 0; break; case I915_PARAM_ALLOW_BATCHBUFFER: value = dev_priv->dri1.allow_batchbuffer ? 1 : 0; break; case I915_PARAM_LAST_DISPATCH: value = READ_BREADCRUMB(dev_priv); break; case I915_PARAM_CHIPSET_ID: value = dev->pci_device; break; case I915_PARAM_HAS_GEM: value = 1; break; case I915_PARAM_NUM_FENCES_AVAIL: value = dev_priv->num_fence_regs - dev_priv->fence_reg_start; break; case I915_PARAM_HAS_OVERLAY: value = dev_priv->overlay ? 1 : 0; break; case I915_PARAM_HAS_PAGEFLIPPING: value = 1; break; case I915_PARAM_HAS_EXECBUF2: value = 1; break; case I915_PARAM_HAS_BSD: value = intel_ring_initialized(&dev_priv->rings[VCS]); break; case I915_PARAM_HAS_BLT: value = intel_ring_initialized(&dev_priv->rings[BCS]); break; case I915_PARAM_HAS_RELAXED_FENCING: value = 1; break; case I915_PARAM_HAS_COHERENT_RINGS: value = 1; break; case I915_PARAM_HAS_EXEC_CONSTANTS: value = INTEL_INFO(dev)->gen >= 4; break; case I915_PARAM_HAS_RELAXED_DELTA: value = 1; break; case I915_PARAM_HAS_GEN7_SOL_RESET: value = 1; break; case I915_PARAM_HAS_LLC: value = HAS_LLC(dev); break; case I915_PARAM_HAS_ALIASING_PPGTT: value = dev_priv->mm.aliasing_ppgtt ? 1 : 0; break; default: DRM_DEBUG_DRIVER("Unknown parameter %d\n", param->param); return -EINVAL; } if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) { DRM_ERROR("DRM_COPY_TO_USER failed\n"); return -EFAULT; } return 0; } static int i915_setparam(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_setparam_t *param = data; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } switch (param->param) { case I915_SETPARAM_USE_MI_BATCHBUFFER_START: break; case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY: break; case I915_SETPARAM_ALLOW_BATCHBUFFER: dev_priv->dri1.allow_batchbuffer = param->value ? 1 : 0; break; case I915_SETPARAM_NUM_USED_FENCES: if (param->value > dev_priv->num_fence_regs || param->value < 0) return -EINVAL; /* Userspace can use first N regs */ dev_priv->fence_reg_start = param->value; break; default: DRM_DEBUG("unknown parameter %d\n", param->param); return -EINVAL; } return 0; } static int i915_set_status_page(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_hws_addr_t *hws = data; struct intel_ring_buffer *ring = LP_RING(dev_priv); if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; if (!I915_NEED_GFX_HWS(dev)) return -EINVAL; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } DRM_DEBUG("set status page addr 0x%08x\n", (u32)hws->addr); if (drm_core_check_feature(dev, DRIVER_MODESET)) { DRM_ERROR("tried to set status page when mode setting active\n"); return 0; } ring->status_page.gfx_addr = dev_priv->status_gfx_addr = hws->addr & (0x1ffff<<12); dev_priv->dri1.gfx_hws_cpu_addr = pmap_mapdev_attr( dev->agp->base + hws->addr, PAGE_SIZE, VM_MEMATTR_WRITE_COMBINING); if (dev_priv->dri1.gfx_hws_cpu_addr == NULL) { i915_dma_cleanup(dev); ring->status_page.gfx_addr = dev_priv->status_gfx_addr = 0; DRM_ERROR("can not ioremap virtual address for" " G33 hw status page\n"); return -ENOMEM; } memset(dev_priv->dri1.gfx_hws_cpu_addr, 0, PAGE_SIZE); I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr); DRM_DEBUG("load hws HWS_PGA with gfx mem 0x%x\n", dev_priv->status_gfx_addr); DRM_DEBUG("load hws at %p\n", dev_priv->hw_status_page); return 0; } static int i915_load_modeset_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; ret = intel_parse_bios(dev); if (ret) DRM_INFO("failed to find VBIOS tables\n"); #if 0 intel_register_dsm_handler(); #endif /* Initialise stolen first so that we may reserve preallocated * objects for the BIOS to KMS transition. */ ret = i915_gem_init_stolen(dev); if (ret) goto cleanup_vga_switcheroo; intel_modeset_init(dev); ret = i915_gem_init(dev); if (ret) goto cleanup_gem_stolen; intel_modeset_gem_init(dev); ret = drm_irq_install(dev); if (ret) goto cleanup_gem; dev->vblank_disable_allowed = 1; ret = intel_fbdev_init(dev); if (ret) goto cleanup_gem; drm_kms_helper_poll_init(dev); /* We're off and running w/KMS */ dev_priv->mm.suspended = 0; return 0; cleanup_gem: DRM_LOCK(dev); i915_gem_cleanup_ringbuffer(dev); DRM_UNLOCK(dev); i915_gem_cleanup_aliasing_ppgtt(dev); cleanup_gem_stolen: i915_gem_cleanup_stolen(dev); cleanup_vga_switcheroo: return (ret); } int i915_master_create(struct drm_device *dev, struct drm_master *master) { struct drm_i915_master_private *master_priv; master_priv = malloc(sizeof(*master_priv), DRM_MEM_DMA, M_NOWAIT | M_ZERO); if (!master_priv) return -ENOMEM; master->driver_priv = master_priv; return 0; } void i915_master_destroy(struct drm_device *dev, struct drm_master *master) { struct drm_i915_master_private *master_priv = master->driver_priv; if (!master_priv) return; free(master_priv, DRM_MEM_DMA); master->driver_priv = NULL; } static int i915_get_bridge_dev(struct drm_device *dev) { struct drm_i915_private *dev_priv; dev_priv = dev->dev_private; dev_priv->bridge_dev = intel_gtt_get_bridge_device(); if (dev_priv->bridge_dev == NULL) { DRM_ERROR("bridge device not found\n"); return (-1); } return (0); } #define MCHBAR_I915 0x44 #define MCHBAR_I965 0x48 #define MCHBAR_SIZE (4*4096) #define DEVEN_REG 0x54 #define DEVEN_MCHBAR_EN (1 << 28) /* Allocate space for the MCH regs if needed, return nonzero on error */ static int intel_alloc_mchbar_resource(struct drm_device *dev) { drm_i915_private_t *dev_priv; device_t vga; int reg; u32 temp_lo, temp_hi; u64 mchbar_addr, temp; dev_priv = dev->dev_private; reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; if (INTEL_INFO(dev)->gen >= 4) temp_hi = pci_read_config(dev_priv->bridge_dev, reg + 4, 4); else temp_hi = 0; temp_lo = pci_read_config(dev_priv->bridge_dev, reg, 4); mchbar_addr = ((u64)temp_hi << 32) | temp_lo; /* If ACPI doesn't have it, assume we need to allocate it ourselves */ #ifdef XXX_CONFIG_PNP if (mchbar_addr && pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) return 0; #endif /* Get some space for it */ vga = device_get_parent(dev->dev); dev_priv->mch_res_rid = 0x100; dev_priv->mch_res = BUS_ALLOC_RESOURCE(device_get_parent(vga), dev->dev, SYS_RES_MEMORY, &dev_priv->mch_res_rid, 0, ~0UL, MCHBAR_SIZE, RF_ACTIVE | RF_SHAREABLE); if (dev_priv->mch_res == NULL) { DRM_ERROR("failed mchbar resource alloc\n"); return (-ENOMEM); } if (INTEL_INFO(dev)->gen >= 4) { temp = rman_get_start(dev_priv->mch_res); temp >>= 32; pci_write_config(dev_priv->bridge_dev, reg + 4, temp, 4); } pci_write_config(dev_priv->bridge_dev, reg, rman_get_start(dev_priv->mch_res) & UINT32_MAX, 4); return (0); } static void intel_setup_mchbar(struct drm_device *dev) { drm_i915_private_t *dev_priv; int mchbar_reg; u32 temp; bool enabled; dev_priv = dev->dev_private; mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; dev_priv->mchbar_need_disable = false; if (IS_I915G(dev) || IS_I915GM(dev)) { temp = pci_read_config(dev_priv->bridge_dev, DEVEN_REG, 4); enabled = (temp & DEVEN_MCHBAR_EN) != 0; } else { temp = pci_read_config(dev_priv->bridge_dev, mchbar_reg, 4); enabled = temp & 1; } /* If it's already enabled, don't have to do anything */ if (enabled) { DRM_DEBUG("mchbar already enabled\n"); return; } if (intel_alloc_mchbar_resource(dev)) return; dev_priv->mchbar_need_disable = true; /* Space is allocated or reserved, so enable it. */ if (IS_I915G(dev) || IS_I915GM(dev)) { pci_write_config(dev_priv->bridge_dev, DEVEN_REG, temp | DEVEN_MCHBAR_EN, 4); } else { temp = pci_read_config(dev_priv->bridge_dev, mchbar_reg, 4); pci_write_config(dev_priv->bridge_dev, mchbar_reg, temp | 1, 4); } } static void intel_teardown_mchbar(struct drm_device *dev) { drm_i915_private_t *dev_priv; device_t vga; int mchbar_reg; u32 temp; dev_priv = dev->dev_private; mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; if (dev_priv->mchbar_need_disable) { if (IS_I915G(dev) || IS_I915GM(dev)) { temp = pci_read_config(dev_priv->bridge_dev, DEVEN_REG, 4); temp &= ~DEVEN_MCHBAR_EN; pci_write_config(dev_priv->bridge_dev, DEVEN_REG, temp, 4); } else { temp = pci_read_config(dev_priv->bridge_dev, mchbar_reg, 4); temp &= ~1; pci_write_config(dev_priv->bridge_dev, mchbar_reg, temp, 4); } } if (dev_priv->mch_res != NULL) { vga = device_get_parent(dev->dev); BUS_DEACTIVATE_RESOURCE(device_get_parent(vga), dev->dev, SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); BUS_RELEASE_RESOURCE(device_get_parent(vga), dev->dev, SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); dev_priv->mch_res = NULL; } } /** * i915_driver_load - setup chip and create an initial config * @dev: DRM device * @flags: startup flags * * The driver load routine has to do several things: * - drive output discovery via intel_modeset_init() * - initialize the memory manager * - allocate initial config memory * - setup the DRM framebuffer with the allocated memory */ int i915_driver_load(struct drm_device *dev, unsigned long flags) { struct drm_i915_private *dev_priv = dev->dev_private; const struct intel_device_info *info; unsigned long base, size; int mmio_bar, ret; info = i915_get_device_id(dev->pci_device); /* Refuse to load on gen6+ without kms enabled. */ if (info->gen >= 6 && !drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; ret = 0; /* i915 has 4 more counters */ dev->counters += 4; dev->types[6] = _DRM_STAT_IRQ; dev->types[7] = _DRM_STAT_PRIMARY; dev->types[8] = _DRM_STAT_SECONDARY; dev->types[9] = _DRM_STAT_DMA; dev_priv = malloc(sizeof(drm_i915_private_t), DRM_MEM_DRIVER, M_ZERO | M_WAITOK); dev->dev_private = (void *)dev_priv; dev_priv->dev = dev; dev_priv->info = info; if (i915_get_bridge_dev(dev)) { free(dev_priv, DRM_MEM_DRIVER); return (-EIO); } dev_priv->mm.gtt = intel_gtt_get(); /* Add register map (needed for suspend/resume) */ mmio_bar = IS_GEN2(dev) ? 1 : 0; base = drm_get_resource_start(dev, mmio_bar); size = drm_get_resource_len(dev, mmio_bar); ret = drm_addmap(dev, base, size, _DRM_REGISTERS, _DRM_KERNEL | _DRM_DRIVER, &dev_priv->mmio_map); if (ret != 0) { DRM_ERROR("Failed to allocate mmio_map: %d\n", ret); free(dev_priv, DRM_MEM_DRIVER); return (ret); } dev_priv->tq = taskqueue_create("915", M_WAITOK, taskqueue_thread_enqueue, &dev_priv->tq); taskqueue_start_threads(&dev_priv->tq, 1, PWAIT, "i915 taskq"); mtx_init(&dev_priv->gt_lock, "915gt", NULL, MTX_DEF); mtx_init(&dev_priv->error_lock, "915err", NULL, MTX_DEF); mtx_init(&dev_priv->error_completion_lock, "915cmp", NULL, MTX_DEF); mtx_init(&dev_priv->rps_lock, "915rps", NULL, MTX_DEF); mtx_init(&dev_priv->dpio_lock, "915dpi", NULL, MTX_DEF); intel_irq_init(dev); intel_setup_mchbar(dev); intel_setup_gmbus(dev); intel_opregion_setup(dev); intel_setup_bios(dev); i915_gem_load(dev); /* On the 945G/GM, the chipset reports the MSI capability on the * integrated graphics even though the support isn't actually there * according to the published specs. It doesn't appear to function * correctly in testing on 945G. * This may be a side effect of MSI having been made available for PEG * and the registers being closely associated. * * According to chipset errata, on the 965GM, MSI interrupts may * be lost or delayed, but we use them anyways to avoid * stuck interrupts on some machines. */ if (!IS_I945G(dev) && !IS_I945GM(dev)) drm_pci_enable_msi(dev); /* Init HWS */ if (!I915_NEED_GFX_HWS(dev)) { ret = i915_init_phys_hws(dev); if (ret != 0) { drm_rmmap(dev, dev_priv->mmio_map); free(dev_priv, DRM_MEM_DRIVER); return ret; } } mtx_init(&dev_priv->irq_lock, "userirq", NULL, MTX_DEF); if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) dev_priv->num_pipe = 3; else if (IS_MOBILE(dev) || !IS_GEN2(dev)) dev_priv->num_pipe = 2; else dev_priv->num_pipe = 1; ret = drm_vblank_init(dev, dev_priv->num_pipe); if (ret) goto out_gem_unload; /* Start out suspended */ dev_priv->mm.suspended = 1; intel_detect_pch(dev); if (drm_core_check_feature(dev, DRIVER_MODESET)) { ret = i915_load_modeset_init(dev); if (ret < 0) { DRM_ERROR("failed to init modeset\n"); goto out_gem_unload; } } intel_opregion_init(dev); callout_init(&dev_priv->hangcheck_timer, 1); callout_reset(&dev_priv->hangcheck_timer, DRM_I915_HANGCHECK_PERIOD, i915_hangcheck_elapsed, dev); if (IS_GEN5(dev)) intel_gpu_ips_init(dev_priv); return (0); out_gem_unload: /* XXXKIB */ (void) i915_driver_unload(dev); return (ret); } int i915_driver_unload(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; DRM_LOCK(dev); ret = i915_gpu_idle(dev); if (ret) DRM_ERROR("failed to idle hardware: %d\n", ret); i915_gem_retire_requests(dev); DRM_UNLOCK(dev); i915_free_hws(dev); intel_teardown_mchbar(dev); if (drm_core_check_feature(dev, DRIVER_MODESET)) { intel_fbdev_fini(dev); intel_modeset_cleanup(dev); } /* Free error state after interrupts are fully disabled. */ callout_stop(&dev_priv->hangcheck_timer); callout_drain(&dev_priv->hangcheck_timer); i915_destroy_error_state(dev); if (dev->msi_enabled) drm_pci_disable_msi(dev); intel_opregion_fini(dev); if (drm_core_check_feature(dev, DRIVER_MODESET)) { DRM_LOCK(dev); i915_gem_free_all_phys_object(dev); i915_gem_cleanup_ringbuffer(dev); i915_gem_context_fini(dev); DRM_UNLOCK(dev); i915_gem_cleanup_aliasing_ppgtt(dev); #if 1 KIB_NOTYET(); #else if (I915_HAS_FBC(dev) && i915_powersave) i915_cleanup_compression(dev); #endif drm_mm_takedown(&dev_priv->mm.stolen); intel_cleanup_overlay(dev); if (!I915_NEED_GFX_HWS(dev)) i915_free_hws(dev); } i915_gem_unload(dev); mtx_destroy(&dev_priv->irq_lock); if (dev_priv->tq != NULL) taskqueue_free(dev_priv->tq); bus_generic_detach(dev->dev); drm_rmmap(dev, dev_priv->mmio_map); intel_teardown_gmbus(dev); mtx_destroy(&dev_priv->dpio_lock); mtx_destroy(&dev_priv->error_lock); mtx_destroy(&dev_priv->error_completion_lock); mtx_destroy(&dev_priv->rps_lock); free(dev->dev_private, DRM_MEM_DRIVER); return 0; } int i915_driver_open(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *i915_file_priv; i915_file_priv = malloc(sizeof(*i915_file_priv), DRM_MEM_FILES, M_WAITOK | M_ZERO); mtx_init(&i915_file_priv->mm.lck, "915fp", NULL, MTX_DEF); INIT_LIST_HEAD(&i915_file_priv->mm.request_list); file->driver_priv = i915_file_priv; drm_gem_names_init(&i915_file_priv->context_idr); return 0; } /** * i915_driver_lastclose - clean up after all DRM clients have exited * @dev: DRM device * * Take care of cleaning up after all DRM clients have exited. In the * mode setting case, we want to restore the kernel's initial mode (just * in case the last client left us in a bad state). * * Additionally, in the non-mode setting case, we'll tear down the GTT * and DMA structures, since the kernel won't be using them, and clea * up any GEM state. */ void i915_driver_lastclose(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; /* On gen6+ we refuse to init without kms enabled, but then the drm core * goes right around and calls lastclose. Check for this and don't clean * up anything. */ if (!dev_priv) return; if (drm_core_check_feature(dev, DRIVER_MODESET)) { #if 1 KIB_NOTYET(); #else drm_fb_helper_restore(); vga_switcheroo_process_delayed_switch(); #endif return; } i915_gem_lastclose(dev); i915_dma_cleanup(dev); } void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv) { i915_gem_context_close(dev, file_priv); i915_gem_release(dev, file_priv); } void i915_driver_postclose(struct drm_device *dev, struct drm_file *file_priv) { struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; mtx_destroy(&i915_file_priv->mm.lck); free(i915_file_priv, DRM_MEM_FILES); } struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF(DRM_I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_FLUSH, i915_flush_ioctl, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_FLIP, i915_flip_bufs, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_BATCHBUFFER, i915_batchbuffer, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_IRQ_EMIT, i915_irq_emit, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_IRQ_WAIT, i915_irq_wait, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_GETPARAM, i915_getparam, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_ALLOC, drm_noop, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_FREE, drm_noop, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_CMDBUFFER, i915_cmdbuffer, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ), DRM_IOCTL_DEF(DRM_I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ), DRM_IOCTL_DEF(DRM_I915_GET_VBLANK_PIPE, i915_vblank_pipe_get, DRM_AUTH ), DRM_IOCTL_DEF(DRM_I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_HWS_ADDR, i915_set_status_page, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_GEM_INIT, i915_gem_init_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH | DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH | DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_PIN, i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_UNPIN, i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_ENTERVT, i915_gem_entervt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_LEAVEVT, i915_gem_leavevt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_CREATE, i915_gem_create_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GET_SPRITE_COLORKEY, intel_sprite_get_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_UNLOCKED), }; #ifdef COMPAT_FREEBSD32 extern struct drm_ioctl_desc i915_compat_ioctls[]; extern int i915_compat_ioctls_nr; #endif struct drm_driver i915_driver_info = { /* * FIXME Linux<->FreeBSD: DRIVER_USE_MTRR is commented out on * Linux. */ .driver_features = DRIVER_USE_AGP | DRIVER_REQUIRE_AGP | DRIVER_USE_MTRR | DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME, .buf_priv_size = sizeof(drm_i915_private_t), .load = i915_driver_load, .open = i915_driver_open, .unload = i915_driver_unload, .preclose = i915_driver_preclose, .lastclose = i915_driver_lastclose, .postclose = i915_driver_postclose, .device_is_agp = i915_driver_device_is_agp, .master_create = i915_master_create, .master_destroy = i915_master_destroy, .gem_init_object = i915_gem_init_object, .gem_free_object = i915_gem_free_object, .gem_pager_ops = &i915_gem_pager_ops, .dumb_create = i915_gem_dumb_create, .dumb_map_offset = i915_gem_mmap_gtt, .dumb_destroy = i915_gem_dumb_destroy, .sysctl_init = i915_sysctl_init, .sysctl_cleanup = i915_sysctl_cleanup, .ioctls = i915_ioctls, #ifdef COMPAT_FREEBSD32 .compat_ioctls = i915_compat_ioctls, .num_compat_ioctls = &i915_compat_ioctls_nr, #endif .num_ioctls = ARRAY_SIZE(i915_ioctls), .name = DRIVER_NAME, .desc = DRIVER_DESC, .date = DRIVER_DATE, .major = DRIVER_MAJOR, .minor = DRIVER_MINOR, .patchlevel = DRIVER_PATCHLEVEL, }; /* * This is really ugly: Because old userspace abused the linux agp interface to * manage the gtt, we need to claim that all intel devices are agp. For * otherwise the drm core refuses to initialize the agp support code. */ int i915_driver_device_is_agp(struct drm_device * dev) { return 1; } Index: head/sys/dev/drm2/i915/i915_drv.h =================================================================== --- head/sys/dev/drm2/i915/i915_drv.h (revision 287176) +++ head/sys/dev/drm2/i915/i915_drv.h (revision 287177) @@ -1,1576 +1,1573 @@ /* i915_drv.h -- Private header for the I915 driver -*- linux-c -*- */ /* * * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * */ #include __FBSDID("$FreeBSD$"); #ifndef _I915_DRV_H_ #define _I915_DRV_H_ #include #include #include #include #include /* General customization: */ #define DRIVER_AUTHOR "Tungsten Graphics, Inc." #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" #define DRIVER_DATE "20080730" MALLOC_DECLARE(DRM_I915_GEM); enum pipe { PIPE_A = 0, PIPE_B, PIPE_C, I915_MAX_PIPES }; #define pipe_name(p) ((p) + 'A') #define I915_NUM_PIPE 2 enum plane { PLANE_A = 0, PLANE_B, PLANE_C, }; #define plane_name(p) ((p) + 'A') enum port { PORT_A = 0, PORT_B, PORT_C, PORT_D, PORT_E, I915_MAX_PORTS }; #define port_name(p) ((p) + 'A') #define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) #define for_each_pipe(p) for ((p) = 0; (p) < dev_priv->num_pipe; (p)++) struct intel_pch_pll { int refcount; /* count of number of CRTCs sharing this PLL */ int active; /* count of number of active CRTCs (i.e. DPMS on) */ bool on; /* is the PLL actually active? Disabled during modeset */ int pll_reg; int fp0_reg; int fp1_reg; }; #define I915_NUM_PLLS 2 /* Interface history: * * 1.1: Original. * 1.2: Add Power Management * 1.3: Add vblank support * 1.4: Fix cmdbuffer path, add heap destroy * 1.5: Add vblank pipe configuration * 1.6: - New ioctl for scheduling buffer swaps on vertical blank * - Support vertical blank on secondary display pipe */ #define DRIVER_MAJOR 1 #define DRIVER_MINOR 6 #define DRIVER_PATCHLEVEL 0 #define WATCH_COHERENCY 0 #define WATCH_BUF 0 #define WATCH_EXEC 0 #define WATCH_LRU 0 #define WATCH_RELOC 0 #define WATCH_INACTIVE 0 #define WATCH_PWRITE 0 #define I915_GEM_PHYS_CURSOR_0 1 #define I915_GEM_PHYS_CURSOR_1 2 #define I915_GEM_PHYS_OVERLAY_REGS 3 #define I915_MAX_PHYS_OBJECT (I915_GEM_PHYS_OVERLAY_REGS) struct drm_i915_gem_phys_object { int id; drm_dma_handle_t *handle; struct drm_i915_gem_object *cur_obj; }; struct drm_i915_private; struct drm_i915_display_funcs { void (*dpms)(struct drm_crtc *crtc, int mode); bool (*fbc_enabled)(struct drm_device *dev); void (*enable_fbc)(struct drm_crtc *crtc, unsigned long interval); void (*disable_fbc)(struct drm_device *dev); int (*get_display_clock_speed)(struct drm_device *dev); int (*get_fifo_size)(struct drm_device *dev, int plane); void (*update_wm)(struct drm_device *dev); void (*update_sprite_wm)(struct drm_device *dev, int pipe, uint32_t sprite_width, int pixel_size); void (*sanitize_pm)(struct drm_device *dev); void (*update_linetime_wm)(struct drm_device *dev, int pipe, struct drm_display_mode *mode); int (*crtc_mode_set)(struct drm_crtc *crtc, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode, int x, int y, struct drm_framebuffer *old_fb); void (*off)(struct drm_crtc *crtc); void (*write_eld)(struct drm_connector *connector, struct drm_crtc *crtc); void (*fdi_link_train)(struct drm_crtc *crtc); void (*init_clock_gating)(struct drm_device *dev); void (*init_pch_clock_gating)(struct drm_device *dev); int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj); void (*force_wake_get)(struct drm_i915_private *dev_priv); void (*force_wake_put)(struct drm_i915_private *dev_priv); int (*update_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb, int x, int y); /* clock updates for mode set */ /* cursor updates */ /* render clock increase/decrease */ /* display clock increase/decrease */ /* pll clock increase/decrease */ }; struct intel_device_info { u8 gen; u8 not_supported:1; u8 is_mobile:1; u8 is_i85x:1; u8 is_i915g:1; u8 is_i945gm:1; u8 is_g33:1; u8 need_gfx_hws:1; u8 is_g4x:1; u8 is_pineview:1; u8 is_broadwater:1; u8 is_crestline:1; u8 is_ivybridge:1; u8 is_valleyview:1; u8 has_pch_split:1; u8 is_haswell:1; u8 has_fbc:1; u8 has_pipe_cxsr:1; u8 has_hotplug:1; u8 cursor_needs_physical:1; u8 has_overlay:1; u8 overlay_needs_physical:1; u8 supports_tv:1; u8 has_bsd_ring:1; u8 has_blt_ring:1; u8 has_llc:1; }; #define I915_PPGTT_PD_ENTRIES 512 #define I915_PPGTT_PT_ENTRIES 1024 struct i915_hw_ppgtt { unsigned num_pd_entries; vm_page_t *pt_pages; uint32_t pd_offset; vm_paddr_t *pt_dma_addr; vm_paddr_t scratch_page_dma_addr; }; /* This must match up with the value previously used for execbuf2.rsvd1. */ #define DEFAULT_CONTEXT_ID 0 struct i915_hw_context { uint32_t id; bool is_initialized; struct drm_i915_file_private *file_priv; struct intel_ring_buffer *ring; struct drm_i915_gem_object *obj; }; enum no_fbc_reason { FBC_NO_OUTPUT, /* no outputs enabled to compress */ FBC_STOLEN_TOO_SMALL, /* not enough space to hold compressed buffers */ FBC_UNSUPPORTED_MODE, /* interlace or doublescanned mode */ FBC_MODE_TOO_LARGE, /* mode too large for compression */ FBC_BAD_PLANE, /* fbc not supported on plane */ FBC_NOT_TILED, /* buffer not tiled */ FBC_MULTIPLE_PIPES, /* more than one pipe active */ FBC_MODULE_PARAM, }; struct mem_block { struct mem_block *next; struct mem_block *prev; int start; int size; struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */ }; struct opregion_header; struct opregion_acpi; struct opregion_swsci; struct opregion_asle; struct intel_opregion { struct opregion_header *header; struct opregion_acpi *acpi; struct opregion_swsci *swsci; struct opregion_asle *asle; void *vbt; u32 *lid_state; }; #define OPREGION_SIZE (8*1024) struct drm_i915_master_private { drm_local_map_t *sarea; struct _drm_i915_sarea *sarea_priv; }; #define I915_FENCE_REG_NONE -1 #define I915_MAX_NUM_FENCES 16 /* 16 fences + sign bit for FENCE_REG_NONE */ #define I915_MAX_NUM_FENCE_BITS 5 struct drm_i915_fence_reg { struct list_head lru_list; struct drm_i915_gem_object *obj; int pin_count; }; struct sdvo_device_mapping { u8 initialized; u8 dvo_port; u8 slave_addr; u8 dvo_wiring; u8 i2c_pin; u8 ddc_pin; }; enum intel_pch { PCH_IBX, /* Ibexpeak PCH */ PCH_CPT, /* Cougarpoint PCH */ PCH_LPT, /* Lynxpoint PCH */ }; #define QUIRK_PIPEA_FORCE (1<<0) #define QUIRK_LVDS_SSC_DISABLE (1<<1) #define QUIRK_INVERT_BRIGHTNESS (1<<2) struct intel_fbdev; struct intel_fbc_work; typedef struct drm_i915_private { struct drm_device *dev; device_t gmbus_bridge[GMBUS_NUM_PORTS + 1]; device_t bbbus_bridge[GMBUS_NUM_PORTS + 1]; device_t gmbus[GMBUS_NUM_PORTS + 1]; device_t bbbus[GMBUS_NUM_PORTS + 1]; /** gmbus_sx protects against concurrent usage of the single hw gmbus * controller on different i2c buses. */ struct sx gmbus_sx; uint32_t gpio_mmio_base; int relative_constants_mode; drm_local_map_t *mmio_map; /** gt_fifo_count and the subsequent register write are synchronized * with dev->struct_mutex. */ unsigned gt_fifo_count; /** forcewake_count is protected by gt_lock */ unsigned forcewake_count; /** gt_lock is also taken in irq contexts. */ struct mtx gt_lock; /* drm_i915_ring_buffer_t ring; */ struct intel_ring_buffer rings[I915_NUM_RINGS]; uint32_t next_seqno; drm_dma_handle_t *status_page_dmah; void *hw_status_page; dma_addr_t dma_status_page; uint32_t counter; unsigned int status_gfx_addr; struct drm_gem_object *hws_obj; struct drm_i915_gem_object *pwrctx; struct drm_i915_gem_object *renderctx; unsigned int cpp; int back_offset; int front_offset; int current_page; int page_flipping; atomic_t irq_received; u32 trace_irq_seqno; /** Cached value of IER to avoid reads in updating the bitfield */ u32 pipestat[2]; u32 irq_mask; u32 gt_irq_mask; u32 pch_irq_mask; struct mtx irq_lock; struct mtx dpio_lock; u32 hotplug_supported_mask; unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds; int num_pipe; int num_pch_pll; /* For hangcheck timer */ #define DRM_I915_HANGCHECK_PERIOD ((1500 /* in ms */ * hz) / 1000) int hangcheck_count; uint32_t last_acthd[I915_NUM_RINGS]; uint32_t last_instdone; uint32_t last_instdone1; unsigned int stop_rings; struct intel_opregion opregion; /* overlay */ struct intel_overlay *overlay; bool sprite_scaling_enabled; /* LVDS info */ int backlight_level; /* restore backlight to this value */ bool backlight_enabled; struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */ struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */ /* Feature bits from the VBIOS */ unsigned int int_tv_support:1; unsigned int lvds_dither:1; unsigned int lvds_vbt:1; unsigned int int_crt_support:1; unsigned int lvds_use_ssc:1; unsigned int display_clock_mode:1; int lvds_ssc_freq; unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */ unsigned int lvds_val; /* used for checking LVDS channel mode */ struct { int rate; int lanes; int preemphasis; int vswing; bool initialized; bool support; int bpp; struct edp_power_seq pps; } edp; bool no_aux_handshake; int crt_ddc_pin; struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */ int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */ int num_fence_regs; /* 8 on pre-965, 16 otherwise */ /* PCH chipset type */ enum intel_pch pch_type; /* Display functions */ struct drm_i915_display_funcs display; unsigned long quirks; /* Register state */ bool modeset_on_lid; u8 saveLBB; u32 saveDSPACNTR; u32 saveDSPBCNTR; u32 saveDSPARB; u32 saveHWS; u32 savePIPEACONF; u32 savePIPEBCONF; u32 savePIPEASRC; u32 savePIPEBSRC; u32 saveFPA0; u32 saveFPA1; u32 saveDPLL_A; u32 saveDPLL_A_MD; u32 saveHTOTAL_A; u32 saveHBLANK_A; u32 saveHSYNC_A; u32 saveVTOTAL_A; u32 saveVBLANK_A; u32 saveVSYNC_A; u32 saveBCLRPAT_A; u32 saveTRANSACONF; u32 saveTRANS_HTOTAL_A; u32 saveTRANS_HBLANK_A; u32 saveTRANS_HSYNC_A; u32 saveTRANS_VTOTAL_A; u32 saveTRANS_VBLANK_A; u32 saveTRANS_VSYNC_A; u32 savePIPEASTAT; u32 saveDSPASTRIDE; u32 saveDSPASIZE; u32 saveDSPAPOS; u32 saveDSPAADDR; u32 saveDSPASURF; u32 saveDSPATILEOFF; u32 savePFIT_PGM_RATIOS; u32 saveBLC_HIST_CTL; u32 saveBLC_PWM_CTL; u32 saveBLC_PWM_CTL2; u32 saveBLC_CPU_PWM_CTL; u32 saveBLC_CPU_PWM_CTL2; u32 saveFPB0; u32 saveFPB1; u32 saveDPLL_B; u32 saveDPLL_B_MD; u32 saveHTOTAL_B; u32 saveHBLANK_B; u32 saveHSYNC_B; u32 saveVTOTAL_B; u32 saveVBLANK_B; u32 saveVSYNC_B; u32 saveBCLRPAT_B; u32 saveTRANSBCONF; u32 saveTRANS_HTOTAL_B; u32 saveTRANS_HBLANK_B; u32 saveTRANS_HSYNC_B; u32 saveTRANS_VTOTAL_B; u32 saveTRANS_VBLANK_B; u32 saveTRANS_VSYNC_B; u32 savePIPEBSTAT; u32 saveDSPBSTRIDE; u32 saveDSPBSIZE; u32 saveDSPBPOS; u32 saveDSPBADDR; u32 saveDSPBSURF; u32 saveDSPBTILEOFF; u32 saveVGA0; u32 saveVGA1; u32 saveVGA_PD; u32 saveVGACNTRL; u32 saveADPA; u32 saveLVDS; u32 savePP_ON_DELAYS; u32 savePP_OFF_DELAYS; u32 saveDVOA; u32 saveDVOB; u32 saveDVOC; u32 savePP_ON; u32 savePP_OFF; u32 savePP_CONTROL; u32 savePP_DIVISOR; u32 savePFIT_CONTROL; u32 save_palette_a[256]; u32 save_palette_b[256]; u32 saveDPFC_CB_BASE; u32 saveFBC_CFB_BASE; u32 saveFBC_LL_BASE; u32 saveFBC_CONTROL; u32 saveFBC_CONTROL2; u32 saveIER; u32 saveIIR; u32 saveIMR; u32 saveDEIER; u32 saveDEIMR; u32 saveGTIER; u32 saveGTIMR; u32 saveFDI_RXA_IMR; u32 saveFDI_RXB_IMR; u32 saveCACHE_MODE_0; u32 saveMI_ARB_STATE; u32 saveSWF0[16]; u32 saveSWF1[16]; u32 saveSWF2[3]; u8 saveMSR; u8 saveSR[8]; u8 saveGR[25]; u8 saveAR_INDEX; u8 saveAR[21]; u8 saveDACMASK; u8 saveCR[37]; uint64_t saveFENCE[I915_MAX_NUM_FENCES]; u32 saveCURACNTR; u32 saveCURAPOS; u32 saveCURABASE; u32 saveCURBCNTR; u32 saveCURBPOS; u32 saveCURBBASE; u32 saveCURSIZE; u32 saveDP_B; u32 saveDP_C; u32 saveDP_D; u32 savePIPEA_GMCH_DATA_M; u32 savePIPEB_GMCH_DATA_M; u32 savePIPEA_GMCH_DATA_N; u32 savePIPEB_GMCH_DATA_N; u32 savePIPEA_DP_LINK_M; u32 savePIPEB_DP_LINK_M; u32 savePIPEA_DP_LINK_N; u32 savePIPEB_DP_LINK_N; u32 saveFDI_RXA_CTL; u32 saveFDI_TXA_CTL; u32 saveFDI_RXB_CTL; u32 saveFDI_TXB_CTL; u32 savePFA_CTL_1; u32 savePFB_CTL_1; u32 savePFA_WIN_SZ; u32 savePFB_WIN_SZ; u32 savePFA_WIN_POS; u32 savePFB_WIN_POS; u32 savePCH_DREF_CONTROL; u32 saveDISP_ARB_CTL; u32 savePIPEA_DATA_M1; u32 savePIPEA_DATA_N1; u32 savePIPEA_LINK_M1; u32 savePIPEA_LINK_N1; u32 savePIPEB_DATA_M1; u32 savePIPEB_DATA_N1; u32 savePIPEB_LINK_M1; u32 savePIPEB_LINK_N1; u32 saveMCHBAR_RENDER_STANDBY; u32 savePCH_PORT_HOTPLUG; struct { /** Memory allocator for GTT stolen memory */ struct drm_mm stolen; /** Memory allocator for GTT */ struct drm_mm gtt_space; /** List of all objects in gtt_space. Used to restore gtt * mappings on resume */ struct list_head gtt_list; /** Usable portion of the GTT for GEM */ unsigned long gtt_start; unsigned long gtt_mappable_end; unsigned long gtt_end; /** PPGTT used for aliasing the PPGTT with the GTT */ struct i915_hw_ppgtt *aliasing_ppgtt; /** * List of objects currently involved in rendering from the * ringbuffer. * * Includes buffers having the contents of their GPU caches * flushed, not necessarily primitives. last_rendering_seqno * represents when the rendering involved will be completed. * * A reference is held on the buffer while on this list. */ struct list_head active_list; /** * List of objects which are not in the ringbuffer but which * still have a write_domain which needs to be flushed before * unbinding. * * A reference is held on the buffer while on this list. */ struct list_head flushing_list; /** * LRU list of objects which are not in the ringbuffer and * are ready to unbind, but are still in the GTT. * * last_rendering_seqno is 0 while an object is in this list. * * A reference is not held on the buffer while on this list, * as merely being GTT-bound shouldn't prevent its being * freed, and we'll pull it off the list in the free path. */ struct list_head inactive_list; /** LRU list of objects with fence regs on them. */ struct list_head fence_list; /** * We leave the user IRQ off as much as possible, * but this means that requests will finish and never * be retired once the system goes idle. Set a timer to * fire periodically while the ring is running. When it * fires, go retire requests. */ struct timeout_task retire_task; /** * Are we in a non-interruptible section of code like * modesetting? */ bool interruptible; uint32_t next_gem_seqno; /** * Waiting sequence number, if any */ uint32_t waiting_gem_seqno; /** * Last seq seen at irq time */ uint32_t irq_gem_seqno; /** * Flag if the X Server, and thus DRM, is not currently in * control of the device. * * This is set between LeaveVT and EnterVT. It needs to be * replaced with a semaphore. It also needs to be * transitioned away from for kernel modesetting. */ int suspended; /** * Flag if the hardware appears to be wedged. * * This is set when attempts to idle the device timeout. * It prevents command submission from occuring and makes * every pending request fail */ int wedged; /** Bit 6 swizzling required for X tiling */ uint32_t bit_6_swizzle_x; /** Bit 6 swizzling required for Y tiling */ uint32_t bit_6_swizzle_y; /* storage for physical objects */ struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT]; /* accounting, useful for userland debugging */ size_t gtt_total; size_t mappable_gtt_total; size_t object_memory; u32 object_count; struct intel_gtt gtt; eventhandler_tag i915_lowmem; } mm; const struct intel_device_info *info; /* Old dri1 support infrastructure, beware the dragons ya fools entering * here! */ struct { unsigned allow_batchbuffer : 1; u32 *gfx_hws_cpu_addr; } dri1; /* Kernel Modesetting */ struct sdvo_device_mapping sdvo_mappings[2]; /* indicate whether the LVDS_BORDER should be enabled or not */ unsigned int lvds_border_bits; /* Panel fitter placement and size for Ironlake+ */ u32 pch_pf_pos, pch_pf_size; struct drm_crtc *plane_to_crtc_mapping[3]; struct drm_crtc *pipe_to_crtc_mapping[3]; /* wait_queue_head_t pending_flip_queue; XXXKIB */ struct intel_pch_pll pch_plls[I915_NUM_PLLS]; /* Reclocking support */ bool render_reclock_avail; bool lvds_downclock_avail; /* indicates the reduced downclock for LVDS*/ int lvds_downclock; struct task idle_task; struct callout idle_callout; bool busy; u16 orig_clock; int child_dev_num; struct child_device_config *child_dev; struct drm_connector *int_lvds_connector; struct drm_connector *int_edp_connector; device_t bridge_dev; bool mchbar_need_disable; int mch_res_rid; struct resource *mch_res; struct mtx rps_lock; u32 pm_iir; struct task rps_task; u8 cur_delay; u8 min_delay; u8 max_delay; u8 fmax; u8 fstart; u64 last_count1; unsigned long last_time1; unsigned long chipset_power; u64 last_count2; struct timespec last_time2; unsigned long gfx_power; int c_m; int r_t; u8 corr; struct mtx *mchdev_lock; enum no_fbc_reason no_fbc_reason; struct drm_mm_node *compressed_fb; struct drm_mm_node *compressed_llb; unsigned long cfb_size; unsigned int cfb_fb; int cfb_plane; int cfb_y; struct intel_fbc_work *fbc_work; unsigned int fsb_freq, mem_freq, is_ddr3; struct taskqueue *tq; struct task error_task; struct task hotplug_task; int error_completion; struct mtx error_completion_lock; /* Protected by dev->error_lock. */ struct drm_i915_error_state *first_error; struct mtx error_lock; struct callout hangcheck_timer; unsigned long last_gpu_reset; struct intel_fbdev *fbdev; struct drm_property *broadcast_rgb_property; struct drm_property *force_audio_property; bool hw_contexts_disabled; uint32_t hw_context_size; } drm_i915_private_t; /* Iterate over initialised rings */ #define for_each_ring(ring__, dev_priv__, i__) \ for ((i__) = 0; (i__) < I915_NUM_RINGS; (i__)++) \ if (((ring__) = &(dev_priv__)->rings[(i__)]), intel_ring_initialized((ring__))) enum hdmi_force_audio { HDMI_AUDIO_OFF_DVI = -2, /* no aux data for HDMI-DVI converter */ HDMI_AUDIO_OFF, /* force turn off HDMI audio */ HDMI_AUDIO_AUTO, /* trust EDID */ HDMI_AUDIO_ON, /* force turn on HDMI audio */ }; enum i915_cache_level { I915_CACHE_NONE, I915_CACHE_LLC, I915_CACHE_LLC_MLC, /* gen6+ */ }; enum intel_chip_family { CHIP_I8XX = 0x01, CHIP_I9XX = 0x02, CHIP_I915 = 0x04, CHIP_I965 = 0x08, }; /** driver private structure attached to each drm_gem_object */ struct drm_i915_gem_object { struct drm_gem_object base; /** Current space allocated to this object in the GTT, if any. */ struct drm_mm_node *gtt_space; struct list_head gtt_list; /** This object's place on the active/flushing/inactive lists */ struct list_head ring_list; struct list_head mm_list; /** This object's place on GPU write list */ struct list_head gpu_write_list; /** This object's place in the batchbuffer or on the eviction list */ struct list_head exec_list; /** * This is set if the object is on the active or flushing lists * (has pending rendering), and is not set if it's on inactive (ready * to be unbound). */ unsigned int active:1; /** * This is set if the object has been written to since last bound * to the GTT */ unsigned int dirty:1; /** * This is set if the object has been written to since the last * GPU flush. */ unsigned int pending_gpu_write:1; /** * Fence register bits (if any) for this object. Will be set * as needed when mapped into the GTT. * Protected by dev->struct_mutex. */ signed int fence_reg:I915_MAX_NUM_FENCE_BITS; /** * Advice: are the backing pages purgeable? */ unsigned int madv:2; /** * Current tiling mode for the object. */ unsigned int tiling_mode:2; /** * Whether the tiling parameters for the currently associated fence * register have changed. Note that for the purposes of tracking * tiling changes we also treat the unfenced register, the register * slot that the object occupies whilst it executes a fenced * command (such as BLT on gen2/3), as a "fence". */ unsigned int fence_dirty:1; /** How many users have pinned this object in GTT space. The following * users can each hold at most one reference: pwrite/pread, pin_ioctl * (via user_pin_count), execbuffer (objects are not allowed multiple * times for the same batchbuffer), and the framebuffer code. When * switching/pageflipping, the framebuffer code has at most two buffers * pinned per crtc. * * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3 * bits with absolutely no headroom. So use 4 bits. */ unsigned int pin_count:4; #define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf /** * Is the object at the current location in the gtt mappable and * fenceable? Used to avoid costly recalculations. */ unsigned int map_and_fenceable:1; /** * Whether the current gtt mapping needs to be mappable (and isn't just * mappable by accident). Track pin and fault separate for a more * accurate mappable working set. */ unsigned int fault_mappable:1; unsigned int pin_mappable:1; unsigned int pin_display:1; /* * Is the GPU currently using a fence to access this buffer, */ unsigned int pending_fenced_gpu_access:1; unsigned int fenced_gpu_access:1; unsigned int cache_level:2; unsigned int has_aliasing_ppgtt_mapping:1; unsigned int has_global_gtt_mapping:1; vm_page_t *pages; int pages_pin_count; /** * DMAR support */ struct sglist *sg_list; /** * Used for performing relocations during execbuffer insertion. */ LIST_ENTRY(drm_i915_gem_object) exec_node; unsigned long exec_handle; struct drm_i915_gem_exec_object2 *exec_entry; /** * Current offset of the object in GTT space. * * This is the same as gtt_space->start */ uint32_t gtt_offset; struct intel_ring_buffer *ring; /** Breadcrumb of last rendering to the buffer. */ uint32_t last_rendering_seqno; /** Breadcrumb of last fenced GPU access to the buffer. */ uint32_t last_fenced_seqno; /** Current tiling stride for the object, if it's tiled. */ uint32_t stride; /** Record of address bit 17 of each page at last unbind. */ unsigned long *bit_17; /** User space pin count and filp owning the pin */ uint32_t user_pin_count; struct drm_file *pin_filp; /** for phy allocated objects */ struct drm_i915_gem_phys_object *phys_obj; /** * Number of crtcs where this object is currently the fb, but * will be page flipped away on the next vblank. When it * reaches 0, dev_priv->pending_flip_queue will be woken up. */ int pending_flip; }; #define to_intel_bo(x) __containerof(x, struct drm_i915_gem_object, base) /** * Request queue structure. * * The request queue allows us to note sequence numbers that have been emitted * and may be associated with active buffers to be retired. * * By keeping this list, we can avoid having to do questionable * sequence-number comparisons on buffer last_rendering_seqnos, and associate * an emission time with seqnos for tracking how far ahead of the GPU we are. */ struct drm_i915_gem_request { /** On Which ring this request was generated */ struct intel_ring_buffer *ring; /** GEM sequence number associated with this request. */ uint32_t seqno; /** Postion in the ringbuffer of the end of the request */ u32 tail; /** Time at which this request was emitted, in jiffies. */ unsigned long emitted_jiffies; /** global list entry for this request */ struct list_head list; struct drm_i915_file_private *file_priv; /** file_priv list entry for this request */ struct list_head client_list; }; struct drm_i915_file_private { struct { struct list_head request_list; struct mtx lck; } mm; struct drm_gem_names context_idr; }; struct drm_i915_error_state { u_int ref; u32 eir; u32 pgtbl_er; u32 ier; bool waiting[I915_NUM_RINGS]; u32 pipestat[I915_MAX_PIPES]; u32 tail[I915_NUM_RINGS]; u32 head[I915_NUM_RINGS]; u32 ipeir[I915_NUM_RINGS]; u32 ipehr[I915_NUM_RINGS]; u32 instdone[I915_NUM_RINGS]; u32 acthd[I915_NUM_RINGS]; u32 semaphore_mboxes[I915_NUM_RINGS][I915_NUM_RINGS - 1]; /* our own tracking of ring head and tail */ u32 cpu_ring_head[I915_NUM_RINGS]; u32 cpu_ring_tail[I915_NUM_RINGS]; u32 error; /* gen6+ */ u32 instpm[I915_NUM_RINGS]; u32 instps[I915_NUM_RINGS]; u32 instdone1; u32 seqno[I915_NUM_RINGS]; u64 bbaddr; u32 fault_reg[I915_NUM_RINGS]; u32 done_reg; u32 faddr[I915_NUM_RINGS]; u64 fence[I915_MAX_NUM_FENCES]; struct timeval time; struct drm_i915_error_ring { struct drm_i915_error_object { int page_count; u32 gtt_offset; u32 *pages[0]; } *ringbuffer, *batchbuffer; struct drm_i915_error_request { long jiffies; u32 seqno; u32 tail; } *requests; int num_requests; } ring[I915_NUM_RINGS]; struct drm_i915_error_buffer { u32 size; u32 name; u32 seqno; u32 gtt_offset; u32 read_domains; u32 write_domain; s32 fence_reg:I915_MAX_NUM_FENCE_BITS; s32 pinned:2; u32 tiling:2; u32 dirty:1; u32 purgeable:1; s32 ring:4; u32 cache_level:2; } *active_bo, *pinned_bo; u32 active_bo_count, pinned_bo_count; struct intel_overlay_error_state *overlay; struct intel_display_error_state *display; }; /** * RC6 is a special power stage which allows the GPU to enter an very * low-voltage mode when idle, using down to 0V while at this stage. This * stage is entered automatically when the GPU is idle when RC6 support is * enabled, and as soon as new workload arises GPU wakes up automatically as well. * * There are different RC6 modes available in Intel GPU, which differentiate * among each other with the latency required to enter and leave RC6 and * voltage consumed by the GPU in different states. * * The combination of the following flags define which states GPU is allowed * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and * RC6pp is deepest RC6. Their support by hardware varies according to the * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one * which brings the most power savings; deeper states save more power, but * require higher latency to switch to and wake up. */ #define INTEL_RC6_ENABLE (1<<0) #define INTEL_RC6p_ENABLE (1<<1) #define INTEL_RC6pp_ENABLE (1<<2) extern int intel_iommu_enabled; extern struct drm_ioctl_desc i915_ioctls[]; extern struct drm_driver i915_driver_info; extern struct cdev_pager_ops i915_gem_pager_ops; extern unsigned int i915_fbpercrtc; extern int i915_panel_ignore_lid; extern int i915_panel_invert_brightness; extern unsigned int i915_powersave; extern int i915_prefault_disable; extern int i915_semaphores; extern unsigned int i915_lvds_downclock; extern int i915_lvds_channel_mode; extern int i915_panel_use_ssc; extern int i915_vbt_sdvo_panel_type; extern int i915_enable_rc6; extern int i915_enable_fbc; extern int i915_enable_ppgtt; extern int i915_enable_hangcheck; const struct intel_device_info *i915_get_device_id(int device); int i915_reset(struct drm_device *dev); extern int intel_gpu_reset(struct drm_device *dev); /* i915_debug.c */ int i915_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx, struct sysctl_oid *top); void i915_sysctl_cleanup(struct drm_device *dev); extern int i915_master_create(struct drm_device *dev, struct drm_master *master); extern void i915_master_destroy(struct drm_device *dev, struct drm_master *master); /* i915_dma.c */ int i915_batchbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv); void i915_update_dri1_breadcrumb(struct drm_device *dev); extern void i915_kernel_lost_context(struct drm_device * dev); extern int i915_driver_load(struct drm_device *, unsigned long flags); extern int i915_driver_unload(struct drm_device *); extern int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv); extern void i915_driver_lastclose(struct drm_device * dev); extern void i915_driver_preclose(struct drm_device *dev, struct drm_file *file_priv); extern void i915_driver_postclose(struct drm_device *dev, struct drm_file *file_priv); extern int i915_driver_device_is_agp(struct drm_device * dev); extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); extern int i915_emit_box(struct drm_device *dev, - struct drm_clip_rect __user *boxes, - int i, int DR1, int DR4); -int i915_emit_box_p(struct drm_device *dev, struct drm_clip_rect *box, - int DR1, int DR4); - + struct drm_clip_rect *box, + int DR1, int DR4); unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); unsigned long i915_mch_val(struct drm_i915_private *dev_priv); void i915_update_gfx_val(struct drm_i915_private *dev_priv); unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); unsigned long i915_read_mch_val(void); bool i915_gpu_raise(void); bool i915_gpu_lower(void); bool i915_gpu_busy(void); bool i915_gpu_turbo_disable(void); /* i915_irq.c */ extern int i915_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv); extern void intel_irq_init(struct drm_device *dev); void intel_enable_asle(struct drm_device *dev); void i915_hangcheck_elapsed(void *context); void i915_handle_error(struct drm_device *dev, bool wedged); void i915_error_state_free(struct drm_i915_error_state *error); void i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask); void i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask); void i915_destroy_error_state(struct drm_device *dev); /* i915_gem.c */ int i915_gem_init_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_execbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_execbuffer2(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_pin_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_unpin_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_entervt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_set_tiling(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); void i915_gem_load(struct drm_device *dev); void i915_gem_unload(struct drm_device *dev); int i915_gem_init_object(struct drm_gem_object *obj); void i915_gem_free_object(struct drm_gem_object *obj); int i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, bool map_and_fenceable); void i915_gem_object_unpin(struct drm_i915_gem_object *obj); int i915_gem_object_unbind(struct drm_i915_gem_object *obj); void i915_gem_lastclose(struct drm_device *dev); uint32_t i915_get_gem_seqno(struct drm_device *dev); static inline bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) { if (obj->fence_reg != I915_FENCE_REG_NONE) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; dev_priv->fence_regs[obj->fence_reg].pin_count++; return true; } else return false; } static inline void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) { if (obj->fence_reg != I915_FENCE_REG_NONE) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; dev_priv->fence_regs[obj->fence_reg].pin_count--; } } void i915_gem_retire_requests(struct drm_device *dev); void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring); void i915_gem_clflush_object(struct drm_i915_gem_object *obj); struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, size_t size); uint32_t i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, int tiling_mode); int i915_mutex_lock_interruptible(struct drm_device *dev); int i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, struct intel_ring_buffer *pipelined); void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj); int i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj); int i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, uint32_t flush_domains); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); int i915_gem_object_sync(struct drm_i915_gem_object *obj, struct intel_ring_buffer *to); int i915_gem_object_put_fence(struct drm_i915_gem_object *obj); int i915_gem_idle(struct drm_device *dev); int i915_gem_init(struct drm_device *dev); int i915_gem_init_hw(struct drm_device *dev); void i915_gem_init_swizzling(struct drm_device *dev); void i915_gem_init_ppgtt(struct drm_device *dev); void i915_gem_cleanup_ringbuffer(struct drm_device *dev); int i915_gpu_idle(struct drm_device *dev); void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, struct intel_ring_buffer *ring, uint32_t seqno); int i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, struct drm_i915_gem_request *request); int i915_gem_object_get_fence(struct drm_i915_gem_object *obj); void i915_gem_reset(struct drm_device *dev); int i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno); int i915_gem_mmap(struct drm_device *dev, uint64_t offset, int prot); int i915_gem_fault(struct drm_device *dev, uint64_t offset, int prot, uint64_t *phys); void i915_gem_release(struct drm_device *dev, struct drm_file *file); int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level); /* i915_gem_context.c */ void i915_gem_context_init(struct drm_device *dev); void i915_gem_context_fini(struct drm_device *dev); void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); int i915_switch_context(struct intel_ring_buffer *ring, struct drm_file *file, int to_id); int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); void i915_gem_free_all_phys_object(struct drm_device *dev); void i915_gem_detach_phys_object(struct drm_device *dev, struct drm_i915_gem_object *obj); int i915_gem_attach_phys_object(struct drm_device *dev, struct drm_i915_gem_object *obj, int id, int align); int i915_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev, uint32_t handle, uint64_t *offset); int i915_gem_dumb_destroy(struct drm_file *file_priv, struct drm_device *dev, uint32_t handle); /* i915_gem_tiling.c */ void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); void i915_gem_object_do_bit_17_swizzle_page(struct drm_i915_gem_object *obj, struct vm_page *m); /* i915_gem_evict.c */ int i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment, bool mappable); int i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only); /* i915_gem_stolen.c */ int i915_gem_init_stolen(struct drm_device *dev); void i915_gem_cleanup_stolen(struct drm_device *dev); /* i915_suspend.c */ extern int i915_save_state(struct drm_device *dev); extern int i915_restore_state(struct drm_device *dev); /* intel_iic.c */ extern int intel_setup_gmbus(struct drm_device *dev); extern void intel_teardown_gmbus(struct drm_device *dev); extern void intel_gmbus_set_speed(device_t idev, int speed); extern void intel_gmbus_force_bit(device_t idev, bool force_bit); extern void intel_iic_reset(struct drm_device *dev); static inline bool intel_gmbus_is_port_valid(unsigned port) { return (port >= GMBUS_PORT_SSC && port <= GMBUS_PORT_DPD); } extern device_t intel_gmbus_get_adapter(struct drm_i915_private *dev_priv, unsigned port); /* intel_opregion.c */ int intel_opregion_setup(struct drm_device *dev); extern void intel_opregion_init(struct drm_device *dev); extern void intel_opregion_fini(struct drm_device *dev); extern void intel_opregion_asle_intr(struct drm_device *dev); extern void intel_opregion_gse_intr(struct drm_device *dev); extern void intel_opregion_enable_asle(struct drm_device *dev); /* i915_gem_gtt.c */ int i915_gem_init_aliasing_ppgtt(struct drm_device *dev); void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev); void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_object *obj, enum i915_cache_level cache_level); void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_object *obj); void i915_gem_restore_gtt_mappings(struct drm_device *dev); int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj); void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level); void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj); void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj); int i915_gem_init_global_gtt(struct drm_device *dev, unsigned long start, unsigned long mappable_end, unsigned long end); /* modesetting */ extern void intel_modeset_init_hw(struct drm_device *dev); extern void intel_modeset_init(struct drm_device *dev); extern void intel_modeset_gem_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state); extern void intel_disable_fbc(struct drm_device *dev); extern bool ironlake_set_drps(struct drm_device *dev, u8 val); extern void ironlake_init_pch_refclk(struct drm_device *dev); extern void ironlake_enable_rc6(struct drm_device *dev); extern void gen6_set_rps(struct drm_device *dev, u8 val); extern void intel_detect_pch(struct drm_device *dev); extern int intel_trans_dp_port_sel(struct drm_crtc *crtc); /* IPS */ extern void intel_gpu_ips_init(struct drm_i915_private *dev_priv); extern void intel_gpu_ips_teardown(void); extern bool i915_semaphore_is_enabled(struct drm_device *dev); extern void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv); extern void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv); extern void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv); extern void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv); extern void vlv_force_wake_get(struct drm_i915_private *dev_priv); extern void vlv_force_wake_put(struct drm_i915_private *dev_priv); extern struct intel_overlay_error_state *intel_overlay_capture_error_state( struct drm_device *dev); extern void intel_overlay_print_error_state(struct sbuf *m, struct intel_overlay_error_state *error); extern struct intel_display_error_state *intel_display_capture_error_state( struct drm_device *dev); extern void intel_display_print_error_state(struct sbuf *m, struct drm_device *dev, struct intel_display_error_state *error); static inline void trace_i915_reg_rw(boolean_t rw, int reg, uint64_t val, int sz) { CTR4(KTR_DRM_REG, "[%x/%d] %c %x", reg, sz, rw ? "w" : "r", val); } /* On SNB platform, before reading ring registers forcewake bit * must be set to prevent GT core from power down and stale values being * returned. */ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv); void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv); int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv); #define __i915_read(x, y) \ u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg); __i915_read(8, 8) __i915_read(16, 16) __i915_read(32, 32) __i915_read(64, 64) #undef __i915_read #define __i915_write(x, y) \ void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val); __i915_write(8, 8) __i915_write(16, 16) __i915_write(32, 32) __i915_write(64, 64) #undef __i915_write #define I915_READ8(reg) i915_read8(dev_priv, (reg)) #define I915_WRITE8(reg, val) i915_write8(dev_priv, (reg), (val)) #define I915_READ16(reg) i915_read16(dev_priv, (reg)) #define I915_WRITE16(reg, val) i915_write16(dev_priv, (reg), (val)) #define I915_READ16_NOTRACE(reg) DRM_READ16(dev_priv->mmio_map, (reg)) #define I915_WRITE16_NOTRACE(reg, val) DRM_WRITE16(dev_priv->mmio_map, (reg), (val)) #define I915_READ(reg) i915_read32(dev_priv, (reg)) #define I915_WRITE(reg, val) i915_write32(dev_priv, (reg), (val)) #define I915_READ_NOTRACE(reg) DRM_READ32(dev_priv->mmio_map, (reg)) #define I915_WRITE_NOTRACE(reg, val) DRM_WRITE32(dev_priv->mmio_map, (reg), (val)) #define I915_WRITE64(reg, val) i915_write64(dev_priv, (reg), (val)) #define I915_READ64(reg) i915_read64(dev_priv, (reg)) #define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg) #define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg) #define I915_VERBOSE 0 /** * Reads a dword out of the status page, which is written to from the command * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or * MI_STORE_DATA_IMM. * * The following dwords have a reserved meaning: * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes. * 0x04: ring 0 head pointer * 0x05: ring 1 head pointer (915-class) * 0x06: ring 2 head pointer (915-class) * 0x10-0x1b: Context status DWords (GM45) * 0x1f: Last written status offset. (GM45) * * The area from dword 0x20 to 0x3ff is available for driver usage. */ #define I915_GEM_HWS_INDEX 0x20 #define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info) #define IS_I830(dev) ((dev)->pci_device == 0x3577) #define IS_845G(dev) ((dev)->pci_device == 0x2562) #define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) #define IS_I865G(dev) ((dev)->pci_device == 0x2572) #define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g) #define IS_I915GM(dev) ((dev)->pci_device == 0x2592) #define IS_I945G(dev) ((dev)->pci_device == 0x2772) #define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm) #define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater) #define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline) #define IS_GM45(dev) ((dev)->pci_device == 0x2A42) #define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x) #define IS_PINEVIEW_G(dev) ((dev)->pci_device == 0xa001) #define IS_PINEVIEW_M(dev) ((dev)->pci_device == 0xa011) #define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview) #define IS_G33(dev) (INTEL_INFO(dev)->is_g33) #define IS_IRONLAKE_D(dev) ((dev)->pci_device == 0x0042) #define IS_IRONLAKE_M(dev) ((dev)->pci_device == 0x0046) #define IS_IVYBRIDGE(dev) (INTEL_INFO(dev)->is_ivybridge) #define IS_VALLEYVIEW(dev) (INTEL_INFO(dev)->is_valleyview) #define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell) #define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) /* XXXKIB LEGACY */ #define IS_I965G(dev) ((dev)->pci_device == 0x2972 || \ (dev)->pci_device == 0x2982 || \ (dev)->pci_device == 0x2992 || \ (dev)->pci_device == 0x29A2 || \ (dev)->pci_device == 0x2A02 || \ (dev)->pci_device == 0x2A12 || \ (dev)->pci_device == 0x2A42 || \ (dev)->pci_device == 0x2E02 || \ (dev)->pci_device == 0x2E12 || \ (dev)->pci_device == 0x2E22 || \ (dev)->pci_device == 0x2E32) #define IS_I965GM(dev) ((dev)->pci_device == 0x2A02) #define IS_IGDG(dev) ((dev)->pci_device == 0xa001) #define IS_IGDGM(dev) ((dev)->pci_device == 0xa011) #define IS_IGD(dev) (IS_IGDG(dev) || IS_IGDGM(dev)) #define IS_I9XX(dev) (IS_I915G(dev) || IS_I915GM(dev) || IS_I945G(dev) || \ IS_I945GM(dev) || IS_I965G(dev) || IS_G33(dev)) /* XXXKIB LEGACY END */ #define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2) #define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3) #define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4) #define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5) #define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6) #define IS_GEN7(dev) (INTEL_INFO(dev)->gen == 7) #define HAS_BSD(dev) (INTEL_INFO(dev)->has_bsd_ring) #define HAS_BLT(dev) (INTEL_INFO(dev)->has_blt_ring) #define HAS_LLC(dev) (INTEL_INFO(dev)->has_llc) #define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws) #define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 6) #define HAS_ALIASING_PPGTT(dev) (INTEL_INFO(dev)->gen >=6) #define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) #define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte * rows, which changed the alignment requirements and fence programming. */ #define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \ IS_I915GM(dev))) #define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) #define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev)) #define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev)) #define SUPPORTS_EDP(dev) (IS_IRONLAKE_M(dev)) #define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) #define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) /* dsparb controlled by hw only */ #define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev)) #define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2) #define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr) #define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc) #define HAS_PCH_SPLIT(dev) (INTEL_INFO(dev)->has_pch_split) #define HAS_PIPE_CONTROL(dev) (INTEL_INFO(dev)->gen >= 5) #define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type) #define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT) #define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT) #define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX) #define PRIMARY_RINGBUFFER_SIZE (128*1024) static inline bool i915_seqno_passed(uint32_t seq1, uint32_t seq2) { return ((int32_t)(seq1 - seq2) >= 0); } static inline void i915_gem_chipset_flush(struct drm_device *dev) { if (INTEL_INFO(dev)->gen < 6) intel_gtt_chipset_flush(); } static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { /* KASSERT(obj->pages != NULL, ("pin and NULL pages")); */ obj->pages_pin_count++; } static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) { KASSERT(obj->pages_pin_count != 0, ("zero pages_pin_count")); obj->pages_pin_count--; } u32 i915_gem_next_request_seqno(struct intel_ring_buffer *ring); #endif Index: head/sys/dev/drm2/i915/i915_gem_execbuffer.c =================================================================== --- head/sys/dev/drm2/i915/i915_gem_execbuffer.c (revision 287176) +++ head/sys/dev/drm2/i915/i915_gem_execbuffer.c (revision 287177) @@ -1,1547 +1,1547 @@ /* * Copyright © 2008,2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Eric Anholt * Chris Wilson * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include struct change_domains { uint32_t invalidate_domains; uint32_t flush_domains; uint32_t flush_rings; uint32_t flips; }; /* * Set the next domain for the specified object. This * may not actually perform the necessary flushing/invaliding though, * as that may want to be batched with other set_domain operations * * This is (we hope) the only really tricky part of gem. The goal * is fairly simple -- track which caches hold bits of the object * and make sure they remain coherent. A few concrete examples may * help to explain how it works. For shorthand, we use the notation * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the * a pair of read and write domain masks. * * Case 1: the batch buffer * * 1. Allocated * 2. Written by CPU * 3. Mapped to GTT * 4. Read by GPU * 5. Unmapped from GTT * 6. Freed * * Let's take these a step at a time * * 1. Allocated * Pages allocated from the kernel may still have * cache contents, so we set them to (CPU, CPU) always. * 2. Written by CPU (using pwrite) * The pwrite function calls set_domain (CPU, CPU) and * this function does nothing (as nothing changes) * 3. Mapped by GTT * This function asserts that the object is not * currently in any GPU-based read or write domains * 4. Read by GPU * i915_gem_execbuffer calls set_domain (COMMAND, 0). * As write_domain is zero, this function adds in the * current read domains (CPU+COMMAND, 0). * flush_domains is set to CPU. * invalidate_domains is set to COMMAND * clflush is run to get data out of the CPU caches * then i915_dev_set_domain calls i915_gem_flush to * emit an MI_FLUSH and drm_agp_chipset_flush * 5. Unmapped from GTT * i915_gem_object_unbind calls set_domain (CPU, CPU) * flush_domains and invalidate_domains end up both zero * so no flushing/invalidating happens * 6. Freed * yay, done * * Case 2: The shared render buffer * * 1. Allocated * 2. Mapped to GTT * 3. Read/written by GPU * 4. set_domain to (CPU,CPU) * 5. Read/written by CPU * 6. Read/written by GPU * * 1. Allocated * Same as last example, (CPU, CPU) * 2. Mapped to GTT * Nothing changes (assertions find that it is not in the GPU) * 3. Read/written by GPU * execbuffer calls set_domain (RENDER, RENDER) * flush_domains gets CPU * invalidate_domains gets GPU * clflush (obj) * MI_FLUSH and drm_agp_chipset_flush * 4. set_domain (CPU, CPU) * flush_domains gets GPU * invalidate_domains gets CPU * wait_rendering (obj) to make sure all drawing is complete. * This will include an MI_FLUSH to get the data from GPU * to memory * clflush (obj) to invalidate the CPU cache * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?) * 5. Read/written by CPU * cache lines are loaded and dirtied * 6. Read written by GPU * Same as last GPU access * * Case 3: The constant buffer * * 1. Allocated * 2. Written by CPU * 3. Read by GPU * 4. Updated (written) by CPU again * 5. Read by GPU * * 1. Allocated * (CPU, CPU) * 2. Written by CPU * (CPU, CPU) * 3. Read by GPU * (CPU+RENDER, 0) * flush_domains = CPU * invalidate_domains = RENDER * clflush (obj) * MI_FLUSH * drm_agp_chipset_flush * 4. Updated (written) by CPU again * (CPU, CPU) * flush_domains = 0 (no previous write domain) * invalidate_domains = 0 (no new read domains) * 5. Read by GPU * (CPU+RENDER, 0) * flush_domains = CPU * invalidate_domains = RENDER * clflush (obj) * MI_FLUSH * drm_agp_chipset_flush */ static void i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj, struct intel_ring_buffer *ring, struct change_domains *cd) { uint32_t invalidate_domains = 0, flush_domains = 0; /* * If the object isn't moving to a new write domain, * let the object stay in multiple read domains */ if (obj->base.pending_write_domain == 0) obj->base.pending_read_domains |= obj->base.read_domains; /* * Flush the current write domain if * the new read domains don't match. Invalidate * any read domains which differ from the old * write domain */ if (obj->base.write_domain && (((obj->base.write_domain != obj->base.pending_read_domains || obj->ring != ring)) || (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) { flush_domains |= obj->base.write_domain; invalidate_domains |= obj->base.pending_read_domains & ~obj->base.write_domain; } /* * Invalidate any read caches which may have * stale data. That is, any new read domains. */ invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains; if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) i915_gem_clflush_object(obj); if (obj->base.pending_write_domain) cd->flips |= atomic_load_acq_int(&obj->pending_flip); /* The actual obj->write_domain will be updated with * pending_write_domain after we emit the accumulated flush for all * of our domain changes in execbuffers (which clears objects' * write_domains). So if we have a current write domain that we * aren't changing, set pending_write_domain to that. */ if (flush_domains == 0 && obj->base.pending_write_domain == 0) obj->base.pending_write_domain = obj->base.write_domain; cd->invalidate_domains |= invalidate_domains; cd->flush_domains |= flush_domains; if (flush_domains & I915_GEM_GPU_DOMAINS) cd->flush_rings |= intel_ring_flag(obj->ring); if (invalidate_domains & I915_GEM_GPU_DOMAINS) cd->flush_rings |= intel_ring_flag(ring); } struct eb_objects { u_long hashmask; LIST_HEAD(, drm_i915_gem_object) *buckets; }; static struct eb_objects * eb_create(int size) { struct eb_objects *eb; eb = malloc(sizeof(*eb), DRM_I915_GEM, M_WAITOK | M_ZERO); eb->buckets = hashinit(size, DRM_I915_GEM, &eb->hashmask); return (eb); } static void eb_reset(struct eb_objects *eb) { int i; for (i = 0; i <= eb->hashmask; i++) LIST_INIT(&eb->buckets[i]); } static void eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj) { LIST_INSERT_HEAD(&eb->buckets[obj->exec_handle & eb->hashmask], obj, exec_node); } static struct drm_i915_gem_object * eb_get_object(struct eb_objects *eb, unsigned long handle) { struct drm_i915_gem_object *obj; LIST_FOREACH(obj, &eb->buckets[handle & eb->hashmask], exec_node) { if (obj->exec_handle == handle) return (obj); } return (NULL); } static void eb_destroy(struct eb_objects *eb) { free(eb->buckets, DRM_I915_GEM); free(eb, DRM_I915_GEM); } static inline int use_cpu_reloc(struct drm_i915_gem_object *obj) { return (obj->base.write_domain == I915_GEM_DOMAIN_CPU || obj->cache_level != I915_CACHE_NONE); } static int i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, struct eb_objects *eb, struct drm_i915_gem_relocation_entry *reloc) { struct drm_device *dev = obj->base.dev; struct drm_gem_object *target_obj; struct drm_i915_gem_object *target_i915_obj; uint32_t target_offset; int ret = -EINVAL; /* we've already hold a reference to all valid objects */ target_obj = &eb_get_object(eb, reloc->target_handle)->base; if (unlikely(target_obj == NULL)) return -ENOENT; target_i915_obj = to_intel_bo(target_obj); target_offset = target_i915_obj->gtt_offset; #if WATCH_RELOC DRM_INFO("%s: obj %p offset %08x target %d " "read %08x write %08x gtt %08x " "presumed %08x delta %08x\n", __func__, obj, (int) reloc->offset, (int) reloc->target_handle, (int) reloc->read_domains, (int) reloc->write_domain, (int) target_offset, (int) reloc->presumed_offset, reloc->delta); #endif /* The target buffer should have appeared before us in the * exec_object list, so it should have a GTT space bound by now. */ if (unlikely(target_offset == 0)) { DRM_DEBUG("No GTT space found for object %d\n", reloc->target_handle); return ret; } /* Validate that the target is in a valid r/w GPU domain */ if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { DRM_DEBUG("reloc with multiple write domains: " "obj %p target %d offset %d " "read %08x write %08x", obj, reloc->target_handle, (int) reloc->offset, reloc->read_domains, reloc->write_domain); return ret; } if (unlikely((reloc->write_domain | reloc->read_domains) & ~I915_GEM_GPU_DOMAINS)) { DRM_DEBUG("reloc with read/write non-GPU domains: " "obj %p target %d offset %d " "read %08x write %08x", obj, reloc->target_handle, (int) reloc->offset, reloc->read_domains, reloc->write_domain); return ret; } if (unlikely(reloc->write_domain && target_obj->pending_write_domain && reloc->write_domain != target_obj->pending_write_domain)) { DRM_DEBUG("Write domain conflict: " "obj %p target %d offset %d " "new %08x old %08x\n", obj, reloc->target_handle, (int) reloc->offset, reloc->write_domain, target_obj->pending_write_domain); return ret; } target_obj->pending_read_domains |= reloc->read_domains; target_obj->pending_write_domain |= reloc->write_domain; /* If the relocation already has the right value in it, no * more work needs to be done. */ if (target_offset == reloc->presumed_offset) return 0; /* Check that the relocation address is valid... */ if (unlikely(reloc->offset > obj->base.size - 4)) { DRM_DEBUG("Relocation beyond object bounds: " "obj %p target %d offset %d size %d.\n", obj, reloc->target_handle, (int) reloc->offset, (int) obj->base.size); return ret; } if (unlikely(reloc->offset & 3)) { DRM_DEBUG("Relocation not 4-byte aligned: " "obj %p target %d offset %d.\n", obj, reloc->target_handle, (int) reloc->offset); return ret; } /* We can't wait for rendering with pagefaults disabled */ if (obj->active && (curthread->td_pflags & TDP_NOFAULTING) != 0) return (-EFAULT); reloc->delta += target_offset; if (use_cpu_reloc(obj)) { uint32_t page_offset = reloc->offset & PAGE_MASK; char *vaddr; struct sf_buf *sf; ret = i915_gem_object_set_to_cpu_domain(obj, 1); if (ret) return ret; sf = sf_buf_alloc(obj->pages[OFF_TO_IDX(reloc->offset)], SFB_NOWAIT); if (sf == NULL) return (-ENOMEM); vaddr = (void *)sf_buf_kva(sf); *(uint32_t *)(vaddr + page_offset) = reloc->delta; sf_buf_free(sf); } else { uint32_t *reloc_entry; char *reloc_page; ret = i915_gem_object_set_to_gtt_domain(obj, true); if (ret) return ret; ret = i915_gem_object_put_fence(obj); if (ret) return ret; /* Map the page containing the relocation we're going to perform. */ reloc->offset += obj->gtt_offset; reloc_page = pmap_mapdev_attr(dev->agp->base + (reloc->offset & ~PAGE_MASK), PAGE_SIZE, PAT_WRITE_COMBINING); reloc_entry = (uint32_t *)(reloc_page + (reloc->offset & PAGE_MASK)); *(volatile uint32_t *)reloc_entry = reloc->delta; pmap_unmapdev((vm_offset_t)reloc_page, PAGE_SIZE); } /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and * pipe_control writes because the gpu doesn't properly redirect them * through the ppgtt for non_secure batchbuffers. */ if (unlikely(IS_GEN6(dev) && reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && !target_i915_obj->has_global_gtt_mapping)) { i915_gem_gtt_bind_object(target_i915_obj, target_i915_obj->cache_level); } /* and update the user's relocation entry */ reloc->presumed_offset = target_offset; return 0; } static int i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj, struct eb_objects *eb) { #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)]; struct drm_i915_gem_relocation_entry *user_relocs; struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; int remain, ret; user_relocs = (void *)(uintptr_t)entry->relocs_ptr; remain = entry->relocation_count; while (remain) { struct drm_i915_gem_relocation_entry *r = stack_reloc; int count = remain; if (count > DRM_ARRAY_SIZE(stack_reloc)) count = DRM_ARRAY_SIZE(stack_reloc); remain -= count; ret = -copyin_nofault(user_relocs, r, count*sizeof(r[0])); if (ret != 0) return (ret); do { u64 offset = r->presumed_offset; ret = i915_gem_execbuffer_relocate_entry(obj, eb, r); if (ret) return ret; if (r->presumed_offset != offset && copyout_nofault(&r->presumed_offset, &user_relocs->presumed_offset, sizeof(r->presumed_offset))) { return -EFAULT; } user_relocs++; r++; } while (--count); } return 0; #undef N_RELOC } static int i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj, struct eb_objects *eb, struct drm_i915_gem_relocation_entry *relocs) { const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; int i, ret; for (i = 0; i < entry->relocation_count; i++) { ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]); if (ret) return ret; } return 0; } static int i915_gem_execbuffer_relocate(struct drm_device *dev, struct eb_objects *eb, struct list_head *objects) { struct drm_i915_gem_object *obj; int ret, pflags; /* Try to move as many of the relocation targets off the active list * to avoid unnecessary fallbacks to the slow path, as we cannot wait * for the retirement with pagefaults disabled. */ i915_gem_retire_requests(dev); ret = 0; pflags = vm_fault_disable_pagefaults(); /* This is the fast path and we cannot handle a pagefault whilst * holding the device lock lest the user pass in the relocations * contained within a mmaped bo. For in such a case we, the page * fault handler would call i915_gem_fault() and we would try to * acquire the device lock again. Obviously this is bad. */ list_for_each_entry(obj, objects, exec_list) { ret = i915_gem_execbuffer_relocate_object(obj, eb); if (ret) break; } vm_fault_enable_pagefaults(pflags); return ret; } #define __EXEC_OBJECT_HAS_FENCE (1<<31) static int need_reloc_mappable(struct drm_i915_gem_object *obj) { struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; return entry->relocation_count && !use_cpu_reloc(obj); } static int pin_and_fence_object(struct drm_i915_gem_object *obj, struct intel_ring_buffer *ring) { struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; bool need_fence, need_mappable; int ret; need_fence = has_fenced_gpu_access && entry->flags & EXEC_OBJECT_NEEDS_FENCE && obj->tiling_mode != I915_TILING_NONE; need_mappable = need_fence || need_reloc_mappable(obj); ret = i915_gem_object_pin(obj, entry->alignment, need_mappable); if (ret) return ret; if (has_fenced_gpu_access) { if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) { ret = i915_gem_object_get_fence(obj); if (ret) goto err_unpin; if (i915_gem_object_pin_fence(obj)) entry->flags |= __EXEC_OBJECT_HAS_FENCE; obj->pending_fenced_gpu_access = true; } } entry->offset = obj->gtt_offset; return 0; err_unpin: i915_gem_object_unpin(obj); return ret; } static int i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, struct drm_file *file, struct list_head *objects) { drm_i915_private_t *dev_priv; struct drm_i915_gem_object *obj; struct list_head ordered_objects; bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; int ret, retry; dev_priv = ring->dev->dev_private; INIT_LIST_HEAD(&ordered_objects); while (!list_empty(objects)) { struct drm_i915_gem_exec_object2 *entry; bool need_fence, need_mappable; obj = list_first_entry(objects, struct drm_i915_gem_object, exec_list); entry = obj->exec_entry; need_fence = has_fenced_gpu_access && entry->flags & EXEC_OBJECT_NEEDS_FENCE && obj->tiling_mode != I915_TILING_NONE; need_mappable = need_fence || need_reloc_mappable(obj); if (need_mappable) list_move(&obj->exec_list, &ordered_objects); else list_move_tail(&obj->exec_list, &ordered_objects); obj->base.pending_read_domains = 0; obj->base.pending_write_domain = 0; } list_splice(&ordered_objects, objects); /* Attempt to pin all of the buffers into the GTT. * This is done in 3 phases: * * 1a. Unbind all objects that do not match the GTT constraints for * the execbuffer (fenceable, mappable, alignment etc). * 1b. Increment pin count for already bound objects. * 2. Bind new objects. * 3. Decrement pin count. * * This avoid unnecessary unbinding of later objects in order to make * room for the earlier objects *unless* we need to defragment. */ retry = 0; do { ret = 0; /* Unbind any ill-fitting objects or pin. */ list_for_each_entry(obj, objects, exec_list) { struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; bool need_fence, need_mappable; if (!obj->gtt_space) continue; need_fence = has_fenced_gpu_access && entry->flags & EXEC_OBJECT_NEEDS_FENCE && obj->tiling_mode != I915_TILING_NONE; need_mappable = need_fence || need_reloc_mappable(obj); if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) || (need_mappable && !obj->map_and_fenceable)) ret = i915_gem_object_unbind(obj); else ret = pin_and_fence_object(obj, ring); if (ret) goto err; } /* Bind fresh objects */ list_for_each_entry(obj, objects, exec_list) { if (obj->gtt_space) continue; ret = pin_and_fence_object(obj, ring); if (ret) { int ret_ignore; /* This can potentially raise a harmless * -EINVAL if we failed to bind in the above * call. It cannot raise -EINTR since we know * that the bo is freshly bound and so will * not need to be flushed or waited upon. */ ret_ignore = i915_gem_object_unbind(obj); (void)ret_ignore; if (obj->gtt_space != NULL) printf("%s: gtt_space\n", __func__); break; } } /* Decrement pin count for bound objects */ list_for_each_entry(obj, objects, exec_list) { struct drm_i915_gem_exec_object2 *entry; if (!obj->gtt_space) continue; entry = obj->exec_entry; if (entry->flags & __EXEC_OBJECT_HAS_FENCE) { i915_gem_object_unpin_fence(obj); entry->flags &= ~__EXEC_OBJECT_HAS_FENCE; } i915_gem_object_unpin(obj); /* ... and ensure ppgtt mapping exist if needed. */ if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) { i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, obj, obj->cache_level); obj->has_aliasing_ppgtt_mapping = 1; } } if (ret != -ENOSPC || retry > 1) return ret; /* First attempt, just clear anything that is purgeable. * Second attempt, clear the entire GTT. */ ret = i915_gem_evict_everything(ring->dev, retry == 0); if (ret) return ret; retry++; } while (1); err: list_for_each_entry_continue_reverse(obj, objects, exec_list) { struct drm_i915_gem_exec_object2 *entry; if (!obj->gtt_space) continue; entry = obj->exec_entry; if (entry->flags & __EXEC_OBJECT_HAS_FENCE) { i915_gem_object_unpin_fence(obj); entry->flags &= ~__EXEC_OBJECT_HAS_FENCE; } i915_gem_object_unpin(obj); } return ret; } static int i915_gem_execbuffer_relocate_slow(struct drm_device *dev, struct drm_file *file, struct intel_ring_buffer *ring, struct list_head *objects, struct eb_objects *eb, struct drm_i915_gem_exec_object2 *exec, int count) { struct drm_i915_gem_relocation_entry *reloc; struct drm_i915_gem_object *obj; int *reloc_offset; int i, total, ret; /* We may process another execbuffer during the unlock... */ while (!list_empty(objects)) { obj = list_first_entry(objects, struct drm_i915_gem_object, exec_list); list_del_init(&obj->exec_list); drm_gem_object_unreference(&obj->base); } DRM_UNLOCK(dev); total = 0; for (i = 0; i < count; i++) total += exec[i].relocation_count; reloc_offset = malloc(count * sizeof(*reloc_offset), DRM_I915_GEM, M_WAITOK | M_ZERO); reloc = malloc(total * sizeof(*reloc), DRM_I915_GEM, M_WAITOK | M_ZERO); total = 0; for (i = 0; i < count; i++) { struct drm_i915_gem_relocation_entry *user_relocs; user_relocs = (void *)(uintptr_t)exec[i].relocs_ptr; ret = -copyin(user_relocs, reloc + total, exec[i].relocation_count * sizeof(*reloc)); if (ret != 0) { DRM_LOCK(dev); goto err; } reloc_offset[i] = total; total += exec[i].relocation_count; } ret = i915_mutex_lock_interruptible(dev); if (ret) { DRM_LOCK(dev); goto err; } /* reacquire the objects */ eb_reset(eb); for (i = 0; i < count; i++) { struct drm_i915_gem_object *obj; obj = to_intel_bo(drm_gem_object_lookup(dev, file, exec[i].handle)); if (&obj->base == NULL) { DRM_DEBUG("Invalid object handle %d at index %d\n", exec[i].handle, i); ret = -ENOENT; goto err; } list_add_tail(&obj->exec_list, objects); obj->exec_handle = exec[i].handle; obj->exec_entry = &exec[i]; eb_add_object(eb, obj); } ret = i915_gem_execbuffer_reserve(ring, file, objects); if (ret) goto err; list_for_each_entry(obj, objects, exec_list) { int offset = obj->exec_entry - exec; ret = i915_gem_execbuffer_relocate_object_slow(obj, eb, reloc + reloc_offset[offset]); if (ret) goto err; } /* Leave the user relocations as are, this is the painfully slow path, * and we want to avoid the complication of dropping the lock whilst * having buffers reserved in the aperture and so causing spurious * ENOSPC for random operations. */ err: free(reloc, DRM_I915_GEM); free(reloc_offset, DRM_I915_GEM); return ret; } static int i915_gem_execbuffer_flush(struct drm_device *dev, uint32_t invalidate_domains, uint32_t flush_domains, uint32_t flush_rings) { drm_i915_private_t *dev_priv = dev->dev_private; int i, ret; if (flush_domains & I915_GEM_DOMAIN_CPU) intel_gtt_chipset_flush(); if (flush_domains & I915_GEM_DOMAIN_GTT) wmb(); if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) { for (i = 0; i < I915_NUM_RINGS; i++) if (flush_rings & (1 << i)) { ret = i915_gem_flush_ring(&dev_priv->rings[i], invalidate_domains, flush_domains); if (ret) return ret; } } return 0; } static int i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips) { u32 plane, flip_mask; int ret; /* Check for any pending flips. As we only maintain a flip queue depth * of 1, we can simply insert a WAIT for the next display flip prior * to executing the batch and avoid stalling the CPU. */ for (plane = 0; flips >> plane; plane++) { if (((flips >> plane) & 1) == 0) continue; if (plane) flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; else flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; ret = intel_ring_begin(ring, 2); if (ret) return ret; intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); } return 0; } static int i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring, struct list_head *objects) { struct drm_i915_gem_object *obj; struct change_domains cd; int ret; memset(&cd, 0, sizeof(cd)); list_for_each_entry(obj, objects, exec_list) i915_gem_object_set_to_gpu_domain(obj, ring, &cd); if (cd.invalidate_domains | cd.flush_domains) { #if WATCH_EXEC DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n", __func__, cd.invalidate_domains, cd.flush_domains); #endif ret = i915_gem_execbuffer_flush(ring->dev, cd.invalidate_domains, cd.flush_domains, cd.flush_rings); if (ret) return ret; } if (cd.flips) { ret = i915_gem_execbuffer_wait_for_flips(ring, cd.flips); if (ret) return ret; } list_for_each_entry(obj, objects, exec_list) { ret = i915_gem_object_sync(obj, ring); if (ret) return ret; } return 0; } static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) { return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0; } static int validate_exec_list(struct drm_i915_gem_exec_object2 *exec, int count, vm_page_t ***map) { vm_page_t *ma; int i, length, page_count; /* XXXKIB various limits checking is missing there */ *map = malloc(count * sizeof(*ma), DRM_I915_GEM, M_WAITOK | M_ZERO); for (i = 0; i < count; i++) { /* First check for malicious input causing overflow */ if (exec[i].relocation_count > INT_MAX / sizeof(struct drm_i915_gem_relocation_entry)) return -EINVAL; length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry); if (length == 0) { (*map)[i] = NULL; continue; } /* * Since both start and end of the relocation region * may be not aligned on the page boundary, be * conservative and request a page slot for each * partial page. Thus +2. */ page_count = howmany(length, PAGE_SIZE) + 2; ma = (*map)[i] = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | M_ZERO); if (vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, exec[i].relocs_ptr, length, VM_PROT_READ | VM_PROT_WRITE, ma, page_count) == -1) { free(ma, DRM_I915_GEM); (*map)[i] = NULL; return (-EFAULT); } } return 0; } static void i915_gem_execbuffer_move_to_active(struct list_head *objects, struct intel_ring_buffer *ring, u32 seqno) { struct drm_i915_gem_object *obj; uint32_t old_read, old_write; list_for_each_entry(obj, objects, exec_list) { old_read = obj->base.read_domains; old_write = obj->base.write_domain; obj->base.read_domains = obj->base.pending_read_domains; obj->base.write_domain = obj->base.pending_write_domain; obj->fenced_gpu_access = obj->pending_fenced_gpu_access; i915_gem_object_move_to_active(obj, ring, seqno); if (obj->base.write_domain) { obj->dirty = 1; obj->pending_gpu_write = true; list_move_tail(&obj->gpu_write_list, &ring->gpu_write_list); if (obj->pin_count) /* check for potential scanout */ intel_mark_busy(ring->dev, obj); } CTR3(KTR_DRM, "object_change_domain move_to_active %p %x %x", obj, old_read, old_write); } intel_mark_busy(ring->dev, NULL); } int i915_gem_sync_exec_requests; static void i915_gem_execbuffer_retire_commands(struct drm_device *dev, struct drm_file *file, struct intel_ring_buffer *ring) { struct drm_i915_gem_request *request; u32 invalidate; /* * Ensure that the commands in the batch buffer are * finished before the interrupt fires. * * The sampler always gets flushed on i965 (sigh). */ invalidate = I915_GEM_DOMAIN_COMMAND; if (INTEL_INFO(dev)->gen >= 4) invalidate |= I915_GEM_DOMAIN_SAMPLER; if (ring->flush(ring, invalidate, 0)) { i915_gem_next_request_seqno(ring); return; } /* Add a breadcrumb for the completion of the batch buffer */ request = malloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO); if (request == NULL || i915_add_request(ring, file, request)) { i915_gem_next_request_seqno(ring); free(request, DRM_I915_GEM); } else if (i915_gem_sync_exec_requests) { i915_wait_request(ring, request->seqno); i915_gem_retire_requests(dev); } } static void i915_gem_fix_mi_batchbuffer_end(struct drm_i915_gem_object *batch_obj, uint32_t batch_start_offset, uint32_t batch_len) { char *mkva; uint64_t po_r, po_w; uint32_t cmd; po_r = batch_obj->base.dev->agp->base + batch_obj->gtt_offset + batch_start_offset + batch_len; if (batch_len > 0) po_r -= 4; mkva = pmap_mapdev_attr(trunc_page(po_r), 2 * PAGE_SIZE, PAT_WRITE_COMBINING); po_r &= PAGE_MASK; cmd = *(uint32_t *)(mkva + po_r); if (cmd != MI_BATCH_BUFFER_END) { /* * batch_len != 0 due to the check at the start of * i915_gem_do_execbuffer */ if (batch_obj->base.size > batch_start_offset + batch_len) { po_w = po_r + 4; /* DRM_DEBUG("batchbuffer does not end by MI_BATCH_BUFFER_END !\n"); */ } else { po_w = po_r; DRM_DEBUG("batchbuffer does not end by MI_BATCH_BUFFER_END, overwriting last bo cmd !\n"); } *(uint32_t *)(mkva + po_w) = MI_BATCH_BUFFER_END; } pmap_unmapdev((vm_offset_t)mkva, 2 * PAGE_SIZE); } int i915_fix_mi_batchbuffer_end = 0; static int i915_reset_gen7_sol_offsets(struct drm_device *dev, struct intel_ring_buffer *ring) { drm_i915_private_t *dev_priv = dev->dev_private; int ret, i; if (!IS_GEN7(dev) || ring != &dev_priv->rings[RCS]) return 0; ret = intel_ring_begin(ring, 4 * 3); if (ret) return ret; for (i = 0; i < 4; i++) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i)); intel_ring_emit(ring, 0); } intel_ring_advance(ring); return 0; } static int i915_gem_do_execbuffer(struct drm_device *dev, void *data, struct drm_file *file, struct drm_i915_gem_execbuffer2 *args, struct drm_i915_gem_exec_object2 *exec) { drm_i915_private_t *dev_priv = dev->dev_private; struct list_head objects; struct eb_objects *eb; struct drm_i915_gem_object *batch_obj; struct drm_clip_rect *cliprects = NULL; struct intel_ring_buffer *ring; vm_page_t **relocs_ma; u32 ctx_id = i915_execbuffer2_get_context_id(*args); u32 exec_start, exec_len; u32 seqno; u32 mask; int ret, mode, i; if (!i915_gem_check_execbuffer(args)) { DRM_DEBUG("execbuf with invalid offset/length\n"); return -EINVAL; } if (args->batch_len == 0) return (0); ret = validate_exec_list(exec, args->buffer_count, &relocs_ma); if (ret != 0) goto pre_struct_lock_err; switch (args->flags & I915_EXEC_RING_MASK) { case I915_EXEC_DEFAULT: case I915_EXEC_RENDER: ring = &dev_priv->rings[RCS]; break; case I915_EXEC_BSD: ring = &dev_priv->rings[VCS]; if (ctx_id != 0) { DRM_DEBUG("Ring %s doesn't support contexts\n", ring->name); return -EPERM; } break; case I915_EXEC_BLT: ring = &dev_priv->rings[BCS]; if (ctx_id != 0) { DRM_DEBUG("Ring %s doesn't support contexts\n", ring->name); return -EPERM; } break; default: DRM_DEBUG("execbuf with unknown ring: %d\n", (int)(args->flags & I915_EXEC_RING_MASK)); ret = -EINVAL; goto pre_struct_lock_err; } if (!intel_ring_initialized(ring)) { DRM_DEBUG("execbuf with invalid ring: %d\n", (int)(args->flags & I915_EXEC_RING_MASK)); return -EINVAL; } mode = args->flags & I915_EXEC_CONSTANTS_MASK; mask = I915_EXEC_CONSTANTS_MASK; switch (mode) { case I915_EXEC_CONSTANTS_REL_GENERAL: case I915_EXEC_CONSTANTS_ABSOLUTE: case I915_EXEC_CONSTANTS_REL_SURFACE: if (ring == &dev_priv->rings[RCS] && mode != dev_priv->relative_constants_mode) { if (INTEL_INFO(dev)->gen < 4) { ret = -EINVAL; goto pre_struct_lock_err; } if (INTEL_INFO(dev)->gen > 5 && mode == I915_EXEC_CONSTANTS_REL_SURFACE) { ret = -EINVAL; goto pre_struct_lock_err; } /* The HW changed the meaning on this bit on gen6 */ if (INTEL_INFO(dev)->gen >= 6) mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE; } break; default: DRM_DEBUG("execbuf with unknown constants: %d\n", mode); ret = -EINVAL; goto pre_struct_lock_err; } if (args->buffer_count < 1) { DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count); ret = -EINVAL; goto pre_struct_lock_err; } if (args->num_cliprects != 0) { if (ring != &dev_priv->rings[RCS]) { DRM_DEBUG("clip rectangles are only valid with the render ring\n"); ret = -EINVAL; goto pre_struct_lock_err; } if (INTEL_INFO(dev)->gen >= 5) { DRM_DEBUG("clip rectangles are only valid on pre-gen5\n"); ret = -EINVAL; goto pre_struct_lock_err; } if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) { DRM_DEBUG("execbuf with %u cliprects\n", args->num_cliprects); ret = -EINVAL; goto pre_struct_lock_err; } cliprects = malloc( sizeof(*cliprects) * args->num_cliprects, DRM_I915_GEM, M_WAITOK | M_ZERO); ret = -copyin((void *)(uintptr_t)args->cliprects_ptr, cliprects, sizeof(*cliprects) * args->num_cliprects); if (ret != 0) goto pre_struct_lock_err; } ret = i915_mutex_lock_interruptible(dev); if (ret) goto pre_struct_lock_err; if (dev_priv->mm.suspended) { DRM_UNLOCK(dev); ret = -EBUSY; goto pre_struct_lock_err; } eb = eb_create(args->buffer_count); if (eb == NULL) { DRM_UNLOCK(dev); ret = -ENOMEM; goto pre_struct_lock_err; } /* Look up object handles */ INIT_LIST_HEAD(&objects); for (i = 0; i < args->buffer_count; i++) { struct drm_i915_gem_object *obj; obj = to_intel_bo(drm_gem_object_lookup(dev, file, exec[i].handle)); if (&obj->base == NULL) { DRM_DEBUG("Invalid object handle %d at index %d\n", exec[i].handle, i); /* prevent error path from reading uninitialized data */ ret = -ENOENT; goto err; } if (!list_empty(&obj->exec_list)) { DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n", obj, exec[i].handle, i); ret = -EINVAL; goto err; } list_add_tail(&obj->exec_list, &objects); obj->exec_handle = exec[i].handle; obj->exec_entry = &exec[i]; eb_add_object(eb, obj); } /* take note of the batch buffer before we might reorder the lists */ batch_obj = list_entry(objects.prev, struct drm_i915_gem_object, exec_list); /* Move the objects en-masse into the GTT, evicting if necessary. */ ret = i915_gem_execbuffer_reserve(ring, file, &objects); if (ret) goto err; /* The objects are in their final locations, apply the relocations. */ ret = i915_gem_execbuffer_relocate(dev, eb, &objects); if (ret) { if (ret == -EFAULT) { ret = i915_gem_execbuffer_relocate_slow(dev, file, ring, &objects, eb, exec, args->buffer_count); DRM_LOCK_ASSERT(dev); } if (ret) goto err; } /* Set the pending read domains for the batch buffer to COMMAND */ if (batch_obj->base.pending_write_domain) { DRM_DEBUG("Attempting to use self-modifying batch buffer\n"); ret = -EINVAL; goto err; } batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND; ret = i915_gem_execbuffer_move_to_gpu(ring, &objects); if (ret) goto err; ret = i915_switch_context(ring, file, ctx_id); if (ret) goto err; seqno = i915_gem_next_request_seqno(ring); for (i = 0; i < I915_NUM_RINGS - 1; i++) { if (seqno < ring->sync_seqno[i]) { /* The GPU can not handle its semaphore value wrapping, * so every billion or so execbuffers, we need to stall * the GPU in order to reset the counters. */ ret = i915_gpu_idle(dev); if (ret) goto err; i915_gem_retire_requests(dev); KASSERT(ring->sync_seqno[i] == 0, ("Non-zero sync_seqno")); } } if (ring == &dev_priv->rings[RCS] && mode != dev_priv->relative_constants_mode) { ret = intel_ring_begin(ring, 4); if (ret) goto err; intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit(ring, INSTPM); intel_ring_emit(ring, mask << 16 | mode); intel_ring_advance(ring); dev_priv->relative_constants_mode = mode; } if (args->flags & I915_EXEC_GEN7_SOL_RESET) { ret = i915_reset_gen7_sol_offsets(dev, ring); if (ret) goto err; } exec_start = batch_obj->gtt_offset + args->batch_start_offset; exec_len = args->batch_len; if (i915_fix_mi_batchbuffer_end) { i915_gem_fix_mi_batchbuffer_end(batch_obj, args->batch_start_offset, args->batch_len); } CTR4(KTR_DRM, "ring_dispatch %s %d exec %x %x", ring->name, seqno, exec_start, exec_len); if (cliprects) { for (i = 0; i < args->num_cliprects; i++) { - ret = i915_emit_box_p(dev, &cliprects[i], + ret = i915_emit_box(dev, &cliprects[i], args->DR1, args->DR4); if (ret) goto err; ret = ring->dispatch_execbuffer(ring, exec_start, exec_len); if (ret) goto err; } } else { ret = ring->dispatch_execbuffer(ring, exec_start, exec_len); if (ret) goto err; } i915_gem_execbuffer_move_to_active(&objects, ring, seqno); i915_gem_execbuffer_retire_commands(dev, file, ring); err: eb_destroy(eb); while (!list_empty(&objects)) { struct drm_i915_gem_object *obj; obj = list_first_entry(&objects, struct drm_i915_gem_object, exec_list); list_del_init(&obj->exec_list); drm_gem_object_unreference(&obj->base); } DRM_UNLOCK(dev); pre_struct_lock_err: for (i = 0; i < args->buffer_count; i++) { if (relocs_ma[i] != NULL) { vm_page_unhold_pages(relocs_ma[i], howmany( exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry), PAGE_SIZE)); free(relocs_ma[i], DRM_I915_GEM); } } free(relocs_ma, DRM_I915_GEM); free(cliprects, DRM_I915_GEM); return ret; } /* * Legacy execbuffer just creates an exec2 list from the original exec object * list array and passes it to the real function. */ int i915_gem_execbuffer(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_execbuffer *args = data; struct drm_i915_gem_execbuffer2 exec2; struct drm_i915_gem_exec_object *exec_list = NULL; struct drm_i915_gem_exec_object2 *exec2_list = NULL; int ret, i; DRM_DEBUG("buffers_ptr %d buffer_count %d len %08x\n", (int) args->buffers_ptr, args->buffer_count, args->batch_len); if (args->buffer_count < 1) { DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count); return -EINVAL; } /* Copy in the exec list from userland */ /* XXXKIB user-controlled malloc size */ exec_list = malloc(sizeof(*exec_list) * args->buffer_count, DRM_I915_GEM, M_WAITOK); exec2_list = malloc(sizeof(*exec2_list) * args->buffer_count, DRM_I915_GEM, M_WAITOK); ret = -copyin((void *)(uintptr_t)args->buffers_ptr, exec_list, sizeof(*exec_list) * args->buffer_count); if (ret != 0) { DRM_DEBUG("copy %d exec entries failed %d\n", args->buffer_count, ret); free(exec_list, DRM_I915_GEM); free(exec2_list, DRM_I915_GEM); return (ret); } for (i = 0; i < args->buffer_count; i++) { exec2_list[i].handle = exec_list[i].handle; exec2_list[i].relocation_count = exec_list[i].relocation_count; exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; exec2_list[i].alignment = exec_list[i].alignment; exec2_list[i].offset = exec_list[i].offset; if (INTEL_INFO(dev)->gen < 4) exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; else exec2_list[i].flags = 0; } exec2.buffers_ptr = args->buffers_ptr; exec2.buffer_count = args->buffer_count; exec2.batch_start_offset = args->batch_start_offset; exec2.batch_len = args->batch_len; exec2.DR1 = args->DR1; exec2.DR4 = args->DR4; exec2.num_cliprects = args->num_cliprects; exec2.cliprects_ptr = args->cliprects_ptr; exec2.flags = I915_EXEC_RENDER; i915_execbuffer2_set_context_id(exec2, 0); ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list); if (!ret) { /* Copy the new buffer offsets back to the user's exec list. */ for (i = 0; i < args->buffer_count; i++) exec_list[i].offset = exec2_list[i].offset; /* ... and back out to userspace */ ret = -copyout(exec_list, (void *)(uintptr_t)args->buffers_ptr, sizeof(*exec_list) * args->buffer_count); if (ret != 0) { DRM_DEBUG("failed to copy %d exec entries " "back to user (%d)\n", args->buffer_count, ret); } } free(exec_list, DRM_I915_GEM); free(exec2_list, DRM_I915_GEM); return ret; } int i915_gem_execbuffer2(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_execbuffer2 *args = data; struct drm_i915_gem_exec_object2 *exec2_list = NULL; int ret; DRM_DEBUG("buffers_ptr %jx buffer_count %d len %08x\n", (uintmax_t)args->buffers_ptr, args->buffer_count, args->batch_len); if (args->buffer_count < 1 || args->buffer_count > UINT_MAX / sizeof(*exec2_list)) { DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count); return -EINVAL; } /* XXXKIB user-controllable malloc size */ exec2_list = malloc(sizeof(*exec2_list) * args->buffer_count, DRM_I915_GEM, M_WAITOK); ret = -copyin((void *)(uintptr_t)args->buffers_ptr, exec2_list, sizeof(*exec2_list) * args->buffer_count); if (ret != 0) { DRM_DEBUG("copy %d exec entries failed %d\n", args->buffer_count, ret); free(exec2_list, DRM_I915_GEM); return -EFAULT; } ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list); if (!ret) { /* Copy the new buffer offsets back to the user's exec list. */ ret = -copyout(exec2_list, (void *)(uintptr_t)args->buffers_ptr, sizeof(*exec2_list) * args->buffer_count); if (ret) { DRM_DEBUG("failed to copy %d exec entries " "back to user (%d)\n", args->buffer_count, ret); } } free(exec2_list, DRM_I915_GEM); return ret; }