Index: head/sys/dev/drm2/drm_gem_names.c =================================================================== --- head/sys/dev/drm2/drm_gem_names.c (revision 271704) +++ head/sys/dev/drm2/drm_gem_names.c (revision 271705) @@ -1,211 +1,229 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include MALLOC_DEFINE(M_GEM_NAMES, "gem_name", "Hash headers for the gem names"); static void drm_gem_names_delete_name(struct drm_gem_names *names, struct drm_gem_name *np); void drm_gem_names_init(struct drm_gem_names *names) { names->unr = new_unrhdr(1, INT_MAX, NULL); /* XXXKIB */ names->names_hash = hashinit(1000 /* XXXKIB */, M_GEM_NAMES, &names->hash_mask); mtx_init(&names->lock, "drmnames", NULL, MTX_DEF); } void drm_gem_names_fini(struct drm_gem_names *names) { struct drm_gem_name *np; int i; mtx_lock(&names->lock); for (i = 0; i <= names->hash_mask; i++) { while ((np = LIST_FIRST(&names->names_hash[i])) != NULL) { drm_gem_names_delete_name(names, np); mtx_lock(&names->lock); } } mtx_unlock(&names->lock); mtx_destroy(&names->lock); hashdestroy(names->names_hash, M_GEM_NAMES, names->hash_mask); delete_unrhdr(names->unr); } static struct drm_gem_names_head * gem_name_hash_index(struct drm_gem_names *names, int name) { return (&names->names_hash[name & names->hash_mask]); } void * drm_gem_name_ref(struct drm_gem_names *names, uint32_t name, void (*ref)(void *)) { struct drm_gem_name *n; mtx_lock(&names->lock); LIST_FOREACH(n, gem_name_hash_index(names, name), link) { if (n->name == name) { if (ref != NULL) ref(n->ptr); mtx_unlock(&names->lock); return (n->ptr); } } mtx_unlock(&names->lock); return (NULL); } struct drm_gem_ptr_match_arg { uint32_t res; void *ptr; }; static int drm_gem_ptr_match(uint32_t name, void *ptr, void *arg) { struct drm_gem_ptr_match_arg *a; a = arg; if (ptr == a->ptr) { a->res = name; return (1); } else return (0); } uint32_t drm_gem_find_name(struct drm_gem_names *names, void *ptr) { struct drm_gem_ptr_match_arg arg; arg.res = 0; arg.ptr = ptr; drm_gem_names_foreach(names, drm_gem_ptr_match, &arg); return (arg.res); } +void * +drm_gem_find_ptr(struct drm_gem_names *names, uint32_t name) +{ + struct drm_gem_name *n; + void *res; + + mtx_lock(&names->lock); + LIST_FOREACH(n, gem_name_hash_index(names, name), link) { + if (n->name == name) { + res = n->ptr; + mtx_unlock(&names->lock); + return (res); + } + } + mtx_unlock(&names->lock); + return (NULL); +} + int drm_gem_name_create(struct drm_gem_names *names, void *p, uint32_t *name) { struct drm_gem_name *np; if (*name != 0) { return (EALREADY); } np = malloc(sizeof(struct drm_gem_name), M_GEM_NAMES, M_WAITOK); mtx_lock(&names->lock); np->name = alloc_unr(names->unr); if (np->name == -1) { mtx_unlock(&names->lock); free(np, M_GEM_NAMES); return (ENOMEM); } *name = np->name; np->ptr = p; LIST_INSERT_HEAD(gem_name_hash_index(names, np->name), np, link); mtx_unlock(&names->lock); return (0); } static void drm_gem_names_delete_name(struct drm_gem_names *names, struct drm_gem_name *np) { mtx_assert(&names->lock, MA_OWNED); LIST_REMOVE(np, link); mtx_unlock(&names->lock); free_unr(names->unr, np->name); free(np, M_GEM_NAMES); } void * drm_gem_names_remove(struct drm_gem_names *names, uint32_t name) { struct drm_gem_name *n; void *res; mtx_lock(&names->lock); LIST_FOREACH(n, gem_name_hash_index(names, name), link) { if (n->name == name) { res = n->ptr; drm_gem_names_delete_name(names, n); return (res); } } mtx_unlock(&names->lock); return (NULL); } void drm_gem_names_foreach(struct drm_gem_names *names, int (*f)(uint32_t, void *, void *), void *arg) { struct drm_gem_name *np; struct drm_gem_name marker; int i, fres; bzero(&marker, sizeof(marker)); marker.name = -1; mtx_lock(&names->lock); for (i = 0; i <= names->hash_mask; i++) { for (np = LIST_FIRST(&names->names_hash[i]); np != NULL; ) { if (np->name == -1) { np = LIST_NEXT(np, link); continue; } LIST_INSERT_AFTER(np, &marker, link); mtx_unlock(&names->lock); fres = f(np->name, np->ptr, arg); mtx_lock(&names->lock); np = LIST_NEXT(&marker, link); LIST_REMOVE(&marker, link); if (fres) break; } } mtx_unlock(&names->lock); } Index: head/sys/dev/drm2/drm_gem_names.h =================================================================== --- head/sys/dev/drm2/drm_gem_names.h (revision 271704) +++ head/sys/dev/drm2/drm_gem_names.h (revision 271705) @@ -1,64 +1,65 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef DRM_GEM_NAMES_H #define DRM_GEM_NAMES_H #include #include #include #include struct drm_gem_name { uint32_t name; void *ptr; LIST_ENTRY(drm_gem_name) link; }; struct drm_gem_names { struct mtx lock; LIST_HEAD(drm_gem_names_head, drm_gem_name) *names_hash; u_long hash_mask; struct unrhdr *unr; }; void drm_gem_names_init(struct drm_gem_names *names); void drm_gem_names_fini(struct drm_gem_names *names); uint32_t drm_gem_find_name(struct drm_gem_names *names, void *ptr); +void *drm_gem_find_ptr(struct drm_gem_names *names, uint32_t name); void *drm_gem_name_ref(struct drm_gem_names *names, uint32_t name, void (*ref)(void *)); int drm_gem_name_create(struct drm_gem_names *names, void *obj, uint32_t *name); void drm_gem_names_foreach(struct drm_gem_names *names, int (*f)(uint32_t, void *, void *), void *arg); void *drm_gem_names_remove(struct drm_gem_names *names, uint32_t name); #endif Index: head/sys/dev/drm2/i915/i915_dma.c =================================================================== --- head/sys/dev/drm2/i915/i915_dma.c (revision 271704) +++ head/sys/dev/drm2/i915/i915_dma.c (revision 271705) @@ -1,2084 +1,2090 @@ /* i915_dma.c -- DMA support for the I915 -*- linux-c -*- */ /*- * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static struct drm_i915_private *i915_mch_dev; /* * Lock protecting IPS related data structures * - i915_mch_dev * - dev_priv->max_delay * - dev_priv->min_delay * - dev_priv->fmax * - dev_priv->gpu_busy */ static struct mtx mchdev_lock; MTX_SYSINIT(mchdev, &mchdev_lock, "mchdev", MTX_DEF); static void i915_pineview_get_mem_freq(struct drm_device *dev); static void i915_ironlake_get_mem_freq(struct drm_device *dev); static int i915_driver_unload_int(struct drm_device *dev, bool locked); static void i915_write_hws_pga(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; u32 addr; addr = dev_priv->status_page_dmah->busaddr; if (INTEL_INFO(dev)->gen >= 4) addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; I915_WRITE(HWS_PGA, addr); } /** * Sets up the hardware status page for devices that need a physical address * in the register. */ static int i915_init_phys_hws(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = LP_RING(dev_priv); /* * Program Hardware Status Page * XXXKIB Keep 4GB limit for allocation for now. This method * of allocation is used on <= 965 hardware, that has several * erratas regarding the use of physical memory > 4 GB. */ DRM_UNLOCK(dev); dev_priv->status_page_dmah = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 0xffffffff); DRM_LOCK(dev); if (!dev_priv->status_page_dmah) { DRM_ERROR("Can not allocate hardware status page\n"); return -ENOMEM; } ring->status_page.page_addr = dev_priv->hw_status_page = dev_priv->status_page_dmah->vaddr; dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr; memset(dev_priv->hw_status_page, 0, PAGE_SIZE); i915_write_hws_pga(dev); DRM_DEBUG("Enabled hardware status page, phys %jx\n", (uintmax_t)dev_priv->dma_status_page); return 0; } /** * Frees the hardware status page, whether it's a physical address or a virtual * address set up by the X Server. */ static void i915_free_hws(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = LP_RING(dev_priv); if (dev_priv->status_page_dmah) { drm_pci_free(dev, dev_priv->status_page_dmah); dev_priv->status_page_dmah = NULL; } if (dev_priv->status_gfx_addr) { dev_priv->status_gfx_addr = 0; ring->status_page.gfx_addr = 0; drm_core_ioremapfree(&dev_priv->hws_map, dev); } /* Need to rewrite hardware status page */ I915_WRITE(HWS_PGA, 0x1ffff000); } void i915_kernel_lost_context(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = LP_RING(dev_priv); /* * We should never lose context on the ring with modesetting * as we don't expose it to userspace */ if (drm_core_check_feature(dev, DRIVER_MODESET)) return; ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; ring->space = ring->head - (ring->tail + 8); if (ring->space < 0) ring->space += ring->size; #if 1 KIB_NOTYET(); #else if (!dev->primary->master) return; #endif if (ring->head == ring->tail && dev_priv->sarea_priv) dev_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY; } static int i915_dma_cleanup(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; int i; /* Make sure interrupts are disabled here because the uninstall ioctl * may not have been called from userspace and after dev_private * is freed, it's too late. */ if (dev->irq_enabled) drm_irq_uninstall(dev); for (i = 0; i < I915_NUM_RINGS; i++) intel_cleanup_ring_buffer(&dev_priv->rings[i]); /* Clear the HWS virtual address at teardown */ if (I915_NEED_GFX_HWS(dev)) i915_free_hws(dev); return 0; } static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) { drm_i915_private_t *dev_priv = dev->dev_private; int ret; dev_priv->sarea = drm_getsarea(dev); if (!dev_priv->sarea) { DRM_ERROR("can not find sarea!\n"); i915_dma_cleanup(dev); return -EINVAL; } dev_priv->sarea_priv = (drm_i915_sarea_t *) ((u8 *) dev_priv->sarea->virtual + init->sarea_priv_offset); if (init->ring_size != 0) { if (LP_RING(dev_priv)->obj != NULL) { i915_dma_cleanup(dev); DRM_ERROR("Client tried to initialize ringbuffer in " "GEM mode\n"); return -EINVAL; } ret = intel_render_ring_init_dri(dev, init->ring_start, init->ring_size); if (ret) { i915_dma_cleanup(dev); return ret; } } dev_priv->cpp = init->cpp; dev_priv->back_offset = init->back_offset; dev_priv->front_offset = init->front_offset; dev_priv->current_page = 0; dev_priv->sarea_priv->pf_current_page = 0; /* Allow hardware batchbuffers unless told otherwise. */ dev_priv->allow_batchbuffer = 1; return 0; } static int i915_dma_resume(struct drm_device * dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; struct intel_ring_buffer *ring = LP_RING(dev_priv); DRM_DEBUG("\n"); if (ring->map.handle == NULL) { DRM_ERROR("can not ioremap virtual address for" " ring buffer\n"); return -ENOMEM; } /* Program Hardware Status Page */ if (!ring->status_page.page_addr) { DRM_ERROR("Can not find hardware status page\n"); return -EINVAL; } DRM_DEBUG("hw status page @ %p\n", ring->status_page.page_addr); if (ring->status_page.gfx_addr != 0) intel_ring_setup_status_page(ring); else i915_write_hws_pga(dev); DRM_DEBUG("Enabled hardware status page\n"); return 0; } static int i915_dma_init(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_init_t *init = data; int retcode = 0; switch (init->func) { case I915_INIT_DMA: retcode = i915_initialize(dev, init); break; case I915_CLEANUP_DMA: retcode = i915_dma_cleanup(dev); break; case I915_RESUME_DMA: retcode = i915_dma_resume(dev); break; default: retcode = -EINVAL; break; } return retcode; } /* Implement basically the same security restrictions as hardware does * for MI_BATCH_NON_SECURE. These can be made stricter at any time. * * Most of the calculations below involve calculating the size of a * particular instruction. It's important to get the size right as * that tells us where the next instruction to check is. Any illegal * instruction detected will be given a size of zero, which is a * signal to abort the rest of the buffer. */ static int do_validate_cmd(int cmd) { switch (((cmd >> 29) & 0x7)) { case 0x0: switch ((cmd >> 23) & 0x3f) { case 0x0: return 1; /* MI_NOOP */ case 0x4: return 1; /* MI_FLUSH */ default: return 0; /* disallow everything else */ } break; case 0x1: return 0; /* reserved */ case 0x2: return (cmd & 0xff) + 2; /* 2d commands */ case 0x3: if (((cmd >> 24) & 0x1f) <= 0x18) return 1; switch ((cmd >> 24) & 0x1f) { case 0x1c: return 1; case 0x1d: switch ((cmd >> 16) & 0xff) { case 0x3: return (cmd & 0x1f) + 2; case 0x4: return (cmd & 0xf) + 2; default: return (cmd & 0xffff) + 2; } case 0x1e: if (cmd & (1 << 23)) return (cmd & 0xffff) + 1; else return 1; case 0x1f: if ((cmd & (1 << 23)) == 0) /* inline vertices */ return (cmd & 0x1ffff) + 2; else if (cmd & (1 << 17)) /* indirect random */ if ((cmd & 0xffff) == 0) return 0; /* unknown length, too hard */ else return (((cmd & 0xffff) + 1) / 2) + 1; else return 2; /* indirect sequential */ default: return 0; } default: return 0; } return 0; } static int validate_cmd(int cmd) { int ret = do_validate_cmd(cmd); /* printk("validate_cmd( %x ): %d\n", cmd, ret); */ return ret; } static int i915_emit_cmds(struct drm_device *dev, int __user *buffer, int dwords) { drm_i915_private_t *dev_priv = dev->dev_private; int i; if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8) return -EINVAL; BEGIN_LP_RING((dwords+1)&~1); for (i = 0; i < dwords;) { int cmd, sz; if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd))) return -EINVAL; if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords) return -EINVAL; OUT_RING(cmd); while (++i, --sz) { if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd))) { return -EINVAL; } OUT_RING(cmd); } } if (dwords & 1) OUT_RING(0); ADVANCE_LP_RING(); return 0; } int i915_emit_box(struct drm_device * dev, struct drm_clip_rect *boxes, int i, int DR1, int DR4) { struct drm_clip_rect box; if (DRM_COPY_FROM_USER_UNCHECKED(&box, &boxes[i], sizeof(box))) { return -EFAULT; } return (i915_emit_box_p(dev, &box, DR1, DR4)); } int i915_emit_box_p(struct drm_device *dev, struct drm_clip_rect *box, int DR1, int DR4) { drm_i915_private_t *dev_priv = dev->dev_private; int ret; if (box->y2 <= box->y1 || box->x2 <= box->x1 || box->y2 <= 0 || box->x2 <= 0) { DRM_ERROR("Bad box %d,%d..%d,%d\n", box->x1, box->y1, box->x2, box->y2); return -EINVAL; } if (INTEL_INFO(dev)->gen >= 4) { ret = BEGIN_LP_RING(4); if (ret != 0) return (ret); OUT_RING(GFX_OP_DRAWRECT_INFO_I965); OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); OUT_RING(DR4); } else { ret = BEGIN_LP_RING(6); if (ret != 0) return (ret); OUT_RING(GFX_OP_DRAWRECT_INFO); OUT_RING(DR1); OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); OUT_RING(DR4); OUT_RING(0); } ADVANCE_LP_RING(); return 0; } /* XXX: Emitting the counter should really be moved to part of the IRQ * emit. For now, do it in both places: */ static void i915_emit_breadcrumb(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; if (++dev_priv->counter > 0x7FFFFFFFUL) dev_priv->counter = 0; if (dev_priv->sarea_priv) dev_priv->sarea_priv->last_enqueue = dev_priv->counter; if (BEGIN_LP_RING(4) == 0) { OUT_RING(MI_STORE_DWORD_INDEX); OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); OUT_RING(dev_priv->counter); OUT_RING(0); ADVANCE_LP_RING(); } } static int i915_dispatch_cmdbuffer(struct drm_device * dev, drm_i915_cmdbuffer_t * cmd, struct drm_clip_rect *cliprects, void *cmdbuf) { int nbox = cmd->num_cliprects; int i = 0, count, ret; if (cmd->sz & 0x3) { DRM_ERROR("alignment\n"); return -EINVAL; } i915_kernel_lost_context(dev); count = nbox ? nbox : 1; for (i = 0; i < count; i++) { if (i < nbox) { ret = i915_emit_box_p(dev, &cmd->cliprects[i], cmd->DR1, cmd->DR4); if (ret) return ret; } ret = i915_emit_cmds(dev, cmdbuf, cmd->sz / 4); if (ret) return ret; } i915_emit_breadcrumb(dev); return 0; } static int i915_dispatch_batchbuffer(struct drm_device * dev, drm_i915_batchbuffer_t * batch, struct drm_clip_rect *cliprects) { drm_i915_private_t *dev_priv = dev->dev_private; int nbox = batch->num_cliprects; int i, count, ret; if ((batch->start | batch->used) & 0x7) { DRM_ERROR("alignment\n"); return -EINVAL; } i915_kernel_lost_context(dev); count = nbox ? nbox : 1; for (i = 0; i < count; i++) { if (i < nbox) { int ret = i915_emit_box_p(dev, &cliprects[i], batch->DR1, batch->DR4); if (ret) return ret; } if (!IS_I830(dev) && !IS_845G(dev)) { ret = BEGIN_LP_RING(2); if (ret != 0) return (ret); if (INTEL_INFO(dev)->gen >= 4) { OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965); OUT_RING(batch->start); } else { OUT_RING(MI_BATCH_BUFFER_START | (2 << 6)); OUT_RING(batch->start | MI_BATCH_NON_SECURE); } } else { ret = BEGIN_LP_RING(4); if (ret != 0) return (ret); OUT_RING(MI_BATCH_BUFFER); OUT_RING(batch->start | MI_BATCH_NON_SECURE); OUT_RING(batch->start + batch->used - 4); OUT_RING(0); } ADVANCE_LP_RING(); } i915_emit_breadcrumb(dev); return 0; } static int i915_dispatch_flip(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; int ret; if (!dev_priv->sarea_priv) return -EINVAL; DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n", __func__, dev_priv->current_page, dev_priv->sarea_priv->pf_current_page); i915_kernel_lost_context(dev); ret = BEGIN_LP_RING(10); if (ret) return ret; OUT_RING(MI_FLUSH | MI_READ_FLUSH); OUT_RING(0); OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP); OUT_RING(0); if (dev_priv->current_page == 0) { OUT_RING(dev_priv->back_offset); dev_priv->current_page = 1; } else { OUT_RING(dev_priv->front_offset); dev_priv->current_page = 0; } OUT_RING(0); OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP); OUT_RING(0); ADVANCE_LP_RING(); if (++dev_priv->counter > 0x7FFFFFFFUL) dev_priv->counter = 0; if (dev_priv->sarea_priv) dev_priv->sarea_priv->last_enqueue = dev_priv->counter; if (BEGIN_LP_RING(4) == 0) { OUT_RING(MI_STORE_DWORD_INDEX); OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); OUT_RING(dev_priv->counter); OUT_RING(0); ADVANCE_LP_RING(); } dev_priv->sarea_priv->pf_current_page = dev_priv->current_page; return 0; } static int i915_quiescent(struct drm_device *dev) { struct intel_ring_buffer *ring = LP_RING(dev->dev_private); i915_kernel_lost_context(dev); return (intel_wait_ring_idle(ring)); } static int i915_flush_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { int ret; RING_LOCK_TEST_WITH_RETURN(dev, file_priv); DRM_LOCK(dev); ret = i915_quiescent(dev); DRM_UNLOCK(dev); return (ret); } int i915_batchbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_sarea_t *sarea_priv; drm_i915_batchbuffer_t *batch = data; struct drm_clip_rect *cliprects; size_t cliplen; int ret; if (!dev_priv->allow_batchbuffer) { DRM_ERROR("Batchbuffer ioctl disabled\n"); return -EINVAL; } DRM_UNLOCK(dev); DRM_DEBUG("i915 batchbuffer, start %x used %d cliprects %d\n", batch->start, batch->used, batch->num_cliprects); cliplen = batch->num_cliprects * sizeof(struct drm_clip_rect); if (batch->num_cliprects < 0) return -EFAULT; if (batch->num_cliprects != 0) { cliprects = malloc(batch->num_cliprects * sizeof(struct drm_clip_rect), DRM_MEM_DMA, M_WAITOK | M_ZERO); ret = -copyin(batch->cliprects, cliprects, batch->num_cliprects * sizeof(struct drm_clip_rect)); if (ret != 0) { DRM_LOCK(dev); goto fail_free; } } else cliprects = NULL; DRM_LOCK(dev); RING_LOCK_TEST_WITH_RETURN(dev, file_priv); ret = i915_dispatch_batchbuffer(dev, batch, cliprects); sarea_priv = (drm_i915_sarea_t *)dev_priv->sarea_priv; if (sarea_priv) sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); fail_free: free(cliprects, DRM_MEM_DMA); return ret; } int i915_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_sarea_t *sarea_priv; drm_i915_cmdbuffer_t *cmdbuf = data; struct drm_clip_rect *cliprects = NULL; void *batch_data; int ret; DRM_DEBUG("i915 cmdbuffer, buf %p sz %d cliprects %d\n", cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects); if (cmdbuf->num_cliprects < 0) return -EINVAL; DRM_UNLOCK(dev); batch_data = malloc(cmdbuf->sz, DRM_MEM_DMA, M_WAITOK); ret = -copyin(cmdbuf->buf, batch_data, cmdbuf->sz); if (ret != 0) { DRM_LOCK(dev); goto fail_batch_free; } if (cmdbuf->num_cliprects) { cliprects = malloc(cmdbuf->num_cliprects * sizeof(struct drm_clip_rect), DRM_MEM_DMA, M_WAITOK | M_ZERO); ret = -copyin(cmdbuf->cliprects, cliprects, cmdbuf->num_cliprects * sizeof(struct drm_clip_rect)); if (ret != 0) { DRM_LOCK(dev); goto fail_clip_free; } } DRM_LOCK(dev); RING_LOCK_TEST_WITH_RETURN(dev, file_priv); ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects, batch_data); if (ret) { DRM_ERROR("i915_dispatch_cmdbuffer failed\n"); goto fail_clip_free; } sarea_priv = (drm_i915_sarea_t *)dev_priv->sarea_priv; if (sarea_priv) sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); fail_clip_free: free(cliprects, DRM_MEM_DMA); fail_batch_free: free(batch_data, DRM_MEM_DMA); return ret; } static int i915_flip_bufs(struct drm_device *dev, void *data, struct drm_file *file_priv) { int ret; DRM_DEBUG("%s\n", __func__); RING_LOCK_TEST_WITH_RETURN(dev, file_priv); ret = i915_dispatch_flip(dev); return ret; } int i915_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_getparam_t *param = data; int value; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } switch (param->param) { case I915_PARAM_IRQ_ACTIVE: value = dev->irq_enabled ? 1 : 0; break; case I915_PARAM_ALLOW_BATCHBUFFER: value = dev_priv->allow_batchbuffer ? 1 : 0; break; case I915_PARAM_LAST_DISPATCH: value = READ_BREADCRUMB(dev_priv); break; case I915_PARAM_CHIPSET_ID: value = dev->pci_device; break; case I915_PARAM_HAS_GEM: value = 1; break; case I915_PARAM_NUM_FENCES_AVAIL: value = dev_priv->num_fence_regs - dev_priv->fence_reg_start; break; case I915_PARAM_HAS_OVERLAY: value = dev_priv->overlay ? 1 : 0; break; case I915_PARAM_HAS_PAGEFLIPPING: value = 1; break; case I915_PARAM_HAS_EXECBUF2: value = 1; break; case I915_PARAM_HAS_BSD: value = HAS_BSD(dev); break; case I915_PARAM_HAS_BLT: value = HAS_BLT(dev); break; case I915_PARAM_HAS_RELAXED_FENCING: value = 1; break; case I915_PARAM_HAS_COHERENT_RINGS: value = 1; break; case I915_PARAM_HAS_EXEC_CONSTANTS: value = INTEL_INFO(dev)->gen >= 4; break; case I915_PARAM_HAS_RELAXED_DELTA: value = 1; break; case I915_PARAM_HAS_GEN7_SOL_RESET: value = 1; break; case I915_PARAM_HAS_LLC: value = HAS_LLC(dev); break; default: DRM_DEBUG_DRIVER("Unknown parameter %d\n", param->param); return -EINVAL; } if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) { DRM_ERROR("DRM_COPY_TO_USER failed\n"); return -EFAULT; } return 0; } static int i915_setparam(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_setparam_t *param = data; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } switch (param->param) { case I915_SETPARAM_USE_MI_BATCHBUFFER_START: break; case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY: dev_priv->tex_lru_log_granularity = param->value; break; case I915_SETPARAM_ALLOW_BATCHBUFFER: dev_priv->allow_batchbuffer = param->value; break; case I915_SETPARAM_NUM_USED_FENCES: if (param->value > dev_priv->num_fence_regs || param->value < 0) return -EINVAL; /* Userspace can use first N regs */ dev_priv->fence_reg_start = param->value; break; default: DRM_DEBUG("unknown parameter %d\n", param->param); return -EINVAL; } return 0; } static int i915_set_status_page(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_hws_addr_t *hws = data; struct intel_ring_buffer *ring = LP_RING(dev_priv); if (!I915_NEED_GFX_HWS(dev)) return -EINVAL; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } DRM_DEBUG("set status page addr 0x%08x\n", (u32)hws->addr); if (drm_core_check_feature(dev, DRIVER_MODESET)) { DRM_ERROR("tried to set status page when mode setting active\n"); return 0; } ring->status_page.gfx_addr = dev_priv->status_gfx_addr = hws->addr & (0x1ffff<<12); dev_priv->hws_map.offset = dev->agp->base + hws->addr; dev_priv->hws_map.size = 4*1024; dev_priv->hws_map.type = 0; dev_priv->hws_map.flags = 0; dev_priv->hws_map.mtrr = 0; drm_core_ioremap_wc(&dev_priv->hws_map, dev); if (dev_priv->hws_map.virtual == NULL) { i915_dma_cleanup(dev); ring->status_page.gfx_addr = dev_priv->status_gfx_addr = 0; DRM_ERROR("can not ioremap virtual address for" " G33 hw status page\n"); return -ENOMEM; } ring->status_page.page_addr = dev_priv->hw_status_page = dev_priv->hws_map.virtual; memset(dev_priv->hw_status_page, 0, PAGE_SIZE); I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr); DRM_DEBUG("load hws HWS_PGA with gfx mem 0x%x\n", dev_priv->status_gfx_addr); DRM_DEBUG("load hws at %p\n", dev_priv->hw_status_page); return 0; } static bool intel_enable_ppgtt(struct drm_device *dev) { if (i915_enable_ppgtt >= 0) return i915_enable_ppgtt; /* Disable ppgtt on SNB if VT-d is on. */ if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled) return false; return true; } static int i915_load_gem_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long prealloc_size, gtt_size, mappable_size; int ret; prealloc_size = dev_priv->mm.gtt.stolen_size; gtt_size = dev_priv->mm.gtt.gtt_total_entries << PAGE_SHIFT; mappable_size = dev_priv->mm.gtt.gtt_mappable_entries << PAGE_SHIFT; /* Basic memrange allocator for stolen space */ drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size); DRM_LOCK(dev); if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { /* PPGTT pdes are stolen from global gtt ptes, so shrink the * aperture accordingly when using aliasing ppgtt. */ gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; /* For paranoia keep the guard page in between. */ gtt_size -= PAGE_SIZE; i915_gem_do_init(dev, 0, mappable_size, gtt_size); ret = i915_gem_init_aliasing_ppgtt(dev); if (ret) { DRM_UNLOCK(dev); return ret; } } else { /* Let GEM Manage all of the aperture. * * However, leave one page at the end still bound to the scratch * page. There are a number of places where the hardware * apparently prefetches past the end of the object, and we've * seen multiple hangs with the GPU head pointer stuck in a * batchbuffer bound at the last page of the aperture. One page * should be enough to keep any prefetching inside of the * aperture. */ i915_gem_do_init(dev, 0, mappable_size, gtt_size - PAGE_SIZE); } ret = i915_gem_init_hw(dev); DRM_UNLOCK(dev); if (ret != 0) { i915_gem_cleanup_aliasing_ppgtt(dev); return (ret); } #if 0 /* Try to set up FBC with a reasonable compressed buffer size */ if (I915_HAS_FBC(dev) && i915_powersave) { int cfb_size; /* Leave 1M for line length buffer & misc. */ /* Try to get a 32M buffer... */ if (prealloc_size > (36*1024*1024)) cfb_size = 32*1024*1024; else /* fall back to 7/8 of the stolen space */ cfb_size = prealloc_size * 7 / 8; i915_setup_compression(dev, cfb_size); } #endif /* Allow hardware batchbuffers unless told otherwise. */ dev_priv->allow_batchbuffer = 1; return 0; } static int i915_load_modeset_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; ret = intel_parse_bios(dev); if (ret) DRM_INFO("failed to find VBIOS tables\n"); #if 0 intel_register_dsm_handler(); #endif /* IIR "flip pending" bit means done if this bit is set */ if (IS_GEN3(dev) && (I915_READ(ECOSKPD) & ECO_FLIP_DONE)) dev_priv->flip_pending_is_done = true; intel_modeset_init(dev); ret = i915_load_gem_init(dev); if (ret != 0) goto cleanup_gem; intel_modeset_gem_init(dev); ret = drm_irq_install(dev); if (ret) goto cleanup_gem; dev->vblank_disable_allowed = 1; ret = intel_fbdev_init(dev); if (ret) goto cleanup_gem; drm_kms_helper_poll_init(dev); /* We're off and running w/KMS */ dev_priv->mm.suspended = 0; return (0); cleanup_gem: DRM_LOCK(dev); i915_gem_cleanup_ringbuffer(dev); DRM_UNLOCK(dev); i915_gem_cleanup_aliasing_ppgtt(dev); return (ret); } static int i915_get_bridge_dev(struct drm_device *dev) { struct drm_i915_private *dev_priv; dev_priv = dev->dev_private; dev_priv->bridge_dev = intel_gtt_get_bridge_device(); if (dev_priv->bridge_dev == NULL) { DRM_ERROR("bridge device not found\n"); return (-1); } return (0); } #define MCHBAR_I915 0x44 #define MCHBAR_I965 0x48 #define MCHBAR_SIZE (4*4096) #define DEVEN_REG 0x54 #define DEVEN_MCHBAR_EN (1 << 28) /* Allocate space for the MCH regs if needed, return nonzero on error */ static int intel_alloc_mchbar_resource(struct drm_device *dev) { drm_i915_private_t *dev_priv; device_t vga; int reg; u32 temp_lo, temp_hi; u64 mchbar_addr, temp; dev_priv = dev->dev_private; reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; if (INTEL_INFO(dev)->gen >= 4) temp_hi = pci_read_config(dev_priv->bridge_dev, reg + 4, 4); else temp_hi = 0; temp_lo = pci_read_config(dev_priv->bridge_dev, reg, 4); mchbar_addr = ((u64)temp_hi << 32) | temp_lo; /* If ACPI doesn't have it, assume we need to allocate it ourselves */ #ifdef XXX_CONFIG_PNP if (mchbar_addr && pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) return 0; #endif /* Get some space for it */ vga = device_get_parent(dev->device); dev_priv->mch_res_rid = 0x100; dev_priv->mch_res = BUS_ALLOC_RESOURCE(device_get_parent(vga), dev->device, SYS_RES_MEMORY, &dev_priv->mch_res_rid, 0, ~0UL, MCHBAR_SIZE, RF_ACTIVE | RF_SHAREABLE); if (dev_priv->mch_res == NULL) { DRM_ERROR("failed mchbar resource alloc\n"); return (-ENOMEM); } if (INTEL_INFO(dev)->gen >= 4) { temp = rman_get_start(dev_priv->mch_res); temp >>= 32; pci_write_config(dev_priv->bridge_dev, reg + 4, temp, 4); } pci_write_config(dev_priv->bridge_dev, reg, rman_get_start(dev_priv->mch_res) & UINT32_MAX, 4); return (0); } static void intel_setup_mchbar(struct drm_device *dev) { drm_i915_private_t *dev_priv; int mchbar_reg; u32 temp; bool enabled; dev_priv = dev->dev_private; mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; dev_priv->mchbar_need_disable = false; if (IS_I915G(dev) || IS_I915GM(dev)) { temp = pci_read_config(dev_priv->bridge_dev, DEVEN_REG, 4); enabled = (temp & DEVEN_MCHBAR_EN) != 0; } else { temp = pci_read_config(dev_priv->bridge_dev, mchbar_reg, 4); enabled = temp & 1; } /* If it's already enabled, don't have to do anything */ if (enabled) { DRM_DEBUG("mchbar already enabled\n"); return; } if (intel_alloc_mchbar_resource(dev)) return; dev_priv->mchbar_need_disable = true; /* Space is allocated or reserved, so enable it. */ if (IS_I915G(dev) || IS_I915GM(dev)) { pci_write_config(dev_priv->bridge_dev, DEVEN_REG, temp | DEVEN_MCHBAR_EN, 4); } else { temp = pci_read_config(dev_priv->bridge_dev, mchbar_reg, 4); pci_write_config(dev_priv->bridge_dev, mchbar_reg, temp | 1, 4); } } static void intel_teardown_mchbar(struct drm_device *dev) { drm_i915_private_t *dev_priv; device_t vga; int mchbar_reg; u32 temp; dev_priv = dev->dev_private; mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; if (dev_priv->mchbar_need_disable) { if (IS_I915G(dev) || IS_I915GM(dev)) { temp = pci_read_config(dev_priv->bridge_dev, DEVEN_REG, 4); temp &= ~DEVEN_MCHBAR_EN; pci_write_config(dev_priv->bridge_dev, DEVEN_REG, temp, 4); } else { temp = pci_read_config(dev_priv->bridge_dev, mchbar_reg, 4); temp &= ~1; pci_write_config(dev_priv->bridge_dev, mchbar_reg, temp, 4); } } if (dev_priv->mch_res != NULL) { vga = device_get_parent(dev->device); BUS_DEACTIVATE_RESOURCE(device_get_parent(vga), dev->device, SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); BUS_RELEASE_RESOURCE(device_get_parent(vga), dev->device, SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); dev_priv->mch_res = NULL; } } int i915_driver_load(struct drm_device *dev, unsigned long flags) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long base, size; int mmio_bar, ret; ret = 0; /* i915 has 4 more counters */ dev->counters += 4; dev->types[6] = _DRM_STAT_IRQ; dev->types[7] = _DRM_STAT_PRIMARY; dev->types[8] = _DRM_STAT_SECONDARY; dev->types[9] = _DRM_STAT_DMA; dev_priv = malloc(sizeof(drm_i915_private_t), DRM_MEM_DRIVER, M_ZERO | M_WAITOK); if (dev_priv == NULL) return -ENOMEM; dev->dev_private = (void *)dev_priv; dev_priv->dev = dev; dev_priv->info = i915_get_device_id(dev->pci_device); if (i915_get_bridge_dev(dev)) { free(dev_priv, DRM_MEM_DRIVER); return (-EIO); } dev_priv->mm.gtt = intel_gtt_get(); /* Add register map (needed for suspend/resume) */ mmio_bar = IS_GEN2(dev) ? 1 : 0; base = drm_get_resource_start(dev, mmio_bar); size = drm_get_resource_len(dev, mmio_bar); ret = drm_addmap(dev, base, size, _DRM_REGISTERS, _DRM_KERNEL | _DRM_DRIVER, &dev_priv->mmio_map); dev_priv->tq = taskqueue_create("915", M_WAITOK, taskqueue_thread_enqueue, &dev_priv->tq); taskqueue_start_threads(&dev_priv->tq, 1, PWAIT, "i915 taskq"); mtx_init(&dev_priv->gt_lock, "915gt", NULL, MTX_DEF); mtx_init(&dev_priv->error_lock, "915err", NULL, MTX_DEF); mtx_init(&dev_priv->error_completion_lock, "915cmp", NULL, MTX_DEF); mtx_init(&dev_priv->rps_lock, "915rps", NULL, MTX_DEF); dev_priv->has_gem = 1; intel_irq_init(dev); intel_setup_mchbar(dev); intel_setup_gmbus(dev); intel_opregion_setup(dev); intel_setup_bios(dev); i915_gem_load(dev); /* Init HWS */ if (!I915_NEED_GFX_HWS(dev)) { ret = i915_init_phys_hws(dev); if (ret != 0) { drm_rmmap(dev, dev_priv->mmio_map); drm_free(dev_priv, sizeof(struct drm_i915_private), DRM_MEM_DRIVER); return ret; } } if (IS_PINEVIEW(dev)) i915_pineview_get_mem_freq(dev); else if (IS_GEN5(dev)) i915_ironlake_get_mem_freq(dev); mtx_init(&dev_priv->irq_lock, "userirq", NULL, MTX_DEF); if (IS_IVYBRIDGE(dev)) dev_priv->num_pipe = 3; else if (IS_MOBILE(dev) || !IS_GEN2(dev)) dev_priv->num_pipe = 2; else dev_priv->num_pipe = 1; ret = drm_vblank_init(dev, dev_priv->num_pipe); if (ret) goto out_gem_unload; /* Start out suspended */ dev_priv->mm.suspended = 1; intel_detect_pch(dev); if (drm_core_check_feature(dev, DRIVER_MODESET)) { DRM_UNLOCK(dev); ret = i915_load_modeset_init(dev); DRM_LOCK(dev); if (ret < 0) { DRM_ERROR("failed to init modeset\n"); goto out_gem_unload; } } intel_opregion_init(dev); callout_init(&dev_priv->hangcheck_timer, 1); callout_reset(&dev_priv->hangcheck_timer, DRM_I915_HANGCHECK_PERIOD, i915_hangcheck_elapsed, dev); if (IS_GEN5(dev)) { mtx_lock(&mchdev_lock); i915_mch_dev = dev_priv; dev_priv->mchdev_lock = &mchdev_lock; mtx_unlock(&mchdev_lock); } return (0); out_gem_unload: /* XXXKIB */ (void) i915_driver_unload_int(dev, true); return (ret); } static int i915_driver_unload_int(struct drm_device *dev, bool locked) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; if (!locked) DRM_LOCK(dev); ret = i915_gpu_idle(dev, true); if (ret) DRM_ERROR("failed to idle hardware: %d\n", ret); if (!locked) DRM_UNLOCK(dev); i915_free_hws(dev); intel_teardown_mchbar(dev); if (locked) DRM_UNLOCK(dev); if (drm_core_check_feature(dev, DRIVER_MODESET)) { intel_fbdev_fini(dev); intel_modeset_cleanup(dev); } /* Free error state after interrupts are fully disabled. */ callout_stop(&dev_priv->hangcheck_timer); callout_drain(&dev_priv->hangcheck_timer); i915_destroy_error_state(dev); intel_opregion_fini(dev); if (locked) DRM_LOCK(dev); if (drm_core_check_feature(dev, DRIVER_MODESET)) { if (!locked) DRM_LOCK(dev); i915_gem_free_all_phys_object(dev); i915_gem_cleanup_ringbuffer(dev); + i915_gem_context_fini(dev); if (!locked) DRM_UNLOCK(dev); i915_gem_cleanup_aliasing_ppgtt(dev); #if 1 KIB_NOTYET(); #else if (I915_HAS_FBC(dev) && i915_powersave) i915_cleanup_compression(dev); #endif drm_mm_takedown(&dev_priv->mm.stolen); intel_cleanup_overlay(dev); if (!I915_NEED_GFX_HWS(dev)) i915_free_hws(dev); } i915_gem_unload(dev); mtx_destroy(&dev_priv->irq_lock); if (dev_priv->tq != NULL) taskqueue_free(dev_priv->tq); bus_generic_detach(dev->device); drm_rmmap(dev, dev_priv->mmio_map); intel_teardown_gmbus(dev); mtx_destroy(&dev_priv->error_lock); mtx_destroy(&dev_priv->error_completion_lock); mtx_destroy(&dev_priv->rps_lock); drm_free(dev->dev_private, sizeof(drm_i915_private_t), DRM_MEM_DRIVER); return (0); } int i915_driver_unload(struct drm_device *dev) { return (i915_driver_unload_int(dev, true)); } int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv) { struct drm_i915_file_private *i915_file_priv; i915_file_priv = malloc(sizeof(*i915_file_priv), DRM_MEM_FILES, M_WAITOK | M_ZERO); mtx_init(&i915_file_priv->mm.lck, "915fp", NULL, MTX_DEF); INIT_LIST_HEAD(&i915_file_priv->mm.request_list); file_priv->driver_priv = i915_file_priv; + drm_gem_names_init(&i915_file_priv->context_idr); + return (0); } void i915_driver_lastclose(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; if (!dev_priv || drm_core_check_feature(dev, DRIVER_MODESET)) { #if 1 KIB_NOTYET(); #else drm_fb_helper_restore(); vga_switcheroo_process_delayed_switch(); #endif return; } i915_gem_lastclose(dev); i915_dma_cleanup(dev); } void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv) { + i915_gem_context_close(dev, file_priv); i915_gem_release(dev, file_priv); } void i915_driver_postclose(struct drm_device *dev, struct drm_file *file_priv) { struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; mtx_destroy(&i915_file_priv->mm.lck); drm_free(i915_file_priv, sizeof(*i915_file_priv), DRM_MEM_FILES); } struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF(DRM_I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_FLUSH, i915_flush_ioctl, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_FLIP, i915_flip_bufs, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_BATCHBUFFER, i915_batchbuffer, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_IRQ_EMIT, i915_irq_emit, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_IRQ_WAIT, i915_irq_wait, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_GETPARAM, i915_getparam, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_ALLOC, drm_noop, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_FREE, drm_noop, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_CMDBUFFER, i915_cmdbuffer, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ), DRM_IOCTL_DEF(DRM_I915_SET_VBLANK_PIPE, i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ), DRM_IOCTL_DEF(DRM_I915_GET_VBLANK_PIPE, i915_vblank_pipe_get, DRM_AUTH ), DRM_IOCTL_DEF(DRM_I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_HWS_ADDR, i915_set_status_page, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_GEM_INIT, i915_gem_init_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH | DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH | DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_PIN, i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_UNPIN, i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH), DRM_IOCTL_DEF(DRM_I915_GEM_ENTERVT, i915_gem_entervt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_GEM_LEAVEVT, i915_gem_leavevt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_I915_GEM_CREATE, i915_gem_create_ioctl, 0), DRM_IOCTL_DEF(DRM_I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_MMAP, i915_gem_mmap_ioctl, 0), DRM_IOCTL_DEF(DRM_I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_SET_TILING, i915_gem_set_tiling, 0), DRM_IOCTL_DEF(DRM_I915_GEM_GET_TILING, i915_gem_get_tiling, 0), DRM_IOCTL_DEF(DRM_I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), DRM_IOCTL_DEF(DRM_I915_GET_SPRITE_COLORKEY, intel_sprite_get_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), + DRM_IOCTL_DEF(DRM_I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_UNLOCKED), + DRM_IOCTL_DEF(DRM_I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_UNLOCKED), }; #ifdef COMPAT_FREEBSD32 extern drm_ioctl_desc_t i915_compat_ioctls[]; extern int i915_compat_ioctls_nr; #endif struct drm_driver_info i915_driver_info = { .driver_features = DRIVER_USE_AGP | DRIVER_REQUIRE_AGP | DRIVER_USE_MTRR | DRIVER_HAVE_IRQ | DRIVER_LOCKLESS_IRQ | DRIVER_GEM /*| DRIVER_MODESET*/, .buf_priv_size = sizeof(drm_i915_private_t), .load = i915_driver_load, .open = i915_driver_open, .unload = i915_driver_unload, .preclose = i915_driver_preclose, .lastclose = i915_driver_lastclose, .postclose = i915_driver_postclose, .device_is_agp = i915_driver_device_is_agp, .gem_init_object = i915_gem_init_object, .gem_free_object = i915_gem_free_object, .gem_pager_ops = &i915_gem_pager_ops, .dumb_create = i915_gem_dumb_create, .dumb_map_offset = i915_gem_mmap_gtt, .dumb_destroy = i915_gem_dumb_destroy, .sysctl_init = i915_sysctl_init, .sysctl_cleanup = i915_sysctl_cleanup, .ioctls = i915_ioctls, #ifdef COMPAT_FREEBSD32 .compat_ioctls = i915_compat_ioctls, .compat_ioctls_nr = &i915_compat_ioctls_nr, #endif .max_ioctl = DRM_ARRAY_SIZE(i915_ioctls), .name = DRIVER_NAME, .desc = DRIVER_DESC, .date = DRIVER_DATE, .major = DRIVER_MAJOR, .minor = DRIVER_MINOR, .patchlevel = DRIVER_PATCHLEVEL, }; /** * Determine if the device really is AGP or not. * * All Intel graphics chipsets are treated as AGP, even if they are really * built-in. * * \param dev The device to be tested. * * \returns * A value of 1 is always retured to indictate every i9x5 is AGP. */ int i915_driver_device_is_agp(struct drm_device * dev) { return 1; } static void i915_pineview_get_mem_freq(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; u32 tmp; tmp = I915_READ(CLKCFG); switch (tmp & CLKCFG_FSB_MASK) { case CLKCFG_FSB_533: dev_priv->fsb_freq = 533; /* 133*4 */ break; case CLKCFG_FSB_800: dev_priv->fsb_freq = 800; /* 200*4 */ break; case CLKCFG_FSB_667: dev_priv->fsb_freq = 667; /* 167*4 */ break; case CLKCFG_FSB_400: dev_priv->fsb_freq = 400; /* 100*4 */ break; } switch (tmp & CLKCFG_MEM_MASK) { case CLKCFG_MEM_533: dev_priv->mem_freq = 533; break; case CLKCFG_MEM_667: dev_priv->mem_freq = 667; break; case CLKCFG_MEM_800: dev_priv->mem_freq = 800; break; } /* detect pineview DDR3 setting */ tmp = I915_READ(CSHRDDR3CTL); dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; } static void i915_ironlake_get_mem_freq(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; u16 ddrpll, csipll; ddrpll = I915_READ16(DDRMPLL1); csipll = I915_READ16(CSIPLL0); switch (ddrpll & 0xff) { case 0xc: dev_priv->mem_freq = 800; break; case 0x10: dev_priv->mem_freq = 1066; break; case 0x14: dev_priv->mem_freq = 1333; break; case 0x18: dev_priv->mem_freq = 1600; break; default: DRM_DEBUG("unknown memory frequency 0x%02x\n", ddrpll & 0xff); dev_priv->mem_freq = 0; break; } dev_priv->r_t = dev_priv->mem_freq; switch (csipll & 0x3ff) { case 0x00c: dev_priv->fsb_freq = 3200; break; case 0x00e: dev_priv->fsb_freq = 3733; break; case 0x010: dev_priv->fsb_freq = 4266; break; case 0x012: dev_priv->fsb_freq = 4800; break; case 0x014: dev_priv->fsb_freq = 5333; break; case 0x016: dev_priv->fsb_freq = 5866; break; case 0x018: dev_priv->fsb_freq = 6400; break; default: DRM_DEBUG("unknown fsb frequency 0x%04x\n", csipll & 0x3ff); dev_priv->fsb_freq = 0; break; } if (dev_priv->fsb_freq == 3200) { dev_priv->c_m = 0; } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) { dev_priv->c_m = 1; } else { dev_priv->c_m = 2; } } static const struct cparams { u16 i; u16 t; u16 m; u16 c; } cparams[] = { { 1, 1333, 301, 28664 }, { 1, 1066, 294, 24460 }, { 1, 800, 294, 25192 }, { 0, 1333, 276, 27605 }, { 0, 1066, 276, 27605 }, { 0, 800, 231, 23784 }, }; unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) { u64 total_count, diff, ret; u32 count1, count2, count3, m = 0, c = 0; unsigned long now = jiffies_to_msecs(jiffies), diff1; int i; diff1 = now - dev_priv->last_time1; /* * sysctl(8) reads the value of sysctl twice in rapid * succession. There is high chance that it happens in the * same timer tick. Use the cached value to not divide by * zero and give the hw a chance to gather more samples. */ if (diff1 <= 10) return (dev_priv->chipset_power); count1 = I915_READ(DMIEC); count2 = I915_READ(DDREC); count3 = I915_READ(CSIEC); total_count = count1 + count2 + count3; /* FIXME: handle per-counter overflow */ if (total_count < dev_priv->last_count1) { diff = ~0UL - dev_priv->last_count1; diff += total_count; } else { diff = total_count - dev_priv->last_count1; } for (i = 0; i < DRM_ARRAY_SIZE(cparams); i++) { if (cparams[i].i == dev_priv->c_m && cparams[i].t == dev_priv->r_t) { m = cparams[i].m; c = cparams[i].c; break; } } diff = diff / diff1; ret = ((m * diff) + c); ret = ret / 10; dev_priv->last_count1 = total_count; dev_priv->last_time1 = now; dev_priv->chipset_power = ret; return (ret); } unsigned long i915_mch_val(struct drm_i915_private *dev_priv) { unsigned long m, x, b; u32 tsfs; tsfs = I915_READ(TSFS); m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT); x = I915_READ8(I915_TR1); b = tsfs & TSFS_INTR_MASK; return ((m * x) / 127) - b; } static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { static const struct v_table { u16 vd; /* in .1 mil */ u16 vm; /* in .1 mil */ } v_table[] = { { 0, 0, }, { 375, 0, }, { 500, 0, }, { 625, 0, }, { 750, 0, }, { 875, 0, }, { 1000, 0, }, { 1125, 0, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4125, 3000, }, { 4250, 3125, }, { 4375, 3250, }, { 4500, 3375, }, { 4625, 3500, }, { 4750, 3625, }, { 4875, 3750, }, { 5000, 3875, }, { 5125, 4000, }, { 5250, 4125, }, { 5375, 4250, }, { 5500, 4375, }, { 5625, 4500, }, { 5750, 4625, }, { 5875, 4750, }, { 6000, 4875, }, { 6125, 5000, }, { 6250, 5125, }, { 6375, 5250, }, { 6500, 5375, }, { 6625, 5500, }, { 6750, 5625, }, { 6875, 5750, }, { 7000, 5875, }, { 7125, 6000, }, { 7250, 6125, }, { 7375, 6250, }, { 7500, 6375, }, { 7625, 6500, }, { 7750, 6625, }, { 7875, 6750, }, { 8000, 6875, }, { 8125, 7000, }, { 8250, 7125, }, { 8375, 7250, }, { 8500, 7375, }, { 8625, 7500, }, { 8750, 7625, }, { 8875, 7750, }, { 9000, 7875, }, { 9125, 8000, }, { 9250, 8125, }, { 9375, 8250, }, { 9500, 8375, }, { 9625, 8500, }, { 9750, 8625, }, { 9875, 8750, }, { 10000, 8875, }, { 10125, 9000, }, { 10250, 9125, }, { 10375, 9250, }, { 10500, 9375, }, { 10625, 9500, }, { 10750, 9625, }, { 10875, 9750, }, { 11000, 9875, }, { 11125, 10000, }, { 11250, 10125, }, { 11375, 10250, }, { 11500, 10375, }, { 11625, 10500, }, { 11750, 10625, }, { 11875, 10750, }, { 12000, 10875, }, { 12125, 11000, }, { 12250, 11125, }, { 12375, 11250, }, { 12500, 11375, }, { 12625, 11500, }, { 12750, 11625, }, { 12875, 11750, }, { 13000, 11875, }, { 13125, 12000, }, { 13250, 12125, }, { 13375, 12250, }, { 13500, 12375, }, { 13625, 12500, }, { 13750, 12625, }, { 13875, 12750, }, { 14000, 12875, }, { 14125, 13000, }, { 14250, 13125, }, { 14375, 13250, }, { 14500, 13375, }, { 14625, 13500, }, { 14750, 13625, }, { 14875, 13750, }, { 15000, 13875, }, { 15125, 14000, }, { 15250, 14125, }, { 15375, 14250, }, { 15500, 14375, }, { 15625, 14500, }, { 15750, 14625, }, { 15875, 14750, }, { 16000, 14875, }, { 16125, 15000, }, }; if (dev_priv->info->is_mobile) return v_table[pxvid].vm; else return v_table[pxvid].vd; } void i915_update_gfx_val(struct drm_i915_private *dev_priv) { struct timespec now, diff1; u64 diff; unsigned long diffms; u32 count; if (dev_priv->info->gen != 5) return; nanotime(&now); diff1 = now; timespecsub(&diff1, &dev_priv->last_time2); /* Don't divide by 0 */ diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000; if (!diffms) return; count = I915_READ(GFXEC); if (count < dev_priv->last_count2) { diff = ~0UL - dev_priv->last_count2; diff += count; } else { diff = count - dev_priv->last_count2; } dev_priv->last_count2 = count; dev_priv->last_time2 = now; /* More magic constants... */ diff = diff * 1181; diff = diff / (diffms * 10); dev_priv->gfx_power = diff; } unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) { unsigned long t, corr, state1, corr2, state2; u32 pxvid, ext_v; pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->cur_delay * 4)); pxvid = (pxvid >> 24) & 0x7f; ext_v = pvid_to_extvid(dev_priv, pxvid); state1 = ext_v; t = i915_mch_val(dev_priv); /* Revel in the empirically derived constants */ /* Correction factor in 1/100000 units */ if (t > 80) corr = ((t * 2349) + 135940); else if (t >= 50) corr = ((t * 964) + 29317); else /* < 50 */ corr = ((t * 301) + 1004); corr = corr * ((150142 * state1) / 10000 - 78642); corr /= 100000; corr2 = (corr * dev_priv->corr); state2 = (corr2 * state1) / 10000; state2 /= 100; /* convert to mW */ i915_update_gfx_val(dev_priv); return dev_priv->gfx_power + state2; } /** * i915_read_mch_val - return value for IPS use * * Calculate and return a value for the IPS driver to use when deciding whether * we have thermal and power headroom to increase CPU or GPU power budget. */ unsigned long i915_read_mch_val(void) { struct drm_i915_private *dev_priv; unsigned long chipset_val, graphics_val, ret = 0; mtx_lock(&mchdev_lock); if (!i915_mch_dev) goto out_unlock; dev_priv = i915_mch_dev; chipset_val = i915_chipset_val(dev_priv); graphics_val = i915_gfx_val(dev_priv); ret = chipset_val + graphics_val; out_unlock: mtx_unlock(&mchdev_lock); return ret; } /** * i915_gpu_raise - raise GPU frequency limit * * Raise the limit; IPS indicates we have thermal headroom. */ bool i915_gpu_raise(void) { struct drm_i915_private *dev_priv; bool ret = true; mtx_lock(&mchdev_lock); if (!i915_mch_dev) { ret = false; goto out_unlock; } dev_priv = i915_mch_dev; if (dev_priv->max_delay > dev_priv->fmax) dev_priv->max_delay--; out_unlock: mtx_unlock(&mchdev_lock); return ret; } /** * i915_gpu_lower - lower GPU frequency limit * * IPS indicates we're close to a thermal limit, so throttle back the GPU * frequency maximum. */ bool i915_gpu_lower(void) { struct drm_i915_private *dev_priv; bool ret = true; mtx_lock(&mchdev_lock); if (!i915_mch_dev) { ret = false; goto out_unlock; } dev_priv = i915_mch_dev; if (dev_priv->max_delay < dev_priv->min_delay) dev_priv->max_delay++; out_unlock: mtx_unlock(&mchdev_lock); return ret; } /** * i915_gpu_busy - indicate GPU business to IPS * * Tell the IPS driver whether or not the GPU is busy. */ bool i915_gpu_busy(void) { struct drm_i915_private *dev_priv; bool ret = false; mtx_lock(&mchdev_lock); if (!i915_mch_dev) goto out_unlock; dev_priv = i915_mch_dev; ret = dev_priv->busy; out_unlock: mtx_unlock(&mchdev_lock); return ret; } /** * i915_gpu_turbo_disable - disable graphics turbo * * Disable graphics turbo by resetting the max frequency and setting the * current frequency to the default. */ bool i915_gpu_turbo_disable(void) { struct drm_i915_private *dev_priv; bool ret = true; mtx_lock(&mchdev_lock); if (!i915_mch_dev) { ret = false; goto out_unlock; } dev_priv = i915_mch_dev; dev_priv->max_delay = dev_priv->fstart; if (!ironlake_set_drps(dev_priv->dev, dev_priv->fstart)) ret = false; out_unlock: mtx_unlock(&mchdev_lock); return ret; } Index: head/sys/dev/drm2/i915/i915_drm.h =================================================================== --- head/sys/dev/drm2/i915/i915_drm.h (revision 271704) +++ head/sys/dev/drm2/i915/i915_drm.h (revision 271705) @@ -1,971 +1,992 @@ /*- * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * */ #include __FBSDID("$FreeBSD$"); #ifndef _I915_DRM_H_ #define _I915_DRM_H_ /* Please note that modifications to all structs defined here are * subject to backwards-compatibility constraints. */ #include /* Each region is a minimum of 16k, and there are at most 255 of them. */ #define I915_NR_TEX_REGIONS 255 /* table size 2k - maximum due to use * of chars for next/prev indices */ #define I915_LOG_MIN_TEX_REGION_SIZE 14 typedef struct _drm_i915_init { enum { I915_INIT_DMA = 0x01, I915_CLEANUP_DMA = 0x02, I915_RESUME_DMA = 0x03, /* Since this struct isn't versioned, just used a new * 'func' code to indicate the presence of dri2 sarea * info. */ I915_INIT_DMA2 = 0x04 } func; unsigned int mmio_offset; int sarea_priv_offset; unsigned int ring_start; unsigned int ring_end; unsigned int ring_size; unsigned int front_offset; unsigned int back_offset; unsigned int depth_offset; unsigned int w; unsigned int h; unsigned int pitch; unsigned int pitch_bits; unsigned int back_pitch; unsigned int depth_pitch; unsigned int cpp; unsigned int chipset; unsigned int sarea_handle; } drm_i915_init_t; typedef struct drm_i915_sarea { struct drm_tex_region texList[I915_NR_TEX_REGIONS + 1]; int last_upload; /* last time texture was uploaded */ int last_enqueue; /* last time a buffer was enqueued */ int last_dispatch; /* age of the most recently dispatched buffer */ int ctxOwner; /* last context to upload state */ int texAge; int pf_enabled; /* is pageflipping allowed? */ int pf_active; int pf_current_page; /* which buffer is being displayed? */ int perf_boxes; /* performance boxes to be displayed */ int width, height; /* screen size in pixels */ drm_handle_t front_handle; int front_offset; int front_size; drm_handle_t back_handle; int back_offset; int back_size; drm_handle_t depth_handle; int depth_offset; int depth_size; drm_handle_t tex_handle; int tex_offset; int tex_size; int log_tex_granularity; int pitch; int rotation; /* 0, 90, 180 or 270 */ int rotated_offset; int rotated_size; int rotated_pitch; int virtualX, virtualY; unsigned int front_tiled; unsigned int back_tiled; unsigned int depth_tiled; unsigned int rotated_tiled; unsigned int rotated2_tiled; int planeA_x; int planeA_y; int planeA_w; int planeA_h; int planeB_x; int planeB_y; int planeB_w; int planeB_h; /* Triple buffering */ drm_handle_t third_handle; int third_offset; int third_size; unsigned int third_tiled; /* buffer object handles for the static buffers. May change * over the lifetime of the client, though it doesn't in our current * implementation. */ unsigned int front_bo_handle; unsigned int back_bo_handle; unsigned int third_bo_handle; unsigned int depth_bo_handle; } drm_i915_sarea_t; /* Driver specific fence types and classes. */ /* The only fence class we support */ #define DRM_I915_FENCE_CLASS_ACCEL 0 /* Fence type that guarantees read-write flush */ #define DRM_I915_FENCE_TYPE_RW 2 /* MI_FLUSH programmed just before the fence */ #define DRM_I915_FENCE_FLAG_FLUSHED 0x01000000 /* Flags for perf_boxes */ #define I915_BOX_RING_EMPTY 0x1 #define I915_BOX_FLIP 0x2 #define I915_BOX_WAIT 0x4 #define I915_BOX_TEXTURE_LOAD 0x8 #define I915_BOX_LOST_CONTEXT 0x10 /* I915 specific ioctls * The device specific ioctl range is 0x40 to 0x79. */ #define DRM_I915_INIT 0x00 #define DRM_I915_FLUSH 0x01 #define DRM_I915_FLIP 0x02 #define DRM_I915_BATCHBUFFER 0x03 #define DRM_I915_IRQ_EMIT 0x04 #define DRM_I915_IRQ_WAIT 0x05 #define DRM_I915_GETPARAM 0x06 #define DRM_I915_SETPARAM 0x07 #define DRM_I915_ALLOC 0x08 #define DRM_I915_FREE 0x09 #define DRM_I915_INIT_HEAP 0x0a #define DRM_I915_CMDBUFFER 0x0b #define DRM_I915_DESTROY_HEAP 0x0c #define DRM_I915_SET_VBLANK_PIPE 0x0d #define DRM_I915_GET_VBLANK_PIPE 0x0e #define DRM_I915_VBLANK_SWAP 0x0f #define DRM_I915_MMIO 0x10 #define DRM_I915_HWS_ADDR 0x11 #define DRM_I915_EXECBUFFER 0x12 #define DRM_I915_GEM_INIT 0x13 #define DRM_I915_GEM_EXECBUFFER 0x14 #define DRM_I915_GEM_PIN 0x15 #define DRM_I915_GEM_UNPIN 0x16 #define DRM_I915_GEM_BUSY 0x17 #define DRM_I915_GEM_THROTTLE 0x18 #define DRM_I915_GEM_ENTERVT 0x19 #define DRM_I915_GEM_LEAVEVT 0x1a #define DRM_I915_GEM_CREATE 0x1b #define DRM_I915_GEM_PREAD 0x1c #define DRM_I915_GEM_PWRITE 0x1d #define DRM_I915_GEM_MMAP 0x1e #define DRM_I915_GEM_SET_DOMAIN 0x1f #define DRM_I915_GEM_SW_FINISH 0x20 #define DRM_I915_GEM_SET_TILING 0x21 #define DRM_I915_GEM_GET_TILING 0x22 #define DRM_I915_GEM_GET_APERTURE 0x23 #define DRM_I915_GEM_MMAP_GTT 0x24 #define DRM_I915_GET_PIPE_FROM_CRTC_ID 0x25 #define DRM_I915_GEM_MADVISE 0x26 #define DRM_I915_OVERLAY_PUT_IMAGE 0x27 #define DRM_I915_OVERLAY_ATTRS 0x28 #define DRM_I915_GEM_EXECBUFFER2 0x29 #define DRM_I915_GET_SPRITE_COLORKEY 0x2a #define DRM_I915_SET_SPRITE_COLORKEY 0x2b +#define DRM_I915_GEM_CONTEXT_CREATE 0x2d +#define DRM_I915_GEM_CONTEXT_DESTROY 0x2e #define DRM_IOCTL_I915_INIT DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t) #define DRM_IOCTL_I915_FLUSH DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH) #define DRM_IOCTL_I915_FLIP DRM_IOW( DRM_COMMAND_BASE + DRM_I915_FLIP, drm_i915_flip_t) #define DRM_IOCTL_I915_BATCHBUFFER DRM_IOW( DRM_COMMAND_BASE + DRM_I915_BATCHBUFFER, drm_i915_batchbuffer_t) #define DRM_IOCTL_I915_IRQ_EMIT DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_IRQ_EMIT, drm_i915_irq_emit_t) #define DRM_IOCTL_I915_IRQ_WAIT DRM_IOW( DRM_COMMAND_BASE + DRM_I915_IRQ_WAIT, drm_i915_irq_wait_t) #define DRM_IOCTL_I915_GETPARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GETPARAM, drm_i915_getparam_t) #define DRM_IOCTL_I915_SETPARAM DRM_IOW( DRM_COMMAND_BASE + DRM_I915_SETPARAM, drm_i915_setparam_t) #define DRM_IOCTL_I915_ALLOC DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_ALLOC, drm_i915_mem_alloc_t) #define DRM_IOCTL_I915_FREE DRM_IOW( DRM_COMMAND_BASE + DRM_I915_FREE, drm_i915_mem_free_t) #define DRM_IOCTL_I915_INIT_HEAP DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT_HEAP, drm_i915_mem_init_heap_t) #define DRM_IOCTL_I915_CMDBUFFER DRM_IOW( DRM_COMMAND_BASE + DRM_I915_CMDBUFFER, drm_i915_cmdbuffer_t) #define DRM_IOCTL_I915_DESTROY_HEAP DRM_IOW( DRM_COMMAND_BASE + DRM_I915_DESTROY_HEAP, drm_i915_mem_destroy_heap_t) #define DRM_IOCTL_I915_SET_VBLANK_PIPE DRM_IOW( DRM_COMMAND_BASE + DRM_I915_SET_VBLANK_PIPE, drm_i915_vblank_pipe_t) #define DRM_IOCTL_I915_GET_VBLANK_PIPE DRM_IOR( DRM_COMMAND_BASE + DRM_I915_GET_VBLANK_PIPE, drm_i915_vblank_pipe_t) #define DRM_IOCTL_I915_VBLANK_SWAP DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_VBLANK_SWAP, drm_i915_vblank_swap_t) #define DRM_IOCTL_I915_MMIO DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_MMIO, drm_i915_mmio) #define DRM_IOCTL_I915_EXECBUFFER DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_EXECBUFFER, struct drm_i915_execbuffer) #define DRM_IOCTL_I915_GEM_INIT DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_INIT, struct drm_i915_gem_init) #define DRM_IOCTL_I915_GEM_EXECBUFFER DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_EXECBUFFER, struct drm_i915_gem_execbuffer) #define DRM_IOCTL_I915_GEM_EXECBUFFER2 DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_EXECBUFFER2, struct drm_i915_gem_execbuffer2) #define DRM_IOCTL_I915_GEM_PIN DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_PIN, struct drm_i915_gem_pin) #define DRM_IOCTL_I915_GEM_UNPIN DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_UNPIN, struct drm_i915_gem_unpin) #define DRM_IOCTL_I915_GEM_BUSY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_BUSY, struct drm_i915_gem_busy) #define DRM_IOCTL_I915_GEM_THROTTLE DRM_IO ( DRM_COMMAND_BASE + DRM_I915_GEM_THROTTLE) #define DRM_IOCTL_I915_GEM_ENTERVT DRM_IO(DRM_COMMAND_BASE + DRM_I915_GEM_ENTERVT) #define DRM_IOCTL_I915_GEM_LEAVEVT DRM_IO(DRM_COMMAND_BASE + DRM_I915_GEM_LEAVEVT) #define DRM_IOCTL_I915_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_CREATE, struct drm_i915_gem_create) #define DRM_IOCTL_I915_GEM_PREAD DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PREAD, struct drm_i915_gem_pread) #define DRM_IOCTL_I915_GEM_PWRITE DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PWRITE, struct drm_i915_gem_pwrite) #define DRM_IOCTL_I915_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP, struct drm_i915_gem_mmap) #define DRM_IOCTL_I915_GEM_MMAP_GTT DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP_GTT, struct drm_i915_gem_mmap_gtt) #define DRM_IOCTL_I915_GEM_SET_DOMAIN DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SET_DOMAIN, struct drm_i915_gem_set_domain) #define DRM_IOCTL_I915_GEM_SW_FINISH DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SW_FINISH, struct drm_i915_gem_sw_finish) #define DRM_IOCTL_I915_GEM_SET_TILING DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_SET_TILING, struct drm_i915_gem_set_tiling) #define DRM_IOCTL_I915_GEM_GET_TILING DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_GET_TILING, struct drm_i915_gem_get_tiling) #define DRM_IOCTL_I915_GEM_GET_APERTURE DRM_IOR (DRM_COMMAND_BASE + DRM_I915_GEM_GET_APERTURE, struct drm_i915_gem_get_aperture) #define DRM_IOCTL_I915_GET_PIPE_FROM_CRTC_ID DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GET_PIPE_FROM_CRTC_ID, struct drm_i915_get_pipe_from_crtc_id) #define DRM_IOCTL_I915_GEM_MADVISE DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MADVISE, struct drm_i915_gem_madvise) #define DRM_IOCTL_I915_OVERLAY_PUT_IMAGE DRM_IOW(DRM_COMMAND_BASE + DRM_IOCTL_I915_OVERLAY_PUT_IMAGE, struct drm_intel_overlay_put_image) #define DRM_IOCTL_I915_OVERLAY_ATTRS DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_OVERLAY_ATTRS, struct drm_intel_overlay_attrs) #define DRM_IOCTL_I915_SET_SPRITE_COLORKEY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_SET_SPRITE_COLORKEY, struct drm_intel_sprite_colorkey) #define DRM_IOCTL_I915_GET_SPRITE_COLORKEY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_SET_SPRITE_COLORKEY, struct drm_intel_sprite_colorkey) +#define DRM_IOCTL_I915_GEM_CONTEXT_CREATE DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_CONTEXT_CREATE, struct drm_i915_gem_context_create) +#define DRM_IOCTL_I915_GEM_CONTEXT_DESTROY DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_CONTEXT_DESTROY, struct drm_i915_gem_context_destroy) /* Asynchronous page flipping: */ typedef struct drm_i915_flip { /* * This is really talking about planes, and we could rename it * except for the fact that some of the duplicated i915_drm.h files * out there check for HAVE_I915_FLIP and so might pick up this * version. */ int pipes; } drm_i915_flip_t; /* Allow drivers to submit batchbuffers directly to hardware, relying * on the security mechanisms provided by hardware. */ typedef struct drm_i915_batchbuffer { int start; /* agp offset */ int used; /* nr bytes in use */ int DR1; /* hw flags for GFX_OP_DRAWRECT_INFO */ int DR4; /* window origin for GFX_OP_DRAWRECT_INFO */ int num_cliprects; /* mulitpass with multiple cliprects? */ struct drm_clip_rect __user *cliprects; /* pointer to userspace cliprects */ } drm_i915_batchbuffer_t; /* As above, but pass a pointer to userspace buffer which can be * validated by the kernel prior to sending to hardware. */ typedef struct _drm_i915_cmdbuffer { char __user *buf; /* pointer to userspace command buffer */ int sz; /* nr bytes in buf */ int DR1; /* hw flags for GFX_OP_DRAWRECT_INFO */ int DR4; /* window origin for GFX_OP_DRAWRECT_INFO */ int num_cliprects; /* mulitpass with multiple cliprects? */ struct drm_clip_rect __user *cliprects; /* pointer to userspace cliprects */ } drm_i915_cmdbuffer_t; /* Userspace can request & wait on irq's: */ typedef struct drm_i915_irq_emit { int __user *irq_seq; } drm_i915_irq_emit_t; typedef struct drm_i915_irq_wait { int irq_seq; } drm_i915_irq_wait_t; /* Ioctl to query kernel params: */ #define I915_PARAM_IRQ_ACTIVE 1 #define I915_PARAM_ALLOW_BATCHBUFFER 2 #define I915_PARAM_LAST_DISPATCH 3 #define I915_PARAM_CHIPSET_ID 4 #define I915_PARAM_HAS_GEM 5 #define I915_PARAM_NUM_FENCES_AVAIL 6 #define I915_PARAM_HAS_OVERLAY 7 #define I915_PARAM_HAS_PAGEFLIPPING 8 #define I915_PARAM_HAS_EXECBUF2 9 #define I915_PARAM_HAS_BSD 10 #define I915_PARAM_HAS_BLT 11 #define I915_PARAM_HAS_RELAXED_FENCING 12 #define I915_PARAM_HAS_COHERENT_RINGS 13 #define I915_PARAM_HAS_EXEC_CONSTANTS 14 #define I915_PARAM_HAS_RELAXED_DELTA 15 #define I915_PARAM_HAS_GEN7_SOL_RESET 16 #define I915_PARAM_HAS_LLC 17 typedef struct drm_i915_getparam { int param; int __user *value; } drm_i915_getparam_t; /* Ioctl to set kernel params: */ #define I915_SETPARAM_USE_MI_BATCHBUFFER_START 1 #define I915_SETPARAM_TEX_LRU_LOG_GRANULARITY 2 #define I915_SETPARAM_ALLOW_BATCHBUFFER 3 #define I915_SETPARAM_NUM_USED_FENCES 4 typedef struct drm_i915_setparam { int param; int value; } drm_i915_setparam_t; /* A memory manager for regions of shared memory: */ #define I915_MEM_REGION_AGP 1 typedef struct drm_i915_mem_alloc { int region; int alignment; int size; int __user *region_offset; /* offset from start of fb or agp */ } drm_i915_mem_alloc_t; typedef struct drm_i915_mem_free { int region; int region_offset; } drm_i915_mem_free_t; typedef struct drm_i915_mem_init_heap { int region; int size; int start; } drm_i915_mem_init_heap_t; /* Allow memory manager to be torn down and re-initialized (eg on * rotate): */ typedef struct drm_i915_mem_destroy_heap { int region; } drm_i915_mem_destroy_heap_t; /* Allow X server to configure which pipes to monitor for vblank signals */ #define DRM_I915_VBLANK_PIPE_A 1 #define DRM_I915_VBLANK_PIPE_B 2 typedef struct drm_i915_vblank_pipe { int pipe; } drm_i915_vblank_pipe_t; /* Schedule buffer swap at given vertical blank: */ typedef struct drm_i915_vblank_swap { drm_drawable_t drawable; enum drm_vblank_seq_type seqtype; unsigned int sequence; } drm_i915_vblank_swap_t; #define I915_MMIO_READ 0 #define I915_MMIO_WRITE 1 #define I915_MMIO_MAY_READ 0x1 #define I915_MMIO_MAY_WRITE 0x2 #define MMIO_REGS_IA_PRIMATIVES_COUNT 0 #define MMIO_REGS_IA_VERTICES_COUNT 1 #define MMIO_REGS_VS_INVOCATION_COUNT 2 #define MMIO_REGS_GS_PRIMITIVES_COUNT 3 #define MMIO_REGS_GS_INVOCATION_COUNT 4 #define MMIO_REGS_CL_PRIMITIVES_COUNT 5 #define MMIO_REGS_CL_INVOCATION_COUNT 6 #define MMIO_REGS_PS_INVOCATION_COUNT 7 #define MMIO_REGS_PS_DEPTH_COUNT 8 typedef struct drm_i915_mmio_entry { unsigned int flag; unsigned int offset; unsigned int size; } drm_i915_mmio_entry_t; typedef struct drm_i915_mmio { unsigned int read_write:1; unsigned int reg:31; void __user *data; } drm_i915_mmio_t; typedef struct drm_i915_hws_addr { uint64_t addr; } drm_i915_hws_addr_t; /* * Relocation header is 4 uint32_ts * 0 - 32 bit reloc count * 1 - 32-bit relocation type * 2-3 - 64-bit user buffer handle ptr for another list of relocs. */ #define I915_RELOC_HEADER 4 /* * type 0 relocation has 4-uint32_t stride * 0 - offset into buffer * 1 - delta to add in * 2 - buffer handle * 3 - reserved (for optimisations later). */ /* * type 1 relocation has 4-uint32_t stride. * Hangs off the first item in the op list. * Performed after all valiations are done. * Try to group relocs into the same relocatee together for * performance reasons. * 0 - offset into buffer * 1 - delta to add in * 2 - buffer index in op list. * 3 - relocatee index in op list. */ #define I915_RELOC_TYPE_0 0 #define I915_RELOC0_STRIDE 4 #define I915_RELOC_TYPE_1 1 #define I915_RELOC1_STRIDE 4 struct drm_i915_op_arg { uint64_t next; uint64_t reloc_ptr; int handled; unsigned int pad64; union { struct drm_bo_op_req req; struct drm_bo_arg_rep rep; } d; }; struct drm_i915_execbuffer { uint64_t ops_list; uint32_t num_buffers; struct drm_i915_batchbuffer batch; drm_context_t context; /* for lockless use in the future */ struct drm_fence_arg fence_arg; }; struct drm_i915_gem_init { /** * Beginning offset in the GTT to be managed by the DRM memory * manager. */ uint64_t gtt_start; /** * Ending offset in the GTT to be managed by the DRM memory * manager. */ uint64_t gtt_end; }; struct drm_i915_gem_create { /** * Requested size for the object. * * The (page-aligned) allocated size for the object will be returned. */ uint64_t size; /** * Returned handle for the object. * * Object handles are nonzero. */ uint32_t handle; uint32_t pad; }; struct drm_i915_gem_pread { /** Handle for the object being read. */ uint32_t handle; uint32_t pad; /** Offset into the object to read from */ uint64_t offset; /** Length of data to read */ uint64_t size; /** Pointer to write the data into. */ uint64_t data_ptr; /* void *, but pointers are not 32/64 compatible */ }; struct drm_i915_gem_pwrite { /** Handle for the object being written to. */ uint32_t handle; uint32_t pad; /** Offset into the object to write to */ uint64_t offset; /** Length of data to write */ uint64_t size; /** Pointer to read the data from. */ uint64_t data_ptr; /* void *, but pointers are not 32/64 compatible */ }; struct drm_i915_gem_mmap { /** Handle for the object being mapped. */ uint32_t handle; uint32_t pad; /** Offset in the object to map. */ uint64_t offset; /** * Length of data to map. * * The value will be page-aligned. */ uint64_t size; /** Returned pointer the data was mapped at */ uint64_t addr_ptr; /* void *, but pointers are not 32/64 compatible */ }; struct drm_i915_gem_mmap_gtt { /** Handle for the object being mapped. */ uint32_t handle; uint32_t pad; /** * Fake offset to use for subsequent mmap call * * This is a fixed-size type for 32/64 compatibility. */ uint64_t offset; }; struct drm_i915_gem_set_domain { /** Handle for the object */ uint32_t handle; /** New read domains */ uint32_t read_domains; /** New write domain */ uint32_t write_domain; }; struct drm_i915_gem_sw_finish { /** Handle for the object */ uint32_t handle; }; struct drm_i915_gem_relocation_entry { /** * Handle of the buffer being pointed to by this relocation entry. * * It's appealing to make this be an index into the mm_validate_entry * list to refer to the buffer, but this allows the driver to create * a relocation list for state buffers and not re-write it per * exec using the buffer. */ uint32_t target_handle; /** * Value to be added to the offset of the target buffer to make up * the relocation entry. */ uint32_t delta; /** Offset in the buffer the relocation entry will be written into */ uint64_t offset; /** * Offset value of the target buffer that the relocation entry was last * written as. * * If the buffer has the same offset as last time, we can skip syncing * and writing the relocation. This value is written back out by * the execbuffer ioctl when the relocation is written. */ uint64_t presumed_offset; /** * Target memory domains read by this operation. */ uint32_t read_domains; /** * Target memory domains written by this operation. * * Note that only one domain may be written by the whole * execbuffer operation, so that where there are conflicts, * the application will get -EINVAL back. */ uint32_t write_domain; }; /** @{ * Intel memory domains * * Most of these just align with the various caches in * the system and are used to flush and invalidate as * objects end up cached in different domains. */ /** CPU cache */ #define I915_GEM_DOMAIN_CPU 0x00000001 /** Render cache, used by 2D and 3D drawing */ #define I915_GEM_DOMAIN_RENDER 0x00000002 /** Sampler cache, used by texture engine */ #define I915_GEM_DOMAIN_SAMPLER 0x00000004 /** Command queue, used to load batch buffers */ #define I915_GEM_DOMAIN_COMMAND 0x00000008 /** Instruction cache, used by shader programs */ #define I915_GEM_DOMAIN_INSTRUCTION 0x00000010 /** Vertex address cache */ #define I915_GEM_DOMAIN_VERTEX 0x00000020 /** GTT domain - aperture and scanout */ #define I915_GEM_DOMAIN_GTT 0x00000040 /** @} */ struct drm_i915_gem_exec_object { /** * User's handle for a buffer to be bound into the GTT for this * operation. */ uint32_t handle; /** Number of relocations to be performed on this buffer */ uint32_t relocation_count; /** * Pointer to array of struct drm_i915_gem_relocation_entry containing * the relocations to be performed in this buffer. */ uint64_t relocs_ptr; /** Required alignment in graphics aperture */ uint64_t alignment; /** * Returned value of the updated offset of the object, for future * presumed_offset writes. */ uint64_t offset; }; struct drm_i915_gem_execbuffer { /** * List of buffers to be validated with their relocations to be * performend on them. * * This is a pointer to an array of struct drm_i915_gem_validate_entry. * * These buffers must be listed in an order such that all relocations * a buffer is performing refer to buffers that have already appeared * in the validate list. */ uint64_t buffers_ptr; uint32_t buffer_count; /** Offset in the batchbuffer to start execution from. */ uint32_t batch_start_offset; /** Bytes used in batchbuffer from batch_start_offset */ uint32_t batch_len; uint32_t DR1; uint32_t DR4; uint32_t num_cliprects; uint64_t cliprects_ptr; /* struct drm_clip_rect *cliprects */ }; struct drm_i915_gem_exec_object2 { /** * User's handle for a buffer to be bound into the GTT for this * operation. */ uint32_t handle; /** Number of relocations to be performed on this buffer */ uint32_t relocation_count; /** * Pointer to array of struct drm_i915_gem_relocation_entry containing * the relocations to be performed in this buffer. */ uint64_t relocs_ptr; /** Required alignment in graphics aperture */ uint64_t alignment; /** * Returned value of the updated offset of the object, for future * presumed_offset writes. */ uint64_t offset; #define EXEC_OBJECT_NEEDS_FENCE (1<<0) uint64_t flags; - uint64_t rsvd1; + uint64_t rsvd1; /* now used for context info */ uint64_t rsvd2; }; struct drm_i915_gem_execbuffer2 { /** * List of gem_exec_object2 structs */ uint64_t buffers_ptr; uint32_t buffer_count; /** Offset in the batchbuffer to start execution from. */ uint32_t batch_start_offset; /** Bytes used in batchbuffer from batch_start_offset */ uint32_t batch_len; uint32_t DR1; uint32_t DR4; uint32_t num_cliprects; /** This is a struct drm_clip_rect *cliprects */ uint64_t cliprects_ptr; #define I915_EXEC_RING_MASK (7<<0) #define I915_EXEC_DEFAULT (0<<0) #define I915_EXEC_RENDER (1<<0) #define I915_EXEC_BSD (2<<0) #define I915_EXEC_BLT (3<<0) /* Used for switching the constants addressing mode on gen4+ RENDER ring. * Gen6+ only supports relative addressing to dynamic state (default) and * absolute addressing. * * These flags are ignored for the BSD and BLT rings. */ #define I915_EXEC_CONSTANTS_MASK (3<<6) #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ uint64_t flags; uint64_t rsvd1; uint64_t rsvd2; }; /** Resets the SO write offset registers for transform feedback on gen7. */ #define I915_EXEC_GEN7_SOL_RESET (1<<8) +#define I915_EXEC_CONTEXT_ID_MASK (0xffffffff) +#define i915_execbuffer2_set_context_id(eb2, context) \ + (eb2).rsvd1 = context & I915_EXEC_CONTEXT_ID_MASK +#define i915_execbuffer2_get_context_id(eb2) \ + ((eb2).rsvd1 & I915_EXEC_CONTEXT_ID_MASK) + struct drm_i915_gem_pin { /** Handle of the buffer to be pinned. */ uint32_t handle; uint32_t pad; /** alignment required within the aperture */ uint64_t alignment; /** Returned GTT offset of the buffer. */ uint64_t offset; }; struct drm_i915_gem_unpin { /** Handle of the buffer to be unpinned. */ uint32_t handle; uint32_t pad; }; struct drm_i915_gem_busy { /** Handle of the buffer to check for busy */ uint32_t handle; /** Return busy status (1 if busy, 0 if idle) */ uint32_t busy; }; #define I915_TILING_NONE 0 #define I915_TILING_X 1 #define I915_TILING_Y 2 #define I915_BIT_6_SWIZZLE_NONE 0 #define I915_BIT_6_SWIZZLE_9 1 #define I915_BIT_6_SWIZZLE_9_10 2 #define I915_BIT_6_SWIZZLE_9_11 3 #define I915_BIT_6_SWIZZLE_9_10_11 4 /* Not seen by userland */ #define I915_BIT_6_SWIZZLE_UNKNOWN 5 /* Seen by userland. */ #define I915_BIT_6_SWIZZLE_9_17 6 #define I915_BIT_6_SWIZZLE_9_10_17 7 struct drm_i915_gem_set_tiling { /** Handle of the buffer to have its tiling state updated */ uint32_t handle; /** * Tiling mode for the object (I915_TILING_NONE, I915_TILING_X, * I915_TILING_Y). * * This value is to be set on request, and will be updated by the * kernel on successful return with the actual chosen tiling layout. * * The tiling mode may be demoted to I915_TILING_NONE when the system * has bit 6 swizzling that can't be managed correctly by GEM. * * Buffer contents become undefined when changing tiling_mode. */ uint32_t tiling_mode; /** * Stride in bytes for the object when in I915_TILING_X or * I915_TILING_Y. */ uint32_t stride; /** * Returned address bit 6 swizzling required for CPU access through * mmap mapping. */ uint32_t swizzle_mode; }; struct drm_i915_gem_get_tiling { /** Handle of the buffer to get tiling state for. */ uint32_t handle; /** * Current tiling mode for the object (I915_TILING_NONE, I915_TILING_X, * I915_TILING_Y). */ uint32_t tiling_mode; /** * Returned address bit 6 swizzling required for CPU access through * mmap mapping. */ uint32_t swizzle_mode; }; struct drm_i915_gem_get_aperture { /** Total size of the aperture used by i915_gem_execbuffer, in bytes */ uint64_t aper_size; /** * Available space in the aperture used by i915_gem_execbuffer, in * bytes */ uint64_t aper_available_size; }; struct drm_i915_get_pipe_from_crtc_id { /** ID of CRTC being requested **/ uint32_t crtc_id; /** pipe of requested CRTC **/ uint32_t pipe; }; #define I915_MADV_WILLNEED 0 #define I915_MADV_DONTNEED 1 #define I915_MADV_PURGED_INTERNAL 2 /* internal state */ struct drm_i915_gem_madvise { /** Handle of the buffer to change the backing store advice */ uint32_t handle; /* Advice: either the buffer will be needed again in the near future, * or wont be and could be discarded under memory pressure. */ uint32_t madv; /** Whether the backing store still exists. */ uint32_t retained; }; #define I915_OVERLAY_TYPE_MASK 0xff #define I915_OVERLAY_YUV_PLANAR 0x01 #define I915_OVERLAY_YUV_PACKED 0x02 #define I915_OVERLAY_RGB 0x03 #define I915_OVERLAY_DEPTH_MASK 0xff00 #define I915_OVERLAY_RGB24 0x1000 #define I915_OVERLAY_RGB16 0x2000 #define I915_OVERLAY_RGB15 0x3000 #define I915_OVERLAY_YUV422 0x0100 #define I915_OVERLAY_YUV411 0x0200 #define I915_OVERLAY_YUV420 0x0300 #define I915_OVERLAY_YUV410 0x0400 #define I915_OVERLAY_SWAP_MASK 0xff0000 #define I915_OVERLAY_NO_SWAP 0x000000 #define I915_OVERLAY_UV_SWAP 0x010000 #define I915_OVERLAY_Y_SWAP 0x020000 #define I915_OVERLAY_Y_AND_UV_SWAP 0x030000 #define I915_OVERLAY_FLAGS_MASK 0xff000000 #define I915_OVERLAY_ENABLE 0x01000000 struct drm_intel_overlay_put_image { /* various flags and src format description */ uint32_t flags; /* source picture description */ uint32_t bo_handle; /* stride values and offsets are in bytes, buffer relative */ uint16_t stride_Y; /* stride for packed formats */ uint16_t stride_UV; uint32_t offset_Y; /* offset for packet formats */ uint32_t offset_U; uint32_t offset_V; /* in pixels */ uint16_t src_width; uint16_t src_height; /* to compensate the scaling factors for partially covered surfaces */ uint16_t src_scan_width; uint16_t src_scan_height; /* output crtc description */ uint32_t crtc_id; uint16_t dst_x; uint16_t dst_y; uint16_t dst_width; uint16_t dst_height; }; /* flags */ #define I915_OVERLAY_UPDATE_ATTRS (1<<0) #define I915_OVERLAY_UPDATE_GAMMA (1<<1) struct drm_intel_overlay_attrs { uint32_t flags; uint32_t color_key; int32_t brightness; uint32_t contrast; uint32_t saturation; uint32_t gamma0; uint32_t gamma1; uint32_t gamma2; uint32_t gamma3; uint32_t gamma4; uint32_t gamma5; }; /* * Intel sprite handling * * Color keying works with a min/mask/max tuple. Both source and destination * color keying is allowed. * * Source keying: * Sprite pixels within the min & max values, masked against the color channels * specified in the mask field, will be transparent. All other pixels will * be displayed on top of the primary plane. For RGB surfaces, only the min * and mask fields will be used; ranged compares are not allowed. * * Destination keying: * Primary plane pixels that match the min value, masked against the color * channels specified in the mask field, will be replaced by corresponding * pixels from the sprite plane. * * Note that source & destination keying are exclusive; only one can be * active on a given plane. */ #define I915_SET_COLORKEY_NONE (1<<0) /* disable color key matching */ #define I915_SET_COLORKEY_DESTINATION (1<<1) #define I915_SET_COLORKEY_SOURCE (1<<2) struct drm_intel_sprite_colorkey { uint32_t plane_id; uint32_t min_value; uint32_t channel_mask; uint32_t max_value; uint32_t flags; +}; + +struct drm_i915_gem_context_create { + /* output: id of new context*/ + uint32_t ctx_id; + uint32_t pad; +}; + +struct drm_i915_gem_context_destroy { + uint32_t ctx_id; + uint32_t pad; }; #endif /* _I915_DRM_H_ */ Index: head/sys/dev/drm2/i915/i915_drv.c =================================================================== --- head/sys/dev/drm2/i915/i915_drv.c (revision 271704) +++ head/sys/dev/drm2/i915/i915_drv.c (revision 271705) @@ -1,846 +1,870 @@ /* i915_drv.c -- Intel i915 driver -*- linux-c -*- * Created: Wed Feb 14 17:10:04 2001 by gareth@valinux.com */ /*- * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: * Gareth Hughes * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "fb_if.h" /* drv_PCI_IDs comes from drm_pciids.h, generated from drm_pciids.txt. */ static drm_pci_id_list_t i915_pciidlist[] = { i915_PCI_IDS }; static const struct intel_device_info intel_i830_info = { .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .has_overlay = 1, .overlay_needs_physical = 1, }; static const struct intel_device_info intel_845g_info = { .gen = 2, .has_overlay = 1, .overlay_needs_physical = 1, }; static const struct intel_device_info intel_i85x_info = { .gen = 2, .is_i85x = 1, .is_mobile = 1, .cursor_needs_physical = 1, .has_overlay = 1, .overlay_needs_physical = 1, }; static const struct intel_device_info intel_i865g_info = { .gen = 2, .has_overlay = 1, .overlay_needs_physical = 1, }; static const struct intel_device_info intel_i915g_info = { .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .has_overlay = 1, .overlay_needs_physical = 1, }; static const struct intel_device_info intel_i915gm_info = { .gen = 3, .is_mobile = 1, .cursor_needs_physical = 1, .has_overlay = 1, .overlay_needs_physical = 1, .supports_tv = 1, }; static const struct intel_device_info intel_i945g_info = { .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .has_overlay = 1, .overlay_needs_physical = 1, }; static const struct intel_device_info intel_i945gm_info = { .gen = 3, .is_i945gm = 1, .is_mobile = 1, .has_hotplug = 1, .cursor_needs_physical = 1, .has_overlay = 1, .overlay_needs_physical = 1, .supports_tv = 1, }; static const struct intel_device_info intel_i965g_info = { .gen = 4, .is_broadwater = 1, .has_hotplug = 1, .has_overlay = 1, }; static const struct intel_device_info intel_i965gm_info = { .gen = 4, .is_crestline = 1, .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1, .has_overlay = 1, .supports_tv = 1, }; static const struct intel_device_info intel_g33_info = { .gen = 3, .is_g33 = 1, .need_gfx_hws = 1, .has_hotplug = 1, .has_overlay = 1, }; static const struct intel_device_info intel_g45_info = { .gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .has_pipe_cxsr = 1, .has_hotplug = 1, .has_bsd_ring = 1, }; static const struct intel_device_info intel_gm45_info = { .gen = 4, .is_g4x = 1, .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, .has_pipe_cxsr = 1, .has_hotplug = 1, .supports_tv = 1, .has_bsd_ring = 1, }; static const struct intel_device_info intel_pineview_info = { .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .need_gfx_hws = 1, .has_hotplug = 1, .has_overlay = 1, }; static const struct intel_device_info intel_ironlake_d_info = { .gen = 5, .need_gfx_hws = 1, .has_hotplug = 1, .has_bsd_ring = 1, }; static const struct intel_device_info intel_ironlake_m_info = { .gen = 5, .is_mobile = 1, .need_gfx_hws = 1, .has_hotplug = 1, .has_fbc = 0, /* disabled due to buggy hardware */ .has_bsd_ring = 1, }; static const struct intel_device_info intel_sandybridge_d_info = { .gen = 6, .need_gfx_hws = 1, .has_hotplug = 1, .has_bsd_ring = 1, .has_blt_ring = 1, .has_llc = 1, }; static const struct intel_device_info intel_sandybridge_m_info = { .gen = 6, .is_mobile = 1, .need_gfx_hws = 1, .has_hotplug = 1, .has_fbc = 1, .has_bsd_ring = 1, .has_blt_ring = 1, .has_llc = 1, }; static const struct intel_device_info intel_ivybridge_d_info = { .is_ivybridge = 1, .gen = 7, .need_gfx_hws = 1, .has_hotplug = 1, .has_bsd_ring = 1, .has_blt_ring = 1, .has_llc = 1, }; static const struct intel_device_info intel_ivybridge_m_info = { .is_ivybridge = 1, .gen = 7, .is_mobile = 1, .need_gfx_hws = 1, .has_hotplug = 1, .has_fbc = 0, /* FBC is not enabled on Ivybridge mobile yet */ .has_bsd_ring = 1, .has_blt_ring = 1, .has_llc = 1, }; #define INTEL_VGA_DEVICE(id, info_) { \ .device = id, \ .info = info_, \ } static const struct intel_gfx_device_id { int device; const struct intel_device_info *info; } pciidlist[] = { /* aka */ INTEL_VGA_DEVICE(0x3577, &intel_i830_info), INTEL_VGA_DEVICE(0x2562, &intel_845g_info), INTEL_VGA_DEVICE(0x3582, &intel_i85x_info), INTEL_VGA_DEVICE(0x358e, &intel_i85x_info), INTEL_VGA_DEVICE(0x2572, &intel_i865g_info), INTEL_VGA_DEVICE(0x2582, &intel_i915g_info), INTEL_VGA_DEVICE(0x258a, &intel_i915g_info), INTEL_VGA_DEVICE(0x2592, &intel_i915gm_info), INTEL_VGA_DEVICE(0x2772, &intel_i945g_info), INTEL_VGA_DEVICE(0x27a2, &intel_i945gm_info), INTEL_VGA_DEVICE(0x27ae, &intel_i945gm_info), INTEL_VGA_DEVICE(0x2972, &intel_i965g_info), INTEL_VGA_DEVICE(0x2982, &intel_i965g_info), INTEL_VGA_DEVICE(0x2992, &intel_i965g_info), INTEL_VGA_DEVICE(0x29a2, &intel_i965g_info), INTEL_VGA_DEVICE(0x29b2, &intel_g33_info), INTEL_VGA_DEVICE(0x29c2, &intel_g33_info), INTEL_VGA_DEVICE(0x29d2, &intel_g33_info), INTEL_VGA_DEVICE(0x2a02, &intel_i965gm_info), INTEL_VGA_DEVICE(0x2a12, &intel_i965gm_info), INTEL_VGA_DEVICE(0x2a42, &intel_gm45_info), INTEL_VGA_DEVICE(0x2e02, &intel_g45_info), INTEL_VGA_DEVICE(0x2e12, &intel_g45_info), INTEL_VGA_DEVICE(0x2e22, &intel_g45_info), INTEL_VGA_DEVICE(0x2e32, &intel_g45_info), INTEL_VGA_DEVICE(0x2e42, &intel_g45_info), INTEL_VGA_DEVICE(0x2e92, &intel_g45_info), INTEL_VGA_DEVICE(0xa001, &intel_pineview_info), INTEL_VGA_DEVICE(0xa011, &intel_pineview_info), INTEL_VGA_DEVICE(0x0042, &intel_ironlake_d_info), INTEL_VGA_DEVICE(0x0046, &intel_ironlake_m_info), INTEL_VGA_DEVICE(0x0102, &intel_sandybridge_d_info), INTEL_VGA_DEVICE(0x0112, &intel_sandybridge_d_info), INTEL_VGA_DEVICE(0x0122, &intel_sandybridge_d_info), INTEL_VGA_DEVICE(0x0106, &intel_sandybridge_m_info), INTEL_VGA_DEVICE(0x0116, &intel_sandybridge_m_info), INTEL_VGA_DEVICE(0x0126, &intel_sandybridge_m_info), INTEL_VGA_DEVICE(0x010A, &intel_sandybridge_d_info), INTEL_VGA_DEVICE(0x0156, &intel_ivybridge_m_info), /* GT1 mobile */ INTEL_VGA_DEVICE(0x0166, &intel_ivybridge_m_info), /* GT2 mobile */ INTEL_VGA_DEVICE(0x0152, &intel_ivybridge_d_info), /* GT1 desktop */ INTEL_VGA_DEVICE(0x0162, &intel_ivybridge_d_info), /* GT2 desktop */ INTEL_VGA_DEVICE(0x015a, &intel_ivybridge_d_info), /* GT1 server */ INTEL_VGA_DEVICE(0x016a, &intel_ivybridge_d_info), /* GT2 server */ {0, 0} }; static int i915_drm_freeze(struct drm_device *dev) { struct drm_i915_private *dev_priv; int error; dev_priv = dev->dev_private; drm_kms_helper_poll_disable(dev); #if 0 pci_save_state(dev->pdev); #endif DRM_LOCK(dev); /* If KMS is active, we do the leavevt stuff here */ if (drm_core_check_feature(dev, DRIVER_MODESET)) { error = -i915_gem_idle(dev); if (error) { DRM_UNLOCK(dev); device_printf(dev->device, "GEM idle failed, resume might fail\n"); return (error); } drm_irq_uninstall(dev); } i915_save_state(dev); intel_opregion_fini(dev); /* Modeset on resume, not lid events */ dev_priv->modeset_on_lid = 0; DRM_UNLOCK(dev); return 0; } static int i915_suspend(device_t kdev) { struct drm_device *dev; int error; dev = device_get_softc(kdev); if (dev == NULL || dev->dev_private == NULL) { DRM_ERROR("DRM not initialized, aborting suspend.\n"); return -ENODEV; } DRM_DEBUG_KMS("starting suspend\n"); error = i915_drm_freeze(dev); if (error) return (error); error = bus_generic_suspend(kdev); DRM_DEBUG_KMS("finished suspend %d\n", error); return (error); } static int i915_drm_thaw(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int error = 0; DRM_LOCK(dev); if (drm_core_check_feature(dev, DRIVER_MODESET)) { i915_gem_restore_gtt_mappings(dev); } i915_restore_state(dev); intel_opregion_setup(dev); /* KMS EnterVT equivalent */ if (drm_core_check_feature(dev, DRIVER_MODESET)) { dev_priv->mm.suspended = 0; error = i915_gem_init_hw(dev); if (HAS_PCH_SPLIT(dev)) ironlake_init_pch_refclk(dev); DRM_UNLOCK(dev); sx_xlock(&dev->mode_config.mutex); drm_mode_config_reset(dev); sx_xunlock(&dev->mode_config.mutex); drm_irq_install(dev); sx_xlock(&dev->mode_config.mutex); /* Resume the modeset for every activated CRTC */ drm_helper_resume_force_mode(dev); sx_xunlock(&dev->mode_config.mutex); if (IS_IRONLAKE_M(dev)) ironlake_enable_rc6(dev); DRM_LOCK(dev); } intel_opregion_init(dev); dev_priv->modeset_on_lid = 0; DRM_UNLOCK(dev); return error; } static int i915_resume(device_t kdev) { struct drm_device *dev; int ret; dev = device_get_softc(kdev); DRM_DEBUG_KMS("starting resume\n"); #if 0 if (pci_enable_device(dev->pdev)) return -EIO; pci_set_master(dev->pdev); #endif ret = -i915_drm_thaw(dev); if (ret != 0) return (ret); drm_kms_helper_poll_enable(dev); ret = bus_generic_resume(kdev); DRM_DEBUG_KMS("finished resume %d\n", ret); return (ret); } static int i915_probe(device_t kdev) { return drm_probe(kdev, i915_pciidlist); } int i915_modeset; static int i915_attach(device_t kdev) { struct drm_device *dev; dev = device_get_softc(kdev); if (i915_modeset == 1) i915_driver_info.driver_features |= DRIVER_MODESET; dev->driver = &i915_driver_info; return (drm_attach(kdev, i915_pciidlist)); } static struct fb_info * i915_fb_helper_getinfo(device_t kdev) { struct intel_fbdev *ifbdev; drm_i915_private_t *dev_priv; struct drm_device *dev; struct fb_info *info; dev = device_get_softc(kdev); dev_priv = dev->dev_private; ifbdev = dev_priv->fbdev; if (ifbdev == NULL) return (NULL); info = ifbdev->helper.fbdev; return (info); } const struct intel_device_info * i915_get_device_id(int device) { const struct intel_gfx_device_id *did; for (did = &pciidlist[0]; did->device != 0; did++) { if (did->device != device) continue; return (did->info); } return (NULL); } static device_method_t i915_methods[] = { /* Device interface */ DEVMETHOD(device_probe, i915_probe), DEVMETHOD(device_attach, i915_attach), DEVMETHOD(device_suspend, i915_suspend), DEVMETHOD(device_resume, i915_resume), DEVMETHOD(device_detach, drm_detach), /* Framebuffer service methods */ DEVMETHOD(fb_getinfo, i915_fb_helper_getinfo), DEVMETHOD_END }; static driver_t i915_driver = { "drmn", i915_methods, sizeof(struct drm_device) }; extern devclass_t drm_devclass; DRIVER_MODULE_ORDERED(i915kms, vgapci, i915_driver, drm_devclass, 0, 0, SI_ORDER_ANY); MODULE_DEPEND(i915kms, drmn, 1, 1, 1); MODULE_DEPEND(i915kms, agp, 1, 1, 1); MODULE_DEPEND(i915kms, iicbus, 1, 1, 1); MODULE_DEPEND(i915kms, iic, 1, 1, 1); MODULE_DEPEND(i915kms, iicbb, 1, 1, 1); int intel_iommu_enabled = 0; TUNABLE_INT("drm.i915.intel_iommu_enabled", &intel_iommu_enabled); int i915_semaphores = -1; TUNABLE_INT("drm.i915.semaphores", &i915_semaphores); static int i915_try_reset = 1; TUNABLE_INT("drm.i915.try_reset", &i915_try_reset); unsigned int i915_lvds_downclock = 0; TUNABLE_INT("drm.i915.lvds_downclock", &i915_lvds_downclock); int i915_vbt_sdvo_panel_type = -1; TUNABLE_INT("drm.i915.vbt_sdvo_panel_type", &i915_vbt_sdvo_panel_type); unsigned int i915_powersave = 1; TUNABLE_INT("drm.i915.powersave", &i915_powersave); int i915_enable_fbc = 0; TUNABLE_INT("drm.i915.enable_fbc", &i915_enable_fbc); int i915_enable_rc6 = 0; TUNABLE_INT("drm.i915.enable_rc6", &i915_enable_rc6); int i915_panel_use_ssc = -1; TUNABLE_INT("drm.i915.panel_use_ssc", &i915_panel_use_ssc); int i915_panel_ignore_lid = 0; TUNABLE_INT("drm.i915.panel_ignore_lid", &i915_panel_ignore_lid); int i915_modeset = 1; TUNABLE_INT("drm.i915.modeset", &i915_modeset); int i915_enable_ppgtt = -1; TUNABLE_INT("drm.i915.enable_ppgtt", &i915_enable_ppgtt); int i915_enable_hangcheck = 1; TUNABLE_INT("drm.i915.enable_hangcheck", &i915_enable_hangcheck); #define PCI_VENDOR_INTEL 0x8086 #define INTEL_PCH_DEVICE_ID_MASK 0xff00 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 #define INTEL_PCH_CPT_DEVICE_ID_TYPE 0x1c00 #define INTEL_PCH_PPT_DEVICE_ID_TYPE 0x1e00 void intel_detect_pch(struct drm_device *dev) { struct drm_i915_private *dev_priv; device_t pch; uint32_t id; dev_priv = dev->dev_private; pch = pci_find_class(PCIC_BRIDGE, PCIS_BRIDGE_ISA); if (pch != NULL && pci_get_vendor(pch) == PCI_VENDOR_INTEL) { id = pci_get_device(pch) & INTEL_PCH_DEVICE_ID_MASK; if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) { dev_priv->pch_type = PCH_IBX; DRM_DEBUG_KMS("Found Ibex Peak PCH\n"); } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) { dev_priv->pch_type = PCH_CPT; DRM_DEBUG_KMS("Found CougarPoint PCH\n"); } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) { /* PantherPoint is CPT compatible */ dev_priv->pch_type = PCH_CPT; DRM_DEBUG_KMS("Found PatherPoint PCH\n"); } else DRM_DEBUG_KMS("No PCH detected\n"); } else DRM_DEBUG_KMS("No Intel PCI-ISA bridge found\n"); } void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv) { int count; count = 0; while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1)) DELAY(10); I915_WRITE_NOTRACE(FORCEWAKE, 1); POSTING_READ(FORCEWAKE); count = 0; while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0) DELAY(10); } void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv) { int count; count = 0; while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1)) DELAY(10); I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 1); POSTING_READ(FORCEWAKE_MT); count = 0; while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1) == 0) DELAY(10); } void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv) { mtx_lock(&dev_priv->gt_lock); if (dev_priv->forcewake_count++ == 0) dev_priv->display.force_wake_get(dev_priv); mtx_unlock(&dev_priv->gt_lock); } static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv) { u32 gtfifodbg; gtfifodbg = I915_READ_NOTRACE(GTFIFODBG); if ((gtfifodbg & GT_FIFO_CPU_ERROR_MASK) != 0) { printf("MMIO read or write has been dropped %x\n", gtfifodbg); I915_WRITE_NOTRACE(GTFIFODBG, GT_FIFO_CPU_ERROR_MASK); } } void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv) { I915_WRITE_NOTRACE(FORCEWAKE, 0); /* The below doubles as a POSTING_READ */ gen6_gt_check_fifodbg(dev_priv); } void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv) { I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 0); /* The below doubles as a POSTING_READ */ gen6_gt_check_fifodbg(dev_priv); } void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv) { mtx_lock(&dev_priv->gt_lock); if (--dev_priv->forcewake_count == 0) dev_priv->display.force_wake_put(dev_priv); mtx_unlock(&dev_priv->gt_lock); } int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv) { int ret = 0; if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) { int loop = 500; u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES); while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) { DELAY(10); fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES); } if (loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES) { printf("%s loop\n", __func__); ++ret; } dev_priv->gt_fifo_count = fifo; } dev_priv->gt_fifo_count--; return (ret); } static int -i8xx_do_reset(struct drm_device *dev, u8 flags) +i8xx_do_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int onems; if (IS_I85X(dev)) return -ENODEV; onems = hz / 1000; if (onems == 0) onems = 1; I915_WRITE(D_STATE, I915_READ(D_STATE) | DSTATE_GFX_RESET_I830); POSTING_READ(D_STATE); if (IS_I830(dev) || IS_845G(dev)) { I915_WRITE(DEBUG_RESET_I830, DEBUG_RESET_DISPLAY | DEBUG_RESET_RENDER | DEBUG_RESET_FULL); POSTING_READ(DEBUG_RESET_I830); pause("i8xxrst1", onems); I915_WRITE(DEBUG_RESET_I830, 0); POSTING_READ(DEBUG_RESET_I830); } pause("i8xxrst2", onems); I915_WRITE(D_STATE, I915_READ(D_STATE) & ~DSTATE_GFX_RESET_I830); POSTING_READ(D_STATE); return 0; } static int i965_reset_complete(struct drm_device *dev) { u8 gdrst; gdrst = pci_read_config(dev->device, I965_GDRST, 1); return (gdrst & 0x1); } static int -i965_do_reset(struct drm_device *dev, u8 flags) +i965_do_reset(struct drm_device *dev) { u8 gdrst; /* * Set the domains we want to reset (GRDOM/bits 2 and 3) as * well as the reset bit (GR/bit 0). Setting the GR bit * triggers the reset; when done, the hardware will clear it. */ gdrst = pci_read_config(dev->device, I965_GDRST, 1); - pci_write_config(dev->device, I965_GDRST, gdrst | flags | 0x1, 1); + pci_write_config(dev->device, I965_GDRST, + gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE, 1); return (_intel_wait_for(dev, i965_reset_complete(dev), 500, 1, "915rst")); } static int -ironlake_do_reset(struct drm_device *dev, u8 flags) +ironlake_do_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv; u32 gdrst; dev_priv = dev->dev_private; gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR); - I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, gdrst | flags | 0x1); + I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, + gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE); return (_intel_wait_for(dev, (I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1) != 0, 500, 1, "915rst")); } static int -gen6_do_reset(struct drm_device *dev, u8 flags) +gen6_do_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv; int ret; dev_priv = dev->dev_private; /* Hold gt_lock across reset to prevent any register access * with forcewake not set correctly */ mtx_lock(&dev_priv->gt_lock); /* Reset the chip */ /* GEN6_GDRST is not in the gt power well, no need to check * for fifo space for the write or forcewake the chip for * the read */ I915_WRITE_NOTRACE(GEN6_GDRST, GEN6_GRDOM_FULL); /* Spin waiting for the device to ack the reset request */ ret = _intel_wait_for(dev, (I915_READ_NOTRACE(GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500, 1, "915rst"); /* If reset with a user forcewake, try to restore, otherwise turn it off */ if (dev_priv->forcewake_count) dev_priv->display.force_wake_get(dev_priv); else dev_priv->display.force_wake_put(dev_priv); /* Restore fifo count */ dev_priv->gt_fifo_count = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES); mtx_unlock(&dev_priv->gt_lock); return (ret); } +int intel_gpu_reset(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret = -ENODEV; + + switch (INTEL_INFO(dev)->gen) { + case 7: + case 6: + ret = gen6_do_reset(dev); + break; + case 5: + ret = ironlake_do_reset(dev); + break; + case 4: + ret = i965_do_reset(dev); + break; + case 2: + ret = i8xx_do_reset(dev); + break; + } + + /* Also reset the gpu hangman. */ + if (dev_priv->stop_rings) { + DRM_DEBUG("Simulated gpu hang, resetting stop_rings\n"); + dev_priv->stop_rings = 0; + if (ret == -ENODEV) { + DRM_ERROR("Reset not implemented, but ignoring " + "error for simulated gpu hangs\n"); + ret = 0; + } + } + + return ret; +} + int -i915_reset(struct drm_device *dev, u8 flags) +i915_reset(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; /* * We really should only reset the display subsystem if we actually * need to */ bool need_display = true; int ret; if (!i915_try_reset) return (0); if (!sx_try_xlock(&dev->dev_struct_lock)) return (-EBUSY); i915_gem_reset(dev); ret = -ENODEV; if (time_second - dev_priv->last_gpu_reset < 5) { DRM_ERROR("GPU hanging too fast, declaring wedged!\n"); - } else { - switch (INTEL_INFO(dev)->gen) { - case 7: - case 6: - ret = gen6_do_reset(dev, flags); - break; - case 5: - ret = ironlake_do_reset(dev, flags); - break; - case 4: - ret = i965_do_reset(dev, flags); - break; - case 2: - ret = i8xx_do_reset(dev, flags); - break; - } - } + } else + ret = intel_gpu_reset(dev); + dev_priv->last_gpu_reset = time_second; if (ret) { DRM_ERROR("Failed to reset chip.\n"); DRM_UNLOCK(dev); return (ret); } if (drm_core_check_feature(dev, DRIVER_MODESET) || !dev_priv->mm.suspended) { dev_priv->mm.suspended = 0; i915_gem_init_swizzling(dev); dev_priv->rings[RCS].init(&dev_priv->rings[RCS]); if (HAS_BSD(dev)) dev_priv->rings[VCS].init(&dev_priv->rings[VCS]); if (HAS_BLT(dev)) dev_priv->rings[BCS].init(&dev_priv->rings[BCS]); + i915_gem_context_init(dev); i915_gem_init_ppgtt(dev); drm_irq_uninstall(dev); drm_mode_config_reset(dev); DRM_UNLOCK(dev); drm_irq_install(dev); DRM_LOCK(dev); } DRM_UNLOCK(dev); if (need_display) { sx_xlock(&dev->mode_config.mutex); drm_helper_resume_force_mode(dev); sx_xunlock(&dev->mode_config.mutex); } return (0); } #define __i915_read(x, y) \ u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \ u##x val = 0; \ if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ mtx_lock(&dev_priv->gt_lock); \ if (dev_priv->forcewake_count == 0) \ dev_priv->display.force_wake_get(dev_priv); \ val = DRM_READ##y(dev_priv->mmio_map, reg); \ if (dev_priv->forcewake_count == 0) \ dev_priv->display.force_wake_put(dev_priv); \ mtx_unlock(&dev_priv->gt_lock); \ } else { \ val = DRM_READ##y(dev_priv->mmio_map, reg); \ } \ trace_i915_reg_rw(false, reg, val, sizeof(val)); \ return val; \ } __i915_read(8, 8) __i915_read(16, 16) __i915_read(32, 32) __i915_read(64, 64) #undef __i915_read #define __i915_write(x, y) \ void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \ u32 __fifo_ret = 0; \ trace_i915_reg_rw(true, reg, val, sizeof(val)); \ if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ } \ DRM_WRITE##y(dev_priv->mmio_map, reg, val); \ if (__predict_false(__fifo_ret)) { \ gen6_gt_check_fifodbg(dev_priv); \ } \ } __i915_write(8, 8) __i915_write(16, 16) __i915_write(32, 32) __i915_write(64, 64) #undef __i915_write Index: head/sys/dev/drm2/i915/i915_drv.h =================================================================== --- head/sys/dev/drm2/i915/i915_drv.h (revision 271704) +++ head/sys/dev/drm2/i915/i915_drv.h (revision 271705) @@ -1,1488 +1,1524 @@ /* i915_drv.h -- Private header for the I915 driver -*- linux-c -*- */ /* * * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * */ #include __FBSDID("$FreeBSD$"); #ifndef _I915_DRV_H_ #define _I915_DRV_H_ #include #include #include #include #include /* General customization: */ #define DRIVER_AUTHOR "Tungsten Graphics, Inc." #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" #define DRIVER_DATE "20080730" MALLOC_DECLARE(DRM_I915_GEM); enum pipe { PIPE_A = 0, PIPE_B, PIPE_C, I915_MAX_PIPES }; #define pipe_name(p) ((p) + 'A') #define I915_NUM_PIPE 2 enum plane { PLANE_A = 0, PLANE_B, PLANE_C, }; #define plane_name(p) ((p) + 'A') #define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) #define for_each_pipe(p) for ((p) = 0; (p) < dev_priv->num_pipe; (p)++) /* Interface history: * * 1.1: Original. * 1.2: Add Power Management * 1.3: Add vblank support * 1.4: Fix cmdbuffer path, add heap destroy * 1.5: Add vblank pipe configuration * 1.6: - New ioctl for scheduling buffer swaps on vertical blank * - Support vertical blank on secondary display pipe */ #define DRIVER_MAJOR 1 #define DRIVER_MINOR 6 #define DRIVER_PATCHLEVEL 0 #define WATCH_COHERENCY 0 #define WATCH_BUF 0 #define WATCH_EXEC 0 #define WATCH_LRU 0 #define WATCH_RELOC 0 #define WATCH_INACTIVE 0 #define WATCH_PWRITE 0 #define I915_GEM_PHYS_CURSOR_0 1 #define I915_GEM_PHYS_CURSOR_1 2 #define I915_GEM_PHYS_OVERLAY_REGS 3 #define I915_MAX_PHYS_OBJECT (I915_GEM_PHYS_OVERLAY_REGS) struct drm_i915_gem_phys_object { int id; drm_dma_handle_t *handle; struct drm_i915_gem_object *cur_obj; }; struct drm_i915_private; struct drm_i915_display_funcs { void (*dpms)(struct drm_crtc *crtc, int mode); bool (*fbc_enabled)(struct drm_device *dev); void (*enable_fbc)(struct drm_crtc *crtc, unsigned long interval); void (*disable_fbc)(struct drm_device *dev); int (*get_display_clock_speed)(struct drm_device *dev); int (*get_fifo_size)(struct drm_device *dev, int plane); void (*update_wm)(struct drm_device *dev); void (*update_sprite_wm)(struct drm_device *dev, int pipe, uint32_t sprite_width, int pixel_size); int (*crtc_mode_set)(struct drm_crtc *crtc, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode, int x, int y, struct drm_framebuffer *old_fb); void (*write_eld)(struct drm_connector *connector, struct drm_crtc *crtc); void (*fdi_link_train)(struct drm_crtc *crtc); void (*init_clock_gating)(struct drm_device *dev); void (*init_pch_clock_gating)(struct drm_device *dev); int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj); void (*force_wake_get)(struct drm_i915_private *dev_priv); void (*force_wake_put)(struct drm_i915_private *dev_priv); int (*update_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb, int x, int y); /* clock updates for mode set */ /* cursor updates */ /* render clock increase/decrease */ /* display clock increase/decrease */ /* pll clock increase/decrease */ }; struct intel_device_info { u8 gen; u8 is_mobile:1; u8 is_i85x:1; u8 is_i915g:1; u8 is_i945gm:1; u8 is_g33:1; u8 need_gfx_hws:1; u8 is_g4x:1; u8 is_pineview:1; u8 is_broadwater:1; u8 is_crestline:1; u8 is_ivybridge:1; u8 has_fbc:1; u8 has_pipe_cxsr:1; u8 has_hotplug:1; u8 cursor_needs_physical:1; u8 has_overlay:1; u8 overlay_needs_physical:1; u8 supports_tv:1; u8 has_bsd_ring:1; u8 has_blt_ring:1; u8 has_llc:1; }; #define I915_PPGTT_PD_ENTRIES 512 #define I915_PPGTT_PT_ENTRIES 1024 struct i915_hw_ppgtt { unsigned num_pd_entries; vm_page_t *pt_pages; uint32_t pd_offset; vm_paddr_t *pt_dma_addr; vm_paddr_t scratch_page_dma_addr; }; + +/* This must match up with the value previously used for execbuf2.rsvd1. */ +#define DEFAULT_CONTEXT_ID 0 +struct i915_hw_context { + uint32_t id; + bool is_initialized; + struct drm_i915_file_private *file_priv; + struct intel_ring_buffer *ring; + struct drm_i915_gem_object *obj; +}; + enum no_fbc_reason { FBC_NO_OUTPUT, /* no outputs enabled to compress */ FBC_STOLEN_TOO_SMALL, /* not enough space to hold compressed buffers */ FBC_UNSUPPORTED_MODE, /* interlace or doublescanned mode */ FBC_MODE_TOO_LARGE, /* mode too large for compression */ FBC_BAD_PLANE, /* fbc not supported on plane */ FBC_NOT_TILED, /* buffer not tiled */ FBC_MULTIPLE_PIPES, /* more than one pipe active */ FBC_MODULE_PARAM, }; struct mem_block { struct mem_block *next; struct mem_block *prev; int start; int size; struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */ }; struct opregion_header; struct opregion_acpi; struct opregion_swsci; struct opregion_asle; struct intel_opregion { struct opregion_header *header; struct opregion_acpi *acpi; struct opregion_swsci *swsci; struct opregion_asle *asle; void *vbt; u32 *lid_state; }; #define OPREGION_SIZE (8*1024) #define I915_FENCE_REG_NONE -1 #define I915_MAX_NUM_FENCES 16 /* 16 fences + sign bit for FENCE_REG_NONE */ #define I915_MAX_NUM_FENCE_BITS 5 struct drm_i915_fence_reg { struct list_head lru_list; struct drm_i915_gem_object *obj; uint32_t setup_seqno; int pin_count; }; struct sdvo_device_mapping { u8 initialized; u8 dvo_port; u8 slave_addr; u8 dvo_wiring; u8 i2c_pin; u8 ddc_pin; }; enum intel_pch { PCH_IBX, /* Ibexpeak PCH */ PCH_CPT, /* Cougarpoint PCH */ }; #define QUIRK_PIPEA_FORCE (1<<0) #define QUIRK_LVDS_SSC_DISABLE (1<<1) struct intel_fbdev; struct intel_fbc_work; typedef struct drm_i915_private { struct drm_device *dev; device_t *gmbus_bridge; device_t *bbbus_bridge; device_t *gmbus; device_t *bbbus; /** gmbus_sx protects against concurrent usage of the single hw gmbus * controller on different i2c buses. */ struct sx gmbus_sx; int has_gem; int relative_constants_mode; drm_local_map_t *sarea; drm_local_map_t *mmio_map; /** gt_fifo_count and the subsequent register write are synchronized * with dev->struct_mutex. */ unsigned gt_fifo_count; /** forcewake_count is protected by gt_lock */ unsigned forcewake_count; /** gt_lock is also taken in irq contexts. */ struct mtx gt_lock; drm_i915_sarea_t *sarea_priv; /* drm_i915_ring_buffer_t ring; */ struct intel_ring_buffer rings[I915_NUM_RINGS]; uint32_t next_seqno; drm_dma_handle_t *status_page_dmah; void *hw_status_page; dma_addr_t dma_status_page; uint32_t counter; unsigned int status_gfx_addr; drm_local_map_t hws_map; struct drm_gem_object *hws_obj; struct drm_i915_gem_object *pwrctx; struct drm_i915_gem_object *renderctx; unsigned int cpp; int back_offset; int front_offset; int current_page; int page_flipping; atomic_t irq_received; u32 trace_irq_seqno; /** Cached value of IER to avoid reads in updating the bitfield */ u32 pipestat[2]; u32 irq_mask; u32 gt_irq_mask; u32 pch_irq_mask; struct mtx irq_lock; u32 hotplug_supported_mask; int tex_lru_log_granularity; int allow_batchbuffer; unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds; int vblank_pipe; int num_pipe; /* For hangcheck timer */ #define DRM_I915_HANGCHECK_PERIOD ((1500 /* in ms */ * hz) / 1000) int hangcheck_count; uint32_t last_acthd; uint32_t last_acthd_bsd; uint32_t last_acthd_blt; uint32_t last_instdone; uint32_t last_instdone1; struct intel_opregion opregion; /* overlay */ struct intel_overlay *overlay; bool sprite_scaling_enabled; /* LVDS info */ int backlight_level; /* restore backlight to this value */ bool backlight_enabled; struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */ struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */ /* Feature bits from the VBIOS */ unsigned int int_tv_support:1; unsigned int lvds_dither:1; unsigned int lvds_vbt:1; unsigned int int_crt_support:1; unsigned int lvds_use_ssc:1; unsigned int display_clock_mode:1; int lvds_ssc_freq; struct { int rate; int lanes; int preemphasis; int vswing; bool initialized; bool support; int bpp; struct edp_power_seq pps; } edp; bool no_aux_handshake; int crt_ddc_pin; struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */ int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */ int num_fence_regs; /* 8 on pre-965, 16 otherwise */ /* PCH chipset type */ enum intel_pch pch_type; /* Display functions */ struct drm_i915_display_funcs display; unsigned long quirks; /* Register state */ bool modeset_on_lid; u8 saveLBB; u32 saveDSPACNTR; u32 saveDSPBCNTR; u32 saveDSPARB; u32 saveHWS; u32 savePIPEACONF; u32 savePIPEBCONF; u32 savePIPEASRC; u32 savePIPEBSRC; u32 saveFPA0; u32 saveFPA1; u32 saveDPLL_A; u32 saveDPLL_A_MD; u32 saveHTOTAL_A; u32 saveHBLANK_A; u32 saveHSYNC_A; u32 saveVTOTAL_A; u32 saveVBLANK_A; u32 saveVSYNC_A; u32 saveBCLRPAT_A; u32 saveTRANSACONF; u32 saveTRANS_HTOTAL_A; u32 saveTRANS_HBLANK_A; u32 saveTRANS_HSYNC_A; u32 saveTRANS_VTOTAL_A; u32 saveTRANS_VBLANK_A; u32 saveTRANS_VSYNC_A; u32 savePIPEASTAT; u32 saveDSPASTRIDE; u32 saveDSPASIZE; u32 saveDSPAPOS; u32 saveDSPAADDR; u32 saveDSPASURF; u32 saveDSPATILEOFF; u32 savePFIT_PGM_RATIOS; u32 saveBLC_HIST_CTL; u32 saveBLC_PWM_CTL; u32 saveBLC_PWM_CTL2; u32 saveBLC_CPU_PWM_CTL; u32 saveBLC_CPU_PWM_CTL2; u32 saveFPB0; u32 saveFPB1; u32 saveDPLL_B; u32 saveDPLL_B_MD; u32 saveHTOTAL_B; u32 saveHBLANK_B; u32 saveHSYNC_B; u32 saveVTOTAL_B; u32 saveVBLANK_B; u32 saveVSYNC_B; u32 saveBCLRPAT_B; u32 saveTRANSBCONF; u32 saveTRANS_HTOTAL_B; u32 saveTRANS_HBLANK_B; u32 saveTRANS_HSYNC_B; u32 saveTRANS_VTOTAL_B; u32 saveTRANS_VBLANK_B; u32 saveTRANS_VSYNC_B; u32 savePIPEBSTAT; u32 saveDSPBSTRIDE; u32 saveDSPBSIZE; u32 saveDSPBPOS; u32 saveDSPBADDR; u32 saveDSPBSURF; u32 saveDSPBTILEOFF; u32 saveVGA0; u32 saveVGA1; u32 saveVGA_PD; u32 saveVGACNTRL; u32 saveADPA; u32 saveLVDS; u32 savePP_ON_DELAYS; u32 savePP_OFF_DELAYS; u32 saveDVOA; u32 saveDVOB; u32 saveDVOC; u32 savePP_ON; u32 savePP_OFF; u32 savePP_CONTROL; u32 savePP_DIVISOR; u32 savePFIT_CONTROL; u32 save_palette_a[256]; u32 save_palette_b[256]; u32 saveDPFC_CB_BASE; u32 saveFBC_CFB_BASE; u32 saveFBC_LL_BASE; u32 saveFBC_CONTROL; u32 saveFBC_CONTROL2; u32 saveIER; u32 saveIIR; u32 saveIMR; u32 saveDEIER; u32 saveDEIMR; u32 saveGTIER; u32 saveGTIMR; u32 saveFDI_RXA_IMR; u32 saveFDI_RXB_IMR; u32 saveCACHE_MODE_0; u32 saveMI_ARB_STATE; u32 saveSWF0[16]; u32 saveSWF1[16]; u32 saveSWF2[3]; u8 saveMSR; u8 saveSR[8]; u8 saveGR[25]; u8 saveAR_INDEX; u8 saveAR[21]; u8 saveDACMASK; u8 saveCR[37]; uint64_t saveFENCE[I915_MAX_NUM_FENCES]; u32 saveCURACNTR; u32 saveCURAPOS; u32 saveCURABASE; u32 saveCURBCNTR; u32 saveCURBPOS; u32 saveCURBBASE; u32 saveCURSIZE; u32 saveDP_B; u32 saveDP_C; u32 saveDP_D; u32 savePIPEA_GMCH_DATA_M; u32 savePIPEB_GMCH_DATA_M; u32 savePIPEA_GMCH_DATA_N; u32 savePIPEB_GMCH_DATA_N; u32 savePIPEA_DP_LINK_M; u32 savePIPEB_DP_LINK_M; u32 savePIPEA_DP_LINK_N; u32 savePIPEB_DP_LINK_N; u32 saveFDI_RXA_CTL; u32 saveFDI_TXA_CTL; u32 saveFDI_RXB_CTL; u32 saveFDI_TXB_CTL; u32 savePFA_CTL_1; u32 savePFB_CTL_1; u32 savePFA_WIN_SZ; u32 savePFB_WIN_SZ; u32 savePFA_WIN_POS; u32 savePFB_WIN_POS; u32 savePCH_DREF_CONTROL; u32 saveDISP_ARB_CTL; u32 savePIPEA_DATA_M1; u32 savePIPEA_DATA_N1; u32 savePIPEA_LINK_M1; u32 savePIPEA_LINK_N1; u32 savePIPEB_DATA_M1; u32 savePIPEB_DATA_N1; u32 savePIPEB_LINK_M1; u32 savePIPEB_LINK_N1; u32 saveMCHBAR_RENDER_STANDBY; u32 savePCH_PORT_HOTPLUG; struct { /** Memory allocator for GTT stolen memory */ struct drm_mm stolen; /** Memory allocator for GTT */ struct drm_mm gtt_space; /** List of all objects in gtt_space. Used to restore gtt * mappings on resume */ struct list_head gtt_list; /** Usable portion of the GTT for GEM */ unsigned long gtt_start; unsigned long gtt_mappable_end; unsigned long gtt_end; /** PPGTT used for aliasing the PPGTT with the GTT */ struct i915_hw_ppgtt *aliasing_ppgtt; /** * List of objects currently involved in rendering from the * ringbuffer. * * Includes buffers having the contents of their GPU caches * flushed, not necessarily primitives. last_rendering_seqno * represents when the rendering involved will be completed. * * A reference is held on the buffer while on this list. */ struct list_head active_list; /** * List of objects which are not in the ringbuffer but which * still have a write_domain which needs to be flushed before * unbinding. * * A reference is held on the buffer while on this list. */ struct list_head flushing_list; /** * LRU list of objects which are not in the ringbuffer and * are ready to unbind, but are still in the GTT. * * last_rendering_seqno is 0 while an object is in this list. * * A reference is not held on the buffer while on this list, * as merely being GTT-bound shouldn't prevent its being * freed, and we'll pull it off the list in the free path. */ struct list_head inactive_list; /** * LRU list of objects which are not in the ringbuffer but * are still pinned in the GTT. */ struct list_head pinned_list; /** LRU list of objects with fence regs on them. */ struct list_head fence_list; /** * List of objects currently pending being freed. * * These objects are no longer in use, but due to a signal * we were prevented from freeing them at the appointed time. */ struct list_head deferred_free_list; /** * We leave the user IRQ off as much as possible, * but this means that requests will finish and never * be retired once the system goes idle. Set a timer to * fire periodically while the ring is running. When it * fires, go retire requests. */ struct timeout_task retire_task; /** * Are we in a non-interruptible section of code like * modesetting? */ bool interruptible; uint32_t next_gem_seqno; /** * Waiting sequence number, if any */ uint32_t waiting_gem_seqno; /** * Last seq seen at irq time */ uint32_t irq_gem_seqno; /** * Flag if the X Server, and thus DRM, is not currently in * control of the device. * * This is set between LeaveVT and EnterVT. It needs to be * replaced with a semaphore. It also needs to be * transitioned away from for kernel modesetting. */ int suspended; /** * Flag if the hardware appears to be wedged. * * This is set when attempts to idle the device timeout. * It prevents command submission from occuring and makes * every pending request fail */ int wedged; /** Bit 6 swizzling required for X tiling */ uint32_t bit_6_swizzle_x; /** Bit 6 swizzling required for Y tiling */ uint32_t bit_6_swizzle_y; /* storage for physical objects */ struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT]; /* accounting, useful for userland debugging */ size_t gtt_total; size_t mappable_gtt_total; size_t object_memory; u32 object_count; struct intel_gtt gtt; eventhandler_tag i915_lowmem; } mm; const struct intel_device_info *info; struct sdvo_device_mapping sdvo_mappings[2]; /* indicate whether the LVDS_BORDER should be enabled or not */ unsigned int lvds_border_bits; /* Panel fitter placement and size for Ironlake+ */ u32 pch_pf_pos, pch_pf_size; struct drm_crtc *plane_to_crtc_mapping[3]; struct drm_crtc *pipe_to_crtc_mapping[3]; /* wait_queue_head_t pending_flip_queue; XXXKIB */ bool flip_pending_is_done; /* Reclocking support */ bool render_reclock_avail; bool lvds_downclock_avail; /* indicates the reduced downclock for LVDS*/ int lvds_downclock; struct task idle_task; struct callout idle_callout; bool busy; u16 orig_clock; int child_dev_num; struct child_device_config *child_dev; struct drm_connector *int_lvds_connector; struct drm_connector *int_edp_connector; device_t bridge_dev; bool mchbar_need_disable; int mch_res_rid; struct resource *mch_res; struct mtx rps_lock; u32 pm_iir; struct task rps_task; u8 cur_delay; u8 min_delay; u8 max_delay; u8 fmax; u8 fstart; u64 last_count1; unsigned long last_time1; unsigned long chipset_power; u64 last_count2; struct timespec last_time2; unsigned long gfx_power; int c_m; int r_t; u8 corr; struct mtx *mchdev_lock; enum no_fbc_reason no_fbc_reason; + unsigned int stop_rings; + unsigned long cfb_size; unsigned int cfb_fb; int cfb_plane; int cfb_y; struct intel_fbc_work *fbc_work; unsigned int fsb_freq, mem_freq, is_ddr3; struct taskqueue *tq; struct task error_task; struct task hotplug_task; int error_completion; struct mtx error_completion_lock; struct drm_i915_error_state *first_error; struct mtx error_lock; struct callout hangcheck_timer; unsigned long last_gpu_reset; struct intel_fbdev *fbdev; struct drm_property *broadcast_rgb_property; struct drm_property *force_audio_property; + + bool hw_contexts_disabled; + uint32_t hw_context_size; } drm_i915_private_t; +/* Iterate over initialised rings */ +#define for_each_ring(ring__, dev_priv__, i__) \ + for ((i__) = 0; (i__) < I915_NUM_RINGS; (i__)++) \ + if (((ring__) = &(dev_priv__)->rings[(i__)]), intel_ring_initialized((ring__))) + enum hdmi_force_audio { HDMI_AUDIO_OFF_DVI = -2, /* no aux data for HDMI-DVI converter */ HDMI_AUDIO_OFF, /* force turn off HDMI audio */ HDMI_AUDIO_AUTO, /* trust EDID */ HDMI_AUDIO_ON, /* force turn on HDMI audio */ }; enum i915_cache_level { I915_CACHE_NONE, I915_CACHE_LLC, I915_CACHE_LLC_MLC, /* gen6+ */ }; enum intel_chip_family { CHIP_I8XX = 0x01, CHIP_I9XX = 0x02, CHIP_I915 = 0x04, CHIP_I965 = 0x08, }; /** driver private structure attached to each drm_gem_object */ struct drm_i915_gem_object { struct drm_gem_object base; /** Current space allocated to this object in the GTT, if any. */ struct drm_mm_node *gtt_space; struct list_head gtt_list; /** This object's place on the active/flushing/inactive lists */ struct list_head ring_list; struct list_head mm_list; /** This object's place on GPU write list */ struct list_head gpu_write_list; /** This object's place in the batchbuffer or on the eviction list */ struct list_head exec_list; /** * This is set if the object is on the active or flushing lists * (has pending rendering), and is not set if it's on inactive (ready * to be unbound). */ unsigned int active:1; /** * This is set if the object has been written to since last bound * to the GTT */ unsigned int dirty:1; /** * This is set if the object has been written to since the last * GPU flush. */ unsigned int pending_gpu_write:1; /** * Fence register bits (if any) for this object. Will be set * as needed when mapped into the GTT. * Protected by dev->struct_mutex. */ signed int fence_reg:I915_MAX_NUM_FENCE_BITS; /** * Advice: are the backing pages purgeable? */ unsigned int madv:2; /** * Current tiling mode for the object. */ unsigned int tiling_mode:2; unsigned int tiling_changed:1; /** How many users have pinned this object in GTT space. The following * users can each hold at most one reference: pwrite/pread, pin_ioctl * (via user_pin_count), execbuffer (objects are not allowed multiple * times for the same batchbuffer), and the framebuffer code. When * switching/pageflipping, the framebuffer code has at most two buffers * pinned per crtc. * * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3 * bits with absolutely no headroom. So use 4 bits. */ unsigned int pin_count:4; #define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf /** * Is the object at the current location in the gtt mappable and * fenceable? Used to avoid costly recalculations. */ unsigned int map_and_fenceable:1; /** * Whether the current gtt mapping needs to be mappable (and isn't just * mappable by accident). Track pin and fault separate for a more * accurate mappable working set. */ unsigned int fault_mappable:1; unsigned int pin_mappable:1; /* * Is the GPU currently using a fence to access this buffer, */ unsigned int pending_fenced_gpu_access:1; unsigned int fenced_gpu_access:1; unsigned int cache_level:2; unsigned int has_aliasing_ppgtt_mapping:1; + unsigned int has_global_gtt_mapping:1; vm_page_t *pages; /** * DMAR support */ struct sglist *sg_list; /** * Used for performing relocations during execbuffer insertion. */ LIST_ENTRY(drm_i915_gem_object) exec_node; unsigned long exec_handle; struct drm_i915_gem_exec_object2 *exec_entry; /** * Current offset of the object in GTT space. * * This is the same as gtt_space->start */ uint32_t gtt_offset; /** Breadcrumb of last rendering to the buffer. */ uint32_t last_rendering_seqno; struct intel_ring_buffer *ring; /** Breadcrumb of last fenced GPU access to the buffer. */ uint32_t last_fenced_seqno; struct intel_ring_buffer *last_fenced_ring; /** Current tiling stride for the object, if it's tiled. */ uint32_t stride; /** Record of address bit 17 of each page at last unbind. */ unsigned long *bit_17; /** * If present, while GEM_DOMAIN_CPU is in the read domain this array * flags which individual pages are valid. */ uint8_t *page_cpu_valid; /** User space pin count and filp owning the pin */ uint32_t user_pin_count; struct drm_file *pin_filp; /** for phy allocated objects */ struct drm_i915_gem_phys_object *phys_obj; /** * Number of crtcs where this object is currently the fb, but * will be page flipped away on the next vblank. When it * reaches 0, dev_priv->pending_flip_queue will be woken up. */ int pending_flip; }; #define to_intel_bo(x) __containerof(x, struct drm_i915_gem_object, base) /** * Request queue structure. * * The request queue allows us to note sequence numbers that have been emitted * and may be associated with active buffers to be retired. * * By keeping this list, we can avoid having to do questionable * sequence-number comparisons on buffer last_rendering_seqnos, and associate * an emission time with seqnos for tracking how far ahead of the GPU we are. */ struct drm_i915_gem_request { /** On Which ring this request was generated */ struct intel_ring_buffer *ring; /** GEM sequence number associated with this request. */ uint32_t seqno; /** Postion in the ringbuffer of the end of the request */ u32 tail; /** Time at which this request was emitted, in jiffies. */ unsigned long emitted_jiffies; /** global list entry for this request */ struct list_head list; struct drm_i915_file_private *file_priv; /** file_priv list entry for this request */ struct list_head client_list; }; struct drm_i915_file_private { struct { struct list_head request_list; struct mtx lck; } mm; + struct drm_gem_names context_idr; }; struct drm_i915_error_state { u32 eir; u32 pgtbl_er; u32 pipestat[I915_MAX_PIPES]; u32 tail[I915_NUM_RINGS]; u32 head[I915_NUM_RINGS]; u32 ipeir[I915_NUM_RINGS]; u32 ipehr[I915_NUM_RINGS]; u32 instdone[I915_NUM_RINGS]; u32 acthd[I915_NUM_RINGS]; u32 semaphore_mboxes[I915_NUM_RINGS][I915_NUM_RINGS - 1]; /* our own tracking of ring head and tail */ u32 cpu_ring_head[I915_NUM_RINGS]; u32 cpu_ring_tail[I915_NUM_RINGS]; u32 error; /* gen6+ */ u32 instpm[I915_NUM_RINGS]; u32 instps[I915_NUM_RINGS]; u32 instdone1; u32 seqno[I915_NUM_RINGS]; u64 bbaddr; u32 fault_reg[I915_NUM_RINGS]; u32 done_reg; u32 faddr[I915_NUM_RINGS]; u64 fence[I915_MAX_NUM_FENCES]; struct timeval time; struct drm_i915_error_ring { struct drm_i915_error_object { int page_count; u32 gtt_offset; u32 *pages[0]; } *ringbuffer, *batchbuffer; struct drm_i915_error_request { long jiffies; u32 seqno; u32 tail; } *requests; int num_requests; } ring[I915_NUM_RINGS]; struct drm_i915_error_buffer { u32 size; u32 name; u32 seqno; u32 gtt_offset; u32 read_domains; u32 write_domain; s32 fence_reg:I915_MAX_NUM_FENCE_BITS; s32 pinned:2; u32 tiling:2; u32 dirty:1; u32 purgeable:1; s32 ring:4; u32 cache_level:2; } *active_bo, *pinned_bo; u32 active_bo_count, pinned_bo_count; struct intel_overlay_error_state *overlay; struct intel_display_error_state *display; }; /** * RC6 is a special power stage which allows the GPU to enter an very * low-voltage mode when idle, using down to 0V while at this stage. This * stage is entered automatically when the GPU is idle when RC6 support is * enabled, and as soon as new workload arises GPU wakes up automatically as well. * * There are different RC6 modes available in Intel GPU, which differentiate * among each other with the latency required to enter and leave RC6 and * voltage consumed by the GPU in different states. * * The combination of the following flags define which states GPU is allowed * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and * RC6pp is deepest RC6. Their support by hardware varies according to the * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one * which brings the most power savings; deeper states save more power, but * require higher latency to switch to and wake up. */ #define INTEL_RC6_ENABLE (1<<0) #define INTEL_RC6p_ENABLE (1<<1) #define INTEL_RC6pp_ENABLE (1<<2) extern int intel_iommu_enabled; extern struct drm_ioctl_desc i915_ioctls[]; extern struct drm_driver_info i915_driver_info; extern struct cdev_pager_ops i915_gem_pager_ops; extern unsigned int i915_fbpercrtc; extern int i915_panel_ignore_lid; extern unsigned int i915_powersave; extern int i915_semaphores; extern unsigned int i915_lvds_downclock; extern int i915_panel_use_ssc; extern int i915_vbt_sdvo_panel_type; extern int i915_enable_rc6; extern int i915_enable_fbc; extern int i915_enable_ppgtt; extern int i915_enable_hangcheck; const struct intel_device_info *i915_get_device_id(int device); -int i915_reset(struct drm_device *dev, u8 flags); +extern int intel_gpu_reset(struct drm_device *dev); +int i915_reset(struct drm_device *dev); /* i915_debug.c */ int i915_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx, struct sysctl_oid *top); void i915_sysctl_cleanup(struct drm_device *dev); /* i915_dma.c */ int i915_batchbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv); extern void i915_kernel_lost_context(struct drm_device * dev); extern int i915_driver_load(struct drm_device *, unsigned long flags); extern int i915_driver_unload(struct drm_device *); extern int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv); extern void i915_driver_lastclose(struct drm_device * dev); extern void i915_driver_preclose(struct drm_device *dev, struct drm_file *file_priv); extern void i915_driver_postclose(struct drm_device *dev, struct drm_file *file_priv); extern int i915_driver_device_is_agp(struct drm_device * dev); extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); extern int i915_emit_box(struct drm_device *dev, struct drm_clip_rect __user *boxes, int i, int DR1, int DR4); int i915_emit_box_p(struct drm_device *dev, struct drm_clip_rect *box, int DR1, int DR4); unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); unsigned long i915_mch_val(struct drm_i915_private *dev_priv); void i915_update_gfx_val(struct drm_i915_private *dev_priv); unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); unsigned long i915_read_mch_val(void); bool i915_gpu_raise(void); bool i915_gpu_lower(void); bool i915_gpu_busy(void); bool i915_gpu_turbo_disable(void); /* i915_irq.c */ extern int i915_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int i915_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv); extern void intel_irq_init(struct drm_device *dev); extern int i915_vblank_pipe_set(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int i915_vblank_pipe_get(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int i915_vblank_swap(struct drm_device *dev, void *data, struct drm_file *file_priv); void intel_enable_asle(struct drm_device *dev); void i915_hangcheck_elapsed(void *context); void i915_handle_error(struct drm_device *dev, bool wedged); void i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask); void i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask); void i915_destroy_error_state(struct drm_device *dev); /* i915_gem.c */ int i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, uint32_t *handle_p); int i915_gem_init_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_execbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_execbuffer2(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_pin_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_unpin_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_entervt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_set_tiling(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); void i915_gem_load(struct drm_device *dev); void i915_gem_unload(struct drm_device *dev); int i915_gem_init_object(struct drm_gem_object *obj); void i915_gem_free_object(struct drm_gem_object *obj); int i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, bool map_and_fenceable); void i915_gem_object_unpin(struct drm_i915_gem_object *obj); int i915_gem_object_unbind(struct drm_i915_gem_object *obj); void i915_gem_lastclose(struct drm_device *dev); uint32_t i915_get_gem_seqno(struct drm_device *dev); static inline void i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) { if (obj->fence_reg != I915_FENCE_REG_NONE) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; dev_priv->fence_regs[obj->fence_reg].pin_count++; } } static inline void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) { if (obj->fence_reg != I915_FENCE_REG_NONE) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; dev_priv->fence_regs[obj->fence_reg].pin_count--; } } void i915_gem_retire_requests(struct drm_device *dev); void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring); void i915_gem_clflush_object(struct drm_i915_gem_object *obj); struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, size_t size); int i915_gem_do_init(struct drm_device *dev, unsigned long start, unsigned long mappable_end, unsigned long end); uint32_t i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, int tiling_mode); int i915_mutex_lock_interruptible(struct drm_device *dev); int i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, struct intel_ring_buffer *pipelined); int i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj); int i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, uint32_t flush_domains); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj); int i915_gem_object_put_fence(struct drm_i915_gem_object *obj); int i915_gem_idle(struct drm_device *dev); int i915_gem_init_hw(struct drm_device *dev); void i915_gem_init_swizzling(struct drm_device *dev); void i915_gem_init_ppgtt(struct drm_device *dev); void i915_gem_cleanup_ringbuffer(struct drm_device *dev); int i915_gpu_idle(struct drm_device *dev, bool do_retire); void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, struct intel_ring_buffer *ring, uint32_t seqno); int i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, struct drm_i915_gem_request *request); int i915_gem_object_get_fence(struct drm_i915_gem_object *obj, struct intel_ring_buffer *pipelined); void i915_gem_reset(struct drm_device *dev); int i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire); int i915_gem_mmap(struct drm_device *dev, uint64_t offset, int prot); int i915_gem_fault(struct drm_device *dev, uint64_t offset, int prot, uint64_t *phys); void i915_gem_release(struct drm_device *dev, struct drm_file *file); int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level); +/* i915_gem_context.c */ +void i915_gem_context_init(struct drm_device *dev); +void i915_gem_context_fini(struct drm_device *dev); +void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); +int i915_switch_context(struct intel_ring_buffer *ring, + struct drm_file *file, int to_id); +int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + void i915_gem_free_all_phys_object(struct drm_device *dev); void i915_gem_detach_phys_object(struct drm_device *dev, struct drm_i915_gem_object *obj); int i915_gem_attach_phys_object(struct drm_device *dev, struct drm_i915_gem_object *obj, int id, int align); int i915_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev, uint32_t handle, uint64_t *offset); int i915_gem_dumb_destroy(struct drm_file *file_priv, struct drm_device *dev, uint32_t handle); /* i915_gem_tiling.c */ void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); /* i915_gem_evict.c */ int i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment, bool mappable); int i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only); int i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only); /* i915_suspend.c */ extern int i915_save_state(struct drm_device *dev); extern int i915_restore_state(struct drm_device *dev); /* intel_iic.c */ extern int intel_setup_gmbus(struct drm_device *dev); extern void intel_teardown_gmbus(struct drm_device *dev); extern void intel_gmbus_set_speed(device_t idev, int speed); extern void intel_gmbus_force_bit(device_t idev, bool force_bit); extern void intel_iic_reset(struct drm_device *dev); /* intel_opregion.c */ int intel_opregion_setup(struct drm_device *dev); extern void intel_opregion_init(struct drm_device *dev); extern void intel_opregion_fini(struct drm_device *dev); extern void intel_opregion_asle_intr(struct drm_device *dev); extern void intel_opregion_gse_intr(struct drm_device *dev); extern void intel_opregion_enable_asle(struct drm_device *dev); /* i915_gem_gtt.c */ int i915_gem_init_aliasing_ppgtt(struct drm_device *dev); void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev); void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_object *obj, enum i915_cache_level cache_level); void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_object *obj); void i915_gem_restore_gtt_mappings(struct drm_device *dev); int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj); void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj); void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level); /* modesetting */ extern void intel_modeset_init(struct drm_device *dev); extern void intel_modeset_gem_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state); extern bool intel_fbc_enabled(struct drm_device *dev); extern void intel_disable_fbc(struct drm_device *dev); extern bool ironlake_set_drps(struct drm_device *dev, u8 val); extern void ironlake_init_pch_refclk(struct drm_device *dev); extern void ironlake_enable_rc6(struct drm_device *dev); extern void gen6_set_rps(struct drm_device *dev, u8 val); extern void intel_detect_pch(struct drm_device *dev); extern int intel_trans_dp_port_sel(struct drm_crtc *crtc); extern void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv); extern void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv); extern void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv); extern void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv); extern struct intel_overlay_error_state *intel_overlay_capture_error_state( struct drm_device *dev); extern void intel_overlay_print_error_state(struct sbuf *m, struct intel_overlay_error_state *error); extern struct intel_display_error_state *intel_display_capture_error_state( struct drm_device *dev); extern void intel_display_print_error_state(struct sbuf *m, struct drm_device *dev, struct intel_display_error_state *error); static inline void trace_i915_reg_rw(boolean_t rw, int reg, uint64_t val, int sz) { CTR4(KTR_DRM_REG, "[%x/%d] %c %x", reg, sz, rw ? "w" : "r", val); } /* On SNB platform, before reading ring registers forcewake bit * must be set to prevent GT core from power down and stale values being * returned. */ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv); void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv); int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv); /* We give fast paths for the really cool registers */ #define NEEDS_FORCE_WAKE(dev_priv, reg) \ (((dev_priv)->info->gen >= 6) && \ ((reg) < 0x40000) && \ ((reg) != FORCEWAKE)) #define __i915_read(x, y) \ u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg); __i915_read(8, 8) __i915_read(16, 16) __i915_read(32, 32) __i915_read(64, 64) #undef __i915_read #define __i915_write(x, y) \ void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val); __i915_write(8, 8) __i915_write(16, 16) __i915_write(32, 32) __i915_write(64, 64) #undef __i915_write #define I915_READ8(reg) i915_read8(dev_priv, (reg)) #define I915_WRITE8(reg, val) i915_write8(dev_priv, (reg), (val)) #define I915_READ16(reg) i915_read16(dev_priv, (reg)) #define I915_WRITE16(reg, val) i915_write16(dev_priv, (reg), (val)) #define I915_READ16_NOTRACE(reg) DRM_READ16(dev_priv->mmio_map, (reg)) #define I915_WRITE16_NOTRACE(reg, val) DRM_WRITE16(dev_priv->mmio_map, (reg), (val)) #define I915_READ(reg) i915_read32(dev_priv, (reg)) #define I915_WRITE(reg, val) i915_write32(dev_priv, (reg), (val)) #define I915_READ_NOTRACE(reg) DRM_READ32(dev_priv->mmio_map, (reg)) #define I915_WRITE_NOTRACE(reg, val) DRM_WRITE32(dev_priv->mmio_map, (reg), (val)) #define I915_WRITE64(reg, val) i915_write64(dev_priv, (reg), (val)) #define I915_READ64(reg) i915_read64(dev_priv, (reg)) #define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg) #define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg) #define I915_VERBOSE 0 #define LP_RING(d) (&((struct drm_i915_private *)(d))->rings[RCS]) #define BEGIN_LP_RING(n) \ intel_ring_begin(LP_RING(dev_priv), (n)) #define OUT_RING(x) \ intel_ring_emit(LP_RING(dev_priv), x) #define ADVANCE_LP_RING() \ intel_ring_advance(LP_RING(dev_priv)) #define RING_LOCK_TEST_WITH_RETURN(dev, file) do { \ if (LP_RING(dev->dev_private)->obj == NULL) \ LOCK_TEST_WITH_RETURN(dev, file); \ } while (0) /** * Reads a dword out of the status page, which is written to from the command * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or * MI_STORE_DATA_IMM. * * The following dwords have a reserved meaning: * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes. * 0x04: ring 0 head pointer * 0x05: ring 1 head pointer (915-class) * 0x06: ring 2 head pointer (915-class) * 0x10-0x1b: Context status DWords (GM45) * 0x1f: Last written status offset. (GM45) * * The area from dword 0x20 to 0x3ff is available for driver usage. */ #define READ_HWSP(dev_priv, reg) (((volatile u32*)(dev_priv->hw_status_page))[reg]) #define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX) #define I915_GEM_HWS_INDEX 0x20 #define I915_BREADCRUMB_INDEX 0x21 #define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info) #define IS_I830(dev) ((dev)->pci_device == 0x3577) #define IS_845G(dev) ((dev)->pci_device == 0x2562) #define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) #define IS_I865G(dev) ((dev)->pci_device == 0x2572) #define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g) #define IS_I915GM(dev) ((dev)->pci_device == 0x2592) #define IS_I945G(dev) ((dev)->pci_device == 0x2772) #define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm) #define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater) #define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline) #define IS_GM45(dev) ((dev)->pci_device == 0x2A42) #define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x) #define IS_PINEVIEW_G(dev) ((dev)->pci_device == 0xa001) #define IS_PINEVIEW_M(dev) ((dev)->pci_device == 0xa011) #define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview) #define IS_G33(dev) (INTEL_INFO(dev)->is_g33) #define IS_IRONLAKE_D(dev) ((dev)->pci_device == 0x0042) #define IS_IRONLAKE_M(dev) ((dev)->pci_device == 0x0046) #define IS_IVYBRIDGE(dev) (INTEL_INFO(dev)->is_ivybridge) #define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) /* XXXKIB LEGACY */ #define IS_I965G(dev) ((dev)->pci_device == 0x2972 || \ (dev)->pci_device == 0x2982 || \ (dev)->pci_device == 0x2992 || \ (dev)->pci_device == 0x29A2 || \ (dev)->pci_device == 0x2A02 || \ (dev)->pci_device == 0x2A12 || \ (dev)->pci_device == 0x2A42 || \ (dev)->pci_device == 0x2E02 || \ (dev)->pci_device == 0x2E12 || \ (dev)->pci_device == 0x2E22 || \ (dev)->pci_device == 0x2E32) #define IS_I965GM(dev) ((dev)->pci_device == 0x2A02) #define IS_IGDG(dev) ((dev)->pci_device == 0xa001) #define IS_IGDGM(dev) ((dev)->pci_device == 0xa011) #define IS_IGD(dev) (IS_IGDG(dev) || IS_IGDGM(dev)) #define IS_I9XX(dev) (IS_I915G(dev) || IS_I915GM(dev) || IS_I945G(dev) || \ IS_I945GM(dev) || IS_I965G(dev) || IS_G33(dev)) /* XXXKIB LEGACY END */ #define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2) #define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3) #define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4) #define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5) #define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6) #define IS_GEN7(dev) (INTEL_INFO(dev)->gen == 7) #define HAS_BSD(dev) (INTEL_INFO(dev)->has_bsd_ring) #define HAS_BLT(dev) (INTEL_INFO(dev)->has_blt_ring) #define HAS_LLC(dev) (INTEL_INFO(dev)->has_llc) #define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws) +#define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 6) #define HAS_ALIASING_PPGTT(dev) (INTEL_INFO(dev)->gen >=6) #define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) #define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte * rows, which changed the alignment requirements and fence programming. */ #define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \ IS_I915GM(dev))) #define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) #define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev)) #define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev)) #define SUPPORTS_EDP(dev) (IS_IRONLAKE_M(dev)) #define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) #define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) /* dsparb controlled by hw only */ #define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev)) #define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2) #define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr) #define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc) #define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev) || IS_IVYBRIDGE(dev)) #define HAS_PIPE_CONTROL(dev) (INTEL_INFO(dev)->gen >= 5) #define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type) #define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT) #define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX) #define PRIMARY_RINGBUFFER_SIZE (128*1024) static inline bool i915_seqno_passed(uint32_t seq1, uint32_t seq2) { return ((int32_t)(seq1 - seq2) >= 0); } u32 i915_gem_next_request_seqno(struct intel_ring_buffer *ring); #endif Index: head/sys/dev/drm2/i915/i915_gem.c =================================================================== --- head/sys/dev/drm2/i915/i915_gem.c (revision 271704) +++ head/sys/dev/drm2/i915/i915_gem.c (revision 271705) @@ -1,3787 +1,3793 @@ /*- * Copyright © 2008 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Eric Anholt * * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include static void i915_gem_object_flush_cpu_write_domain( struct drm_i915_gem_object *obj); static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode); static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, int tiling_mode); static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, unsigned alignment, bool map_and_fenceable); static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, int flags); static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); static void i915_gem_object_set_to_full_cpu_read_domain( struct drm_i915_gem_object *obj); static int i915_gem_object_set_cpu_read_domain_range( struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, uint32_t flush_domains); static void i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg); static void i915_gem_reset_fences(struct drm_device *dev); static void i915_gem_retire_task_handler(void *arg, int pending); static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, uint64_t size, struct drm_file *file_priv); static void i915_gem_lowmem(void *arg); MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); long i915_gem_wired_pages_cnt; static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) { dev_priv->mm.object_count++; dev_priv->mm.object_memory += size; } static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) { dev_priv->mm.object_count--; dev_priv->mm.object_memory -= size; } static int i915_gem_wait_for_error(struct drm_device *dev) { struct drm_i915_private *dev_priv; int ret; dev_priv = dev->dev_private; if (!atomic_load_acq_int(&dev_priv->mm.wedged)) return (0); mtx_lock(&dev_priv->error_completion_lock); while (dev_priv->error_completion == 0) { ret = -msleep(&dev_priv->error_completion, &dev_priv->error_completion_lock, PCATCH, "915wco", 0); if (ret != 0) { mtx_unlock(&dev_priv->error_completion_lock); return (ret); } } mtx_unlock(&dev_priv->error_completion_lock); if (atomic_load_acq_int(&dev_priv->mm.wedged)) { mtx_lock(&dev_priv->error_completion_lock); dev_priv->error_completion++; mtx_unlock(&dev_priv->error_completion_lock); } return (0); } int i915_mutex_lock_interruptible(struct drm_device *dev) { struct drm_i915_private *dev_priv; int ret; dev_priv = dev->dev_private; ret = i915_gem_wait_for_error(dev); if (ret != 0) return (ret); /* * interruptible shall it be. might indeed be if dev_lock is * changed to sx */ ret = sx_xlock_sig(&dev->dev_struct_lock); if (ret != 0) return (-ret); return (0); } static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj) { struct drm_device *dev; drm_i915_private_t *dev_priv; int ret; dev = obj->base.dev; dev_priv = dev->dev_private; ret = i915_gem_object_unbind(obj); if (ret == -ERESTART) { list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); return; } CTR1(KTR_DRM, "object_destroy_tail %p", obj); drm_gem_free_mmap_offset(&obj->base); drm_gem_object_release(&obj->base); i915_gem_info_remove_obj(dev_priv, obj->base.size); free(obj->page_cpu_valid, DRM_I915_GEM); free(obj->bit_17, DRM_I915_GEM); free(obj, DRM_I915_GEM); } void i915_gem_free_object(struct drm_gem_object *gem_obj) { struct drm_i915_gem_object *obj; struct drm_device *dev; obj = to_intel_bo(gem_obj); dev = obj->base.dev; while (obj->pin_count > 0) i915_gem_object_unpin(obj); if (obj->phys_obj != NULL) i915_gem_detach_phys_object(dev, obj); i915_gem_free_object_tail(obj); } static void init_ring_lists(struct intel_ring_buffer *ring) { INIT_LIST_HEAD(&ring->active_list); INIT_LIST_HEAD(&ring->request_list); INIT_LIST_HEAD(&ring->gpu_write_list); } void i915_gem_load(struct drm_device *dev) { drm_i915_private_t *dev_priv; int i; dev_priv = dev->dev_private; INIT_LIST_HEAD(&dev_priv->mm.active_list); INIT_LIST_HEAD(&dev_priv->mm.flushing_list); INIT_LIST_HEAD(&dev_priv->mm.inactive_list); INIT_LIST_HEAD(&dev_priv->mm.pinned_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); INIT_LIST_HEAD(&dev_priv->mm.gtt_list); for (i = 0; i < I915_NUM_RINGS; i++) init_ring_lists(&dev_priv->rings[i]); for (i = 0; i < I915_MAX_NUM_FENCES; i++) INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, i915_gem_retire_task_handler, dev_priv); dev_priv->error_completion = 0; /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ if (IS_GEN3(dev)) { u32 tmp = I915_READ(MI_ARB_STATE); if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { /* * arb state is a masked write, so set bit + * bit in mask. */ tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); I915_WRITE(MI_ARB_STATE, tmp); } } dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; /* Old X drivers will take 0-2 for front, back, depth buffers */ if (!drm_core_check_feature(dev, DRIVER_MODESET)) dev_priv->fence_reg_start = 3; if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) dev_priv->num_fence_regs = 16; else dev_priv->num_fence_regs = 8; /* Initialize fence registers to zero */ for (i = 0; i < dev_priv->num_fence_regs; i++) { i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); } i915_gem_detect_bit_6_swizzle(dev); dev_priv->mm.interruptible = true; dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); } int i915_gem_do_init(struct drm_device *dev, unsigned long start, unsigned long mappable_end, unsigned long end) { drm_i915_private_t *dev_priv; unsigned long mappable; int error; dev_priv = dev->dev_private; mappable = min(end, mappable_end) - start; drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); dev_priv->mm.gtt_start = start; dev_priv->mm.gtt_mappable_end = mappable_end; dev_priv->mm.gtt_end = end; dev_priv->mm.gtt_total = end - start; dev_priv->mm.mappable_gtt_total = mappable; /* Take over this portion of the GTT */ intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); device_printf(dev->device, "taking over the fictitious range 0x%lx-0x%lx\n", dev->agp->base + start, dev->agp->base + start + mappable); error = -vm_phys_fictitious_reg_range(dev->agp->base + start, dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); return (error); } int i915_gem_init_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_init *args; drm_i915_private_t *dev_priv; dev_priv = dev->dev_private; args = data; if (args->gtt_start >= args->gtt_end || (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) return (-EINVAL); if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) return (-EBUSY); /* * XXXKIB. The second-time initialization should be guarded * against. */ return (i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end)); } int i915_gem_idle(struct drm_device *dev) { drm_i915_private_t *dev_priv; int ret; dev_priv = dev->dev_private; if (dev_priv->mm.suspended) return (0); ret = i915_gpu_idle(dev, true); if (ret != 0) return (ret); /* Under UMS, be paranoid and evict. */ if (!drm_core_check_feature(dev, DRIVER_MODESET)) { ret = i915_gem_evict_inactive(dev, false); if (ret != 0) return ret; } i915_gem_reset_fences(dev); /* Hack! Don't let anybody do execbuf while we don't control the chip. * We need to replace this with a semaphore, or something. * And not confound mm.suspended! */ dev_priv->mm.suspended = 1; callout_stop(&dev_priv->hangcheck_timer); i915_kernel_lost_context(dev); i915_gem_cleanup_ringbuffer(dev); /* Cancel the retire work handler, which should be idle now. */ taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); return (ret); } void i915_gem_init_swizzling(struct drm_device *dev) { drm_i915_private_t *dev_priv; dev_priv = dev->dev_private; if (INTEL_INFO(dev)->gen < 5 || dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) return; I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | DISP_TILE_SURFACE_SWIZZLING); if (IS_GEN5(dev)) return; I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); if (IS_GEN6(dev)) I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); else I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); } void i915_gem_init_ppgtt(struct drm_device *dev) { drm_i915_private_t *dev_priv; struct i915_hw_ppgtt *ppgtt; uint32_t pd_offset, pd_entry; vm_paddr_t pt_addr; struct intel_ring_buffer *ring; u_int first_pd_entry_in_global_pt, i; dev_priv = dev->dev_private; ppgtt = dev_priv->mm.aliasing_ppgtt; if (ppgtt == NULL) return; first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; for (i = 0; i < ppgtt->num_pd_entries; i++) { pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); pd_entry |= GEN6_PDE_VALID; intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); } intel_gtt_read_pte(first_pd_entry_in_global_pt); pd_offset = ppgtt->pd_offset; pd_offset /= 64; /* in cachelines, */ pd_offset <<= 16; if (INTEL_INFO(dev)->gen == 6) { uint32_t ecochk = I915_READ(GAM_ECOCHK); I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); } else if (INTEL_INFO(dev)->gen >= 7) { I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); /* GFX_MODE is per-ring on gen7+ */ } for (i = 0; i < I915_NUM_RINGS; i++) { ring = &dev_priv->rings[i]; if (INTEL_INFO(dev)->gen >= 7) I915_WRITE(RING_MODE_GEN7(ring), GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); } } int i915_gem_init_hw(struct drm_device *dev) { drm_i915_private_t *dev_priv; int ret; dev_priv = dev->dev_private; i915_gem_init_swizzling(dev); ret = intel_init_render_ring_buffer(dev); if (ret != 0) return (ret); if (HAS_BSD(dev)) { ret = intel_init_bsd_ring_buffer(dev); if (ret != 0) goto cleanup_render_ring; } if (HAS_BLT(dev)) { ret = intel_init_blt_ring_buffer(dev); if (ret != 0) goto cleanup_bsd_ring; } dev_priv->next_seqno = 1; + i915_gem_context_init(dev); i915_gem_init_ppgtt(dev); return (0); cleanup_bsd_ring: intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); cleanup_render_ring: intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); return (ret); } int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_private *dev_priv; struct drm_i915_gem_get_aperture *args; struct drm_i915_gem_object *obj; size_t pinned; dev_priv = dev->dev_private; args = data; if (!(dev->driver->driver_features & DRIVER_GEM)) return (-ENODEV); pinned = 0; DRM_LOCK(dev); list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) pinned += obj->gtt_space->size; DRM_UNLOCK(dev); args->aper_size = dev_priv->mm.gtt_total; args->aper_available_size = args->aper_size - pinned; return (0); } int i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, bool map_and_fenceable) { struct drm_device *dev; struct drm_i915_private *dev_priv; int ret; dev = obj->base.dev; dev_priv = dev->dev_private; KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, ("Max pin count")); if (obj->gtt_space != NULL) { if ((alignment && obj->gtt_offset & (alignment - 1)) || (map_and_fenceable && !obj->map_and_fenceable)) { DRM_DEBUG("bo is already pinned with incorrect alignment:" " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," " obj->map_and_fenceable=%d\n", obj->gtt_offset, alignment, map_and_fenceable, obj->map_and_fenceable); ret = i915_gem_object_unbind(obj); if (ret != 0) return (ret); } } if (obj->gtt_space == NULL) { ret = i915_gem_object_bind_to_gtt(obj, alignment, map_and_fenceable); if (ret) return (ret); } if (obj->pin_count++ == 0 && !obj->active) list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); obj->pin_mappable |= map_and_fenceable; #if 1 KIB_NOTYET(); #else WARN_ON(i915_verify_lists(dev)); #endif return (0); } void i915_gem_object_unpin(struct drm_i915_gem_object *obj) { struct drm_device *dev; drm_i915_private_t *dev_priv; dev = obj->base.dev; dev_priv = dev->dev_private; #if 1 KIB_NOTYET(); #else WARN_ON(i915_verify_lists(dev)); #endif KASSERT(obj->pin_count != 0, ("zero pin count")); KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); if (--obj->pin_count == 0) { if (!obj->active) list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); obj->pin_mappable = false; } #if 1 KIB_NOTYET(); #else WARN_ON(i915_verify_lists(dev)); #endif } int i915_gem_pin_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pin *args; struct drm_i915_gem_object *obj; struct drm_gem_object *gobj; int ret; args = data; ret = i915_mutex_lock_interruptible(dev); if (ret != 0) return ret; gobj = drm_gem_object_lookup(dev, file, args->handle); if (gobj == NULL) { ret = -ENOENT; goto unlock; } obj = to_intel_bo(gobj); if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to pin a purgeable buffer\n"); ret = -EINVAL; goto out; } if (obj->pin_filp != NULL && obj->pin_filp != file) { DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", args->handle); ret = -EINVAL; goto out; } obj->user_pin_count++; obj->pin_filp = file; if (obj->user_pin_count == 1) { ret = i915_gem_object_pin(obj, args->alignment, true); if (ret != 0) goto out; } /* XXX - flush the CPU caches for pinned objects * as the X server doesn't manage domains yet */ i915_gem_object_flush_cpu_write_domain(obj); args->offset = obj->gtt_offset; out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return (ret); } int i915_gem_unpin_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pin *args; struct drm_i915_gem_object *obj; int ret; args = data; ret = i915_mutex_lock_interruptible(dev); if (ret != 0) return (ret); obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (obj->pin_filp != file) { DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", args->handle); ret = -EINVAL; goto out; } obj->user_pin_count--; if (obj->user_pin_count == 0) { obj->pin_filp = NULL; i915_gem_object_unpin(obj); } out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return (ret); } int i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_busy *args; struct drm_i915_gem_object *obj; struct drm_i915_gem_request *request; int ret; args = data; ret = i915_mutex_lock_interruptible(dev); if (ret != 0) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } args->busy = obj->active; if (args->busy) { if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); } else if (obj->ring->outstanding_lazy_request == obj->last_rendering_seqno) { request = malloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO); ret = i915_add_request(obj->ring, NULL, request); if (ret != 0) free(request, DRM_I915_GEM); } i915_gem_retire_requests_ring(obj->ring); args->busy = obj->active; } drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return (ret); } static int i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) { struct drm_i915_private *dev_priv; struct drm_i915_file_private *file_priv; unsigned long recent_enough; struct drm_i915_gem_request *request; struct intel_ring_buffer *ring; u32 seqno; int ret; dev_priv = dev->dev_private; if (atomic_load_acq_int(&dev_priv->mm.wedged)) return (-EIO); file_priv = file->driver_priv; recent_enough = ticks - (20 * hz / 1000); ring = NULL; seqno = 0; mtx_lock(&file_priv->mm.lck); list_for_each_entry(request, &file_priv->mm.request_list, client_list) { if (time_after_eq(request->emitted_jiffies, recent_enough)) break; ring = request->ring; seqno = request->seqno; } mtx_unlock(&file_priv->mm.lck); if (seqno == 0) return (0); ret = 0; mtx_lock(&ring->irq_lock); if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { if (ring->irq_get(ring)) { while (ret == 0 && !(i915_seqno_passed(ring->get_seqno(ring), seqno) || atomic_load_acq_int(&dev_priv->mm.wedged))) ret = -msleep(ring, &ring->irq_lock, PCATCH, "915thr", 0); ring->irq_put(ring); if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged)) ret = -EIO; } else if (_intel_wait_for(dev, i915_seqno_passed(ring->get_seqno(ring), seqno) || atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { ret = -EBUSY; } } mtx_unlock(&ring->irq_lock); if (ret == 0) taskqueue_enqueue_timeout(dev_priv->tq, &dev_priv->mm.retire_task, 0); return (ret); } int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { return (i915_gem_ring_throttle(dev, file_priv)); } int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_i915_gem_madvise *args; struct drm_i915_gem_object *obj; int ret; args = data; switch (args->madv) { case I915_MADV_DONTNEED: case I915_MADV_WILLNEED: break; default: return (-EINVAL); } ret = i915_mutex_lock_interruptible(dev); if (ret != 0) return (ret); obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (obj->pin_count != 0) { ret = -EINVAL; goto out; } if (obj->madv != I915_MADV_PURGED_INTERNAL) obj->madv = args->madv; if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) i915_gem_object_truncate(obj); args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return (ret); } void i915_gem_cleanup_ringbuffer(struct drm_device *dev) { drm_i915_private_t *dev_priv; int i; dev_priv = dev->dev_private; for (i = 0; i < I915_NUM_RINGS; i++) intel_cleanup_ring_buffer(&dev_priv->rings[i]); } int i915_gem_entervt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv; int ret, i; if (drm_core_check_feature(dev, DRIVER_MODESET)) return (0); dev_priv = dev->dev_private; if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { DRM_ERROR("Reenabling wedged hardware, good luck\n"); atomic_store_rel_int(&dev_priv->mm.wedged, 0); } dev_priv->mm.suspended = 0; ret = i915_gem_init_hw(dev); if (ret != 0) { return (ret); } KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); for (i = 0; i < I915_NUM_RINGS; i++) { KASSERT(list_empty(&dev_priv->rings[i].active_list), ("ring %d active list", i)); KASSERT(list_empty(&dev_priv->rings[i].request_list), ("ring %d request list", i)); } DRM_UNLOCK(dev); ret = drm_irq_install(dev); DRM_LOCK(dev); if (ret) goto cleanup_ringbuffer; return (0); cleanup_ringbuffer: i915_gem_cleanup_ringbuffer(dev); dev_priv->mm.suspended = 1; return (ret); } int i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { if (drm_core_check_feature(dev, DRIVER_MODESET)) return 0; drm_irq_uninstall(dev); return (i915_gem_idle(dev)); } int i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, uint32_t *handle_p) { struct drm_i915_gem_object *obj; uint32_t handle; int ret; size = roundup(size, PAGE_SIZE); if (size == 0) return (-EINVAL); obj = i915_gem_alloc_object(dev, size); if (obj == NULL) return (-ENOMEM); handle = 0; ret = drm_gem_handle_create(file, &obj->base, &handle); if (ret != 0) { drm_gem_object_release(&obj->base); i915_gem_info_remove_obj(dev->dev_private, obj->base.size); free(obj, DRM_I915_GEM); return (-ret); } /* drop reference from allocate - handle holds it now */ drm_gem_object_unreference(&obj->base); CTR2(KTR_DRM, "object_create %p %x", obj, size); *handle_p = handle; return (0); } int i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, struct drm_mode_create_dumb *args) { /* have to work out size/pitch and return them */ args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); args->size = args->pitch * args->height; return (i915_gem_create(file, dev, args->size, &args->handle)); } int i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, uint32_t handle) { return (drm_gem_handle_delete(file, handle)); } int i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_create *args = data; return (i915_gem_create(file, dev, args->size, &args->handle)); } static int i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) { vm_object_t vm_obj; vm_page_t m; struct sf_buf *sf; vm_offset_t mkva; vm_pindex_t obj_pi; int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; if (obj->gtt_offset != 0 && rw == UIO_READ) do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); else do_bit17_swizzling = 0; obj->dirty = 1; vm_obj = obj->base.vm_obj; ret = 0; VM_OBJECT_WLOCK(vm_obj); vm_object_pip_add(vm_obj, 1); while (size > 0) { obj_pi = OFF_TO_IDX(offset); obj_po = offset & PAGE_MASK; m = i915_gem_wire_page(vm_obj, obj_pi); VM_OBJECT_WUNLOCK(vm_obj); sched_pin(); sf = sf_buf_alloc(m, SFB_CPUPRIVATE); mkva = sf_buf_kva(sf); length = min(size, PAGE_SIZE - obj_po); while (length > 0) { if (do_bit17_swizzling && (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { cnt = roundup2(obj_po + 1, 64); cnt = min(cnt - obj_po, length); swizzled_po = obj_po ^ 64; } else { cnt = length; swizzled_po = obj_po; } if (rw == UIO_READ) ret = -copyout_nofault( (char *)mkva + swizzled_po, (void *)(uintptr_t)data_ptr, cnt); else ret = -copyin_nofault( (void *)(uintptr_t)data_ptr, (char *)mkva + swizzled_po, cnt); if (ret != 0) break; data_ptr += cnt; size -= cnt; length -= cnt; offset += cnt; obj_po += cnt; } sf_buf_free(sf); sched_unpin(); VM_OBJECT_WLOCK(vm_obj); if (rw == UIO_WRITE) vm_page_dirty(m); vm_page_reference(m); vm_page_lock(m); vm_page_unwire(m, PQ_ACTIVE); vm_page_unlock(m); atomic_add_long(&i915_gem_wired_pages_cnt, -1); if (ret != 0) break; } vm_object_pip_wakeup(vm_obj); VM_OBJECT_WUNLOCK(vm_obj); return (ret); } static int i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) { vm_offset_t mkva; vm_pindex_t obj_pi; int obj_po, ret; obj_pi = OFF_TO_IDX(offset); obj_po = offset & PAGE_MASK; mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + IDX_TO_OFF(obj_pi), size, PAT_WRITE_COMBINING); ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva + obj_po, size); pmap_unmapdev(mkva, size); return (ret); } static int i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) { struct drm_i915_gem_object *obj; vm_page_t *ma; vm_offset_t start, end; int npages, ret; if (size == 0) return (0); start = trunc_page(data_ptr); end = round_page(data_ptr + size); npages = howmany(end - start, PAGE_SIZE); ma = malloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | M_ZERO); npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, (vm_offset_t)data_ptr, size, (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); if (npages == -1) { ret = -EFAULT; goto free_ma; } ret = i915_mutex_lock_interruptible(dev); if (ret != 0) goto unlocked; obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (offset > obj->base.size || size > obj->base.size - offset) { ret = -EINVAL; goto out; } if (rw == UIO_READ) { CTR3(KTR_DRM, "object_pread %p %jx %jx", obj, offset, size); ret = i915_gem_object_set_cpu_read_domain_range(obj, offset, size); if (ret != 0) goto out; ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, UIO_READ, file); } else { if (obj->phys_obj) { CTR3(KTR_DRM, "object_phys_write %p %jx %jx", obj, offset, size); ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, size, file); } else if (obj->gtt_space && obj->base.write_domain != I915_GEM_DOMAIN_CPU) { CTR3(KTR_DRM, "object_gtt_write %p %jx %jx", obj, offset, size); ret = i915_gem_object_pin(obj, 0, true); if (ret != 0) goto out; ret = i915_gem_object_set_to_gtt_domain(obj, true); if (ret != 0) goto out_unpin; ret = i915_gem_object_put_fence(obj); if (ret != 0) goto out_unpin; ret = i915_gem_gtt_write(dev, obj, data_ptr, size, offset, file); out_unpin: i915_gem_object_unpin(obj); } else { CTR3(KTR_DRM, "object_pwrite %p %jx %jx", obj, offset, size); ret = i915_gem_object_set_to_cpu_domain(obj, true); if (ret != 0) goto out; ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, UIO_WRITE, file); } } out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); unlocked: vm_page_unhold_pages(ma, npages); free_ma: free(ma, DRM_I915_GEM); return (ret); } int i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pread *args; args = data; return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, args->offset, UIO_READ, file)); } int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pwrite *args; args = data; return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, args->offset, UIO_WRITE, file)); } int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_set_domain *args; struct drm_i915_gem_object *obj; uint32_t read_domains; uint32_t write_domain; int ret; if ((dev->driver->driver_features & DRIVER_GEM) == 0) return (-ENODEV); args = data; read_domains = args->read_domains; write_domain = args->write_domain; if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || (read_domains & I915_GEM_GPU_DOMAINS) != 0 || (write_domain != 0 && read_domains != write_domain)) return (-EINVAL); ret = i915_mutex_lock_interruptible(dev); if (ret != 0) return (ret); obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); if (ret == -EINVAL) ret = 0; } else ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return (ret); } int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_sw_finish *args; struct drm_i915_gem_object *obj; int ret; args = data; ret = 0; if ((dev->driver->driver_features & DRIVER_GEM) == 0) return (ENODEV); ret = i915_mutex_lock_interruptible(dev); if (ret != 0) return (ret); obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (obj->pin_count != 0) i915_gem_object_flush_cpu_write_domain(obj); drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return (ret); } int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_mmap *args; struct drm_gem_object *obj; struct proc *p; vm_map_t map; vm_offset_t addr; vm_size_t size; int error, rv; args = data; if ((dev->driver->driver_features & DRIVER_GEM) == 0) return (-ENODEV); obj = drm_gem_object_lookup(dev, file, args->handle); if (obj == NULL) return (-ENOENT); error = 0; if (args->size == 0) goto out; p = curproc; map = &p->p_vmspace->vm_map; size = round_page(args->size); PROC_LOCK(p); if (map->size + size > lim_cur(p, RLIMIT_VMEM)) { PROC_UNLOCK(p); error = ENOMEM; goto out; } PROC_UNLOCK(p); addr = 0; vm_object_reference(obj->vm_obj); DRM_UNLOCK(dev); rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); if (rv != KERN_SUCCESS) { vm_object_deallocate(obj->vm_obj); error = -vm_mmap_to_errno(rv); } else { args->addr_ptr = (uint64_t)addr; } DRM_LOCK(dev); out: drm_gem_object_unreference(obj); return (error); } static int i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color) { *color = 0; /* XXXKIB */ return (0); } int i915_intr_pf; static int i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres) { struct drm_gem_object *gem_obj; struct drm_i915_gem_object *obj; struct drm_device *dev; drm_i915_private_t *dev_priv; vm_page_t m, oldm; int cause, ret; bool write; gem_obj = vm_obj->handle; obj = to_intel_bo(gem_obj); dev = obj->base.dev; dev_priv = dev->dev_private; #if 0 write = (prot & VM_PROT_WRITE) != 0; #else write = true; #endif vm_object_pip_add(vm_obj, 1); /* * Remove the placeholder page inserted by vm_fault() from the * object before dropping the object lock. If * i915_gem_release_mmap() is active in parallel on this gem * object, then it owns the drm device sx and might find the * placeholder already. Then, since the page is busy, * i915_gem_release_mmap() sleeps waiting for the busy state * of the page cleared. We will be not able to acquire drm * device lock until i915_gem_release_mmap() is able to make a * progress. */ if (*mres != NULL) { oldm = *mres; vm_page_lock(oldm); vm_page_remove(oldm); vm_page_unlock(oldm); *mres = NULL; } else oldm = NULL; VM_OBJECT_WUNLOCK(vm_obj); retry: cause = ret = 0; m = NULL; if (i915_intr_pf) { ret = i915_mutex_lock_interruptible(dev); if (ret != 0) { cause = 10; goto out; } } else DRM_LOCK(dev); /* * Since the object lock was dropped, other thread might have * faulted on the same GTT address and instantiated the * mapping for the page. Recheck. */ VM_OBJECT_WLOCK(vm_obj); m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); if (m != NULL) { if (vm_page_busied(m)) { DRM_UNLOCK(dev); vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); vm_page_busy_sleep(m, "915pee"); goto retry; } goto have_page; } else VM_OBJECT_WUNLOCK(vm_obj); /* Now bind it into the GTT if needed */ if (!obj->map_and_fenceable) { ret = i915_gem_object_unbind(obj); if (ret != 0) { cause = 20; goto unlock; } } if (!obj->gtt_space) { ret = i915_gem_object_bind_to_gtt(obj, 0, true); if (ret != 0) { cause = 30; goto unlock; } ret = i915_gem_object_set_to_gtt_domain(obj, write); if (ret != 0) { cause = 40; goto unlock; } } if (obj->tiling_mode == I915_TILING_NONE) ret = i915_gem_object_put_fence(obj); else ret = i915_gem_object_get_fence(obj, NULL); if (ret != 0) { cause = 50; goto unlock; } if (i915_gem_object_is_inactive(obj)) list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); obj->fault_mappable = true; VM_OBJECT_WLOCK(vm_obj); m = PHYS_TO_VM_PAGE(dev->agp->base + obj->gtt_offset + offset); KASSERT((m->flags & PG_FICTITIOUS) != 0, ("physical address %#jx not fictitious", (uintmax_t)(dev->agp->base + obj->gtt_offset + offset))); if (m == NULL) { VM_OBJECT_WUNLOCK(vm_obj); cause = 60; ret = -EFAULT; goto unlock; } KASSERT((m->flags & PG_FICTITIOUS) != 0, ("not fictitious %p", m)); KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); if (vm_page_busied(m)) { DRM_UNLOCK(dev); vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); vm_page_busy_sleep(m, "915pbs"); goto retry; } if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { DRM_UNLOCK(dev); VM_OBJECT_WUNLOCK(vm_obj); VM_WAIT; goto retry; } m->valid = VM_PAGE_BITS_ALL; have_page: *mres = m; vm_page_xbusy(m); CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, m->phys_addr); DRM_UNLOCK(dev); if (oldm != NULL) { vm_page_lock(oldm); vm_page_free(oldm); vm_page_unlock(oldm); } vm_object_pip_wakeup(vm_obj); return (VM_PAGER_OK); unlock: DRM_UNLOCK(dev); out: KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); CTR5(KTR_DRM, "fault_fail %p %jx %x err %d %d", gem_obj, offset, prot, -ret, cause); if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { kern_yield(PRI_USER); goto retry; } VM_OBJECT_WLOCK(vm_obj); vm_object_pip_wakeup(vm_obj); return (VM_PAGER_ERROR); } static void i915_gem_pager_dtor(void *handle) { struct drm_gem_object *obj; struct drm_device *dev; obj = handle; dev = obj->dev; DRM_LOCK(dev); drm_gem_free_mmap_offset(obj); i915_gem_release_mmap(to_intel_bo(obj)); drm_gem_object_unreference(obj); DRM_UNLOCK(dev); } struct cdev_pager_ops i915_gem_pager_ops = { .cdev_pg_fault = i915_gem_pager_fault, .cdev_pg_ctor = i915_gem_pager_ctor, .cdev_pg_dtor = i915_gem_pager_dtor }; int i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, uint32_t handle, uint64_t *offset) { struct drm_i915_private *dev_priv; struct drm_i915_gem_object *obj; int ret; if (!(dev->driver->driver_features & DRIVER_GEM)) return (-ENODEV); dev_priv = dev->dev_private; ret = i915_mutex_lock_interruptible(dev); if (ret != 0) return (ret); obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (obj->base.size > dev_priv->mm.gtt_mappable_end) { ret = -E2BIG; goto out; } if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to mmap a purgeable buffer\n"); ret = -EINVAL; goto out; } ret = drm_gem_create_mmap_offset(&obj->base); if (ret != 0) goto out; *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | DRM_GEM_MAPPING_KEY; out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return (ret); } int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_private *dev_priv; struct drm_i915_gem_mmap_gtt *args; dev_priv = dev->dev_private; args = data; return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); } struct drm_i915_gem_object * i915_gem_alloc_object(struct drm_device *dev, size_t size) { struct drm_i915_private *dev_priv; struct drm_i915_gem_object *obj; dev_priv = dev->dev_private; obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); if (drm_gem_object_init(dev, &obj->base, size) != 0) { free(obj, DRM_I915_GEM); return (NULL); } obj->base.write_domain = I915_GEM_DOMAIN_CPU; obj->base.read_domains = I915_GEM_DOMAIN_CPU; if (HAS_LLC(dev)) obj->cache_level = I915_CACHE_LLC; else obj->cache_level = I915_CACHE_NONE; obj->base.driver_private = NULL; obj->fence_reg = I915_FENCE_REG_NONE; INIT_LIST_HEAD(&obj->mm_list); INIT_LIST_HEAD(&obj->gtt_list); INIT_LIST_HEAD(&obj->ring_list); INIT_LIST_HEAD(&obj->exec_list); INIT_LIST_HEAD(&obj->gpu_write_list); obj->madv = I915_MADV_WILLNEED; /* Avoid an unnecessary call to unbind on the first bind. */ obj->map_and_fenceable = true; i915_gem_info_add_obj(dev_priv, size); return (obj); } void i915_gem_clflush_object(struct drm_i915_gem_object *obj) { /* If we don't have a page list set up, then we're not pinned * to GPU, and we can ignore the cache flush because it'll happen * again at bind time. */ if (obj->pages == NULL) return; /* If the GPU is snooping the contents of the CPU cache, * we do not need to manually clear the CPU cache lines. However, * the caches are only snooped when the render cache is * flushed/invalidated. As we always have to emit invalidations * and flushes when moving into and out of the RENDER domain, correct * snooping behaviour occurs naturally as the result of our domain * tracking. */ if (obj->cache_level != I915_CACHE_NONE) return; CTR1(KTR_DRM, "object_clflush %p", obj); drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); } static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) { uint32_t old_write_domain; if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) return; i915_gem_clflush_object(obj); intel_gtt_chipset_flush(); old_write_domain = obj->base.write_domain; obj->base.write_domain = 0; CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, obj->base.read_domains, old_write_domain); } static int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) { if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) return (0); return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); } static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) { uint32_t old_write_domain; if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) return; wmb(); old_write_domain = obj->base.write_domain; obj->base.write_domain = 0; CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, obj->base.read_domains, old_write_domain); } int i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) { uint32_t old_write_domain, old_read_domains; int ret; if (obj->gtt_space == NULL) return (-EINVAL); if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) return 0; ret = i915_gem_object_flush_gpu_write_domain(obj); if (ret != 0) return (ret); if (obj->pending_gpu_write || write) { ret = i915_gem_object_wait_rendering(obj); if (ret != 0) return (ret); } i915_gem_object_flush_cpu_write_domain(obj); old_write_domain = obj->base.write_domain; old_read_domains = obj->base.read_domains; KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, ("In GTT write domain")); obj->base.read_domains |= I915_GEM_DOMAIN_GTT; if (write) { obj->base.read_domains = I915_GEM_DOMAIN_GTT; obj->base.write_domain = I915_GEM_DOMAIN_GTT; obj->dirty = 1; } CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, old_read_domains, old_write_domain); return (0); } int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level) { struct drm_device *dev; drm_i915_private_t *dev_priv; int ret; if (obj->cache_level == cache_level) return 0; if (obj->pin_count) { DRM_DEBUG("can not change the cache level of pinned objects\n"); return (-EBUSY); } dev = obj->base.dev; dev_priv = dev->dev_private; if (obj->gtt_space) { ret = i915_gem_object_finish_gpu(obj); if (ret != 0) return (ret); i915_gem_object_finish_gtt(obj); /* Before SandyBridge, you could not use tiling or fence * registers with snooped memory, so relinquish any fences * currently pointing to our region in the aperture. */ if (INTEL_INFO(obj->base.dev)->gen < 6) { ret = i915_gem_object_put_fence(obj); if (ret != 0) return (ret); } i915_gem_gtt_rebind_object(obj, cache_level); if (obj->has_aliasing_ppgtt_mapping) i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, obj, cache_level); } if (cache_level == I915_CACHE_NONE) { u32 old_read_domains, old_write_domain; /* If we're coming from LLC cached, then we haven't * actually been tracking whether the data is in the * CPU cache or not, since we only allow one bit set * in obj->write_domain and have been skipping the clflushes. * Just set it to the CPU cache for now. */ KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, ("obj %p in CPU write domain", obj)); KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, ("obj %p in CPU read domain", obj)); old_read_domains = obj->base.read_domains; old_write_domain = obj->base.write_domain; obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->base.write_domain = I915_GEM_DOMAIN_CPU; CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", obj, old_read_domains, old_write_domain); } obj->cache_level = cache_level; return (0); } int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, struct intel_ring_buffer *pipelined) { u32 old_read_domains, old_write_domain; int ret; ret = i915_gem_object_flush_gpu_write_domain(obj); if (ret != 0) return (ret); if (pipelined != obj->ring) { ret = i915_gem_object_wait_rendering(obj); if (ret == -ERESTART || ret == -EINTR) return (ret); } ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); if (ret != 0) return (ret); ret = i915_gem_object_pin(obj, alignment, true); if (ret != 0) return (ret); i915_gem_object_flush_cpu_write_domain(obj); old_write_domain = obj->base.write_domain; old_read_domains = obj->base.read_domains; KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, ("obj %p in GTT write domain", obj)); obj->base.read_domains |= I915_GEM_DOMAIN_GTT; CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", obj, old_read_domains, obj->base.write_domain); return (0); } int i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) { int ret; if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) return (0); if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); if (ret != 0) return (ret); } ret = i915_gem_object_wait_rendering(obj); if (ret != 0) return (ret); obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; return (0); } static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) { uint32_t old_write_domain, old_read_domains; int ret; if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) return 0; ret = i915_gem_object_flush_gpu_write_domain(obj); if (ret != 0) return (ret); ret = i915_gem_object_wait_rendering(obj); if (ret != 0) return (ret); i915_gem_object_flush_gtt_write_domain(obj); i915_gem_object_set_to_full_cpu_read_domain(obj); old_write_domain = obj->base.write_domain; old_read_domains = obj->base.read_domains; if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { i915_gem_clflush_object(obj); obj->base.read_domains |= I915_GEM_DOMAIN_CPU; } KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, ("In cpu write domain")); if (write) { obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->base.write_domain = I915_GEM_DOMAIN_CPU; } CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, old_read_domains, old_write_domain); return (0); } static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) { int i; if (obj->page_cpu_valid == NULL) return; if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { if (obj->page_cpu_valid[i] != 0) continue; drm_clflush_pages(obj->pages + i, 1); } } free(obj->page_cpu_valid, DRM_I915_GEM); obj->page_cpu_valid = NULL; } static int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size) { uint32_t old_read_domains; int i, ret; if (offset == 0 && size == obj->base.size) return (i915_gem_object_set_to_cpu_domain(obj, 0)); ret = i915_gem_object_flush_gpu_write_domain(obj); if (ret != 0) return (ret); ret = i915_gem_object_wait_rendering(obj); if (ret != 0) return (ret); i915_gem_object_flush_gtt_write_domain(obj); if (obj->page_cpu_valid == NULL && (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) return (0); if (obj->page_cpu_valid == NULL) { obj->page_cpu_valid = malloc(obj->base.size / PAGE_SIZE, DRM_I915_GEM, M_WAITOK | M_ZERO); } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; i++) { if (obj->page_cpu_valid[i]) continue; drm_clflush_pages(obj->pages + i, 1); obj->page_cpu_valid[i] = 1; } KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, ("In gpu write domain")); old_read_domains = obj->base.read_domains; obj->base.read_domains |= I915_GEM_DOMAIN_CPU; CTR3(KTR_DRM, "object_change_domain set_cpu_read %p %x %x", obj, old_read_domains, obj->base.write_domain); return (0); } static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) { uint32_t gtt_size; if (INTEL_INFO(dev)->gen >= 4 || tiling_mode == I915_TILING_NONE) return (size); /* Previous chips need a power-of-two fence region when tiling */ if (INTEL_INFO(dev)->gen == 3) gtt_size = 1024*1024; else gtt_size = 512*1024; while (gtt_size < size) gtt_size <<= 1; return (gtt_size); } /** * i915_gem_get_gtt_alignment - return required GTT alignment for an object * @obj: object to check * * Return the required GTT alignment for an object, taking into account * potential fence register mapping. */ static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, int tiling_mode) { /* * Minimum alignment is 4k (GTT page size), but might be greater * if a fence register is needed for the object. */ if (INTEL_INFO(dev)->gen >= 4 || tiling_mode == I915_TILING_NONE) return (4096); /* * Previous chips need to be aligned to the size of the smallest * fence register that can contain the object. */ return (i915_gem_get_gtt_size(dev, size, tiling_mode)); } uint32_t i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, int tiling_mode) { if (tiling_mode == I915_TILING_NONE) return (4096); /* * Minimum alignment is 4k (GTT page size) for sane hw. */ if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) return (4096); /* * Previous hardware however needs to be aligned to a power-of-two * tile height. The simplest method for determining this is to reuse * the power-of-tile object size. */ return (i915_gem_get_gtt_size(dev, size, tiling_mode)); } static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, unsigned alignment, bool map_and_fenceable) { struct drm_device *dev; struct drm_i915_private *dev_priv; struct drm_mm_node *free_space; uint32_t size, fence_size, fence_alignment, unfenced_alignment; bool mappable, fenceable; int ret; dev = obj->base.dev; dev_priv = dev->dev_private; if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to bind a purgeable object\n"); return (-EINVAL); } fence_size = i915_gem_get_gtt_size(dev, obj->base.size, obj->tiling_mode); fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, obj->tiling_mode); unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, obj->base.size, obj->tiling_mode); if (alignment == 0) alignment = map_and_fenceable ? fence_alignment : unfenced_alignment; if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { DRM_ERROR("Invalid object alignment requested %u\n", alignment); return (-EINVAL); } size = map_and_fenceable ? fence_size : obj->base.size; /* If the object is bigger than the entire aperture, reject it early * before evicting everything in a vain attempt to find space. */ if (obj->base.size > (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { DRM_ERROR( "Attempting to bind an object larger than the aperture\n"); return (-E2BIG); } search_free: if (map_and_fenceable) free_space = drm_mm_search_free_in_range( &dev_priv->mm.gtt_space, size, alignment, 0, dev_priv->mm.gtt_mappable_end, 0); else free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, size, alignment, 0); if (free_space != NULL) { if (map_and_fenceable) obj->gtt_space = drm_mm_get_block_range_generic( free_space, size, alignment, 0, dev_priv->mm.gtt_mappable_end, 1); else obj->gtt_space = drm_mm_get_block_generic(free_space, size, alignment, 1); } if (obj->gtt_space == NULL) { ret = i915_gem_evict_something(dev, size, alignment, map_and_fenceable); if (ret != 0) return (ret); goto search_free; } ret = i915_gem_object_get_pages_gtt(obj, 0); if (ret != 0) { drm_mm_put_block(obj->gtt_space); obj->gtt_space = NULL; /* * i915_gem_object_get_pages_gtt() cannot return * ENOMEM, since we use vm_page_grab(). */ return (ret); } ret = i915_gem_gtt_bind_object(obj); if (ret != 0) { i915_gem_object_put_pages_gtt(obj); drm_mm_put_block(obj->gtt_space); obj->gtt_space = NULL; if (i915_gem_evict_everything(dev, false)) return (ret); goto search_free; } list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, ("Object in gpu read domain")); KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, ("Object in gpu write domain")); obj->gtt_offset = obj->gtt_space->start; fenceable = obj->gtt_space->size == fence_size && (obj->gtt_space->start & (fence_alignment - 1)) == 0; mappable = obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; obj->map_and_fenceable = mappable && fenceable; CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, obj->base.size, map_and_fenceable); return (0); } static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) { u32 old_write_domain, old_read_domains; /* Act a barrier for all accesses through the GTT */ mb(); /* Force a pagefault for domain tracking on next user access */ i915_gem_release_mmap(obj); if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) return; old_read_domains = obj->base.read_domains; old_write_domain = obj->base.write_domain; obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", obj, old_read_domains, old_write_domain); } int i915_gem_object_unbind(struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv; int ret; dev_priv = obj->base.dev->dev_private; ret = 0; if (obj->gtt_space == NULL) return (0); if (obj->pin_count != 0) { DRM_ERROR("Attempting to unbind pinned buffer\n"); return (-EINVAL); } ret = i915_gem_object_finish_gpu(obj); if (ret == -ERESTART || ret == -EINTR) return (ret); i915_gem_object_finish_gtt(obj); if (ret == 0) ret = i915_gem_object_set_to_cpu_domain(obj, 1); if (ret == -ERESTART || ret == -EINTR) return (ret); if (ret != 0) { i915_gem_clflush_object(obj); obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; } ret = i915_gem_object_put_fence(obj); if (ret == -ERESTART) return (ret); i915_gem_gtt_unbind_object(obj); if (obj->has_aliasing_ppgtt_mapping) { i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); obj->has_aliasing_ppgtt_mapping = 0; } i915_gem_object_put_pages_gtt(obj); list_del_init(&obj->gtt_list); list_del_init(&obj->mm_list); obj->map_and_fenceable = true; drm_mm_put_block(obj->gtt_space); obj->gtt_space = NULL; obj->gtt_offset = 0; if (i915_gem_object_is_purgeable(obj)) i915_gem_object_truncate(obj); CTR1(KTR_DRM, "object_unbind %p", obj); return (ret); } static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, int flags) { struct drm_device *dev; vm_object_t vm_obj; vm_page_t m; int page_count, i, j; dev = obj->base.dev; KASSERT(obj->pages == NULL, ("Obj already has pages")); page_count = obj->base.size / PAGE_SIZE; obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK); vm_obj = obj->base.vm_obj; VM_OBJECT_WLOCK(vm_obj); for (i = 0; i < page_count; i++) { if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) goto failed; } VM_OBJECT_WUNLOCK(vm_obj); if (i915_gem_object_needs_bit17_swizzle(obj)) i915_gem_object_do_bit_17_swizzle(obj); return (0); failed: for (j = 0; j < i; j++) { m = obj->pages[j]; vm_page_lock(m); vm_page_unwire(m, PQ_INACTIVE); vm_page_unlock(m); atomic_add_long(&i915_gem_wired_pages_cnt, -1); } VM_OBJECT_WUNLOCK(vm_obj); free(obj->pages, DRM_I915_GEM); obj->pages = NULL; return (-EIO); } #define GEM_PARANOID_CHECK_GTT 0 #if GEM_PARANOID_CHECK_GTT static void i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, int page_count) { struct drm_i915_private *dev_priv; vm_paddr_t pa; unsigned long start, end; u_int i; int j; dev_priv = dev->dev_private; start = OFF_TO_IDX(dev_priv->mm.gtt_start); end = OFF_TO_IDX(dev_priv->mm.gtt_end); for (i = start; i < end; i++) { pa = intel_gtt_read_pte_paddr(i); for (j = 0; j < page_count; j++) { if (pa == VM_PAGE_TO_PHYS(ma[j])) { panic("Page %p in GTT pte index %d pte %x", ma[i], i, intel_gtt_read_pte(i)); } } } } #endif static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) { vm_page_t m; int page_count, i; KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); if (obj->tiling_mode != I915_TILING_NONE) i915_gem_object_save_bit_17_swizzle(obj); if (obj->madv == I915_MADV_DONTNEED) obj->dirty = 0; page_count = obj->base.size / PAGE_SIZE; VM_OBJECT_WLOCK(obj->base.vm_obj); #if GEM_PARANOID_CHECK_GTT i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); #endif for (i = 0; i < page_count; i++) { m = obj->pages[i]; if (obj->dirty) vm_page_dirty(m); if (obj->madv == I915_MADV_WILLNEED) vm_page_reference(m); vm_page_lock(m); vm_page_unwire(obj->pages[i], PQ_ACTIVE); vm_page_unlock(m); atomic_add_long(&i915_gem_wired_pages_cnt, -1); } VM_OBJECT_WUNLOCK(obj->base.vm_obj); obj->dirty = 0; free(obj->pages, DRM_I915_GEM); obj->pages = NULL; } void i915_gem_release_mmap(struct drm_i915_gem_object *obj) { vm_object_t devobj; vm_page_t m; int i, page_count; if (!obj->fault_mappable) return; CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, OFF_TO_IDX(obj->base.size)); devobj = cdev_pager_lookup(obj); if (devobj != NULL) { page_count = OFF_TO_IDX(obj->base.size); VM_OBJECT_WLOCK(devobj); retry: for (i = 0; i < page_count; i++) { m = vm_page_lookup(devobj, i); if (m == NULL) continue; if (vm_page_sleep_if_busy(m, "915unm")) goto retry; cdev_pager_free_page(devobj, m); } VM_OBJECT_WUNLOCK(devobj); vm_object_deallocate(devobj); } obj->fault_mappable = false; } int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) { int ret; KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, ("In GPU write domain")); CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj, obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset, obj->active, obj->last_rendering_seqno); if (obj->active) { ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, true); if (ret != 0) return (ret); } return (0); } void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, struct intel_ring_buffer *ring, uint32_t seqno) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_fence_reg *reg; obj->ring = ring; KASSERT(ring != NULL, ("NULL ring")); /* Add a reference if we're newly entering the active list. */ if (!obj->active) { drm_gem_object_reference(&obj->base); obj->active = 1; } /* Move from whatever list we were on to the tail of execution. */ list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); list_move_tail(&obj->ring_list, &ring->active_list); obj->last_rendering_seqno = seqno; if (obj->fenced_gpu_access) { obj->last_fenced_seqno = seqno; obj->last_fenced_ring = ring; /* Bump MRU to take account of the delayed flush */ if (obj->fence_reg != I915_FENCE_REG_NONE) { reg = &dev_priv->fence_regs[obj->fence_reg]; list_move_tail(®->lru_list, &dev_priv->mm.fence_list); } } } static void i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) { list_del_init(&obj->ring_list); obj->last_rendering_seqno = 0; obj->last_fenced_seqno = 0; } static void i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) { struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; KASSERT(obj->active, ("Object not active")); list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); i915_gem_object_move_off_active(obj); } static void i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; if (obj->pin_count != 0) list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); else list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); KASSERT(obj->active, ("Object not active")); obj->ring = NULL; obj->last_fenced_ring = NULL; i915_gem_object_move_off_active(obj); obj->fenced_gpu_access = false; obj->active = 0; obj->pending_gpu_write = false; drm_gem_object_unreference(&obj->base); #if 1 KIB_NOTYET(); #else WARN_ON(i915_verify_lists(dev)); #endif } static void i915_gem_object_truncate(struct drm_i915_gem_object *obj) { vm_object_t vm_obj; vm_obj = obj->base.vm_obj; VM_OBJECT_WLOCK(vm_obj); vm_object_page_remove(vm_obj, 0, 0, false); VM_OBJECT_WUNLOCK(vm_obj); obj->madv = I915_MADV_PURGED_INTERNAL; } static inline int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) { return (obj->madv == I915_MADV_DONTNEED); } static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, uint32_t flush_domains) { struct drm_i915_gem_object *obj, *next; uint32_t old_write_domain; list_for_each_entry_safe(obj, next, &ring->gpu_write_list, gpu_write_list) { if (obj->base.write_domain & flush_domains) { old_write_domain = obj->base.write_domain; obj->base.write_domain = 0; list_del_init(&obj->gpu_write_list); i915_gem_object_move_to_active(obj, ring, i915_gem_next_request_seqno(ring)); CTR3(KTR_DRM, "object_change_domain process_flush %p %x %x", obj, obj->base.read_domains, old_write_domain); } } } static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv; dev_priv = obj->base.dev->dev_private; return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && obj->tiling_mode != I915_TILING_NONE); } static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) { vm_page_t m; int rv; VM_OBJECT_ASSERT_WLOCKED(object); m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); if (m->valid != VM_PAGE_BITS_ALL) { if (vm_pager_has_page(object, pindex, NULL, NULL)) { rv = vm_pager_get_pages(object, &m, 1, 0); m = vm_page_lookup(object, pindex); if (m == NULL) return (NULL); if (rv != VM_PAGER_OK) { vm_page_lock(m); vm_page_free(m); vm_page_unlock(m); return (NULL); } } else { pmap_zero_page(m); m->valid = VM_PAGE_BITS_ALL; m->dirty = 0; } } vm_page_lock(m); vm_page_wire(m); vm_page_unlock(m); vm_page_xunbusy(m); atomic_add_long(&i915_gem_wired_pages_cnt, 1); return (m); } int i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, uint32_t flush_domains) { int ret; if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) return 0; CTR3(KTR_DRM, "ring_flush %s %x %x", ring->name, invalidate_domains, flush_domains); ret = ring->flush(ring, invalidate_domains, flush_domains); if (ret) return ret; if (flush_domains & I915_GEM_GPU_DOMAINS) i915_gem_process_flushing_list(ring, flush_domains); return 0; } static int i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) { int ret; if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) return 0; if (!list_empty(&ring->gpu_write_list)) { ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); if (ret != 0) return ret; } return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), do_retire)); } int i915_gpu_idle(struct drm_device *dev, bool do_retire) { drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring; int ret, i; /* Flush everything onto the inactive list. */ - for (i = 0; i < I915_NUM_RINGS; i++) { - ret = i915_ring_idle(&dev_priv->rings[i], do_retire); + for_each_ring(ring, dev_priv, i) { + ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); + if (ret) + return ret; + + ret = i915_ring_idle(ring, do_retire); if (ret) return ret; } return 0; } int i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) { drm_i915_private_t *dev_priv; struct drm_i915_gem_request *request; uint32_t ier; int flags, ret; bool recovery_complete; KASSERT(seqno != 0, ("Zero seqno")); dev_priv = ring->dev->dev_private; ret = 0; if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { /* Give the error handler a chance to run. */ mtx_lock(&dev_priv->error_completion_lock); recovery_complete = (&dev_priv->error_completion) > 0; mtx_unlock(&dev_priv->error_completion_lock); return (recovery_complete ? -EIO : -EAGAIN); } if (seqno == ring->outstanding_lazy_request) { request = malloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO); if (request == NULL) return (-ENOMEM); ret = i915_add_request(ring, NULL, request); if (ret != 0) { free(request, DRM_I915_GEM); return (ret); } seqno = request->seqno; } if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { if (HAS_PCH_SPLIT(ring->dev)) ier = I915_READ(DEIER) | I915_READ(GTIER); else ier = I915_READ(IER); if (!ier) { DRM_ERROR("something (likely vbetool) disabled " "interrupts, re-enabling\n"); ring->dev->driver->irq_preinstall(ring->dev); ring->dev->driver->irq_postinstall(ring->dev); } CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); ring->waiting_seqno = seqno; mtx_lock(&ring->irq_lock); if (ring->irq_get(ring)) { flags = dev_priv->mm.interruptible ? PCATCH : 0; while (!i915_seqno_passed(ring->get_seqno(ring), seqno) && !atomic_load_acq_int(&dev_priv->mm.wedged) && ret == 0) { ret = -msleep(ring, &ring->irq_lock, flags, "915gwr", 0); } ring->irq_put(ring); mtx_unlock(&ring->irq_lock); } else { mtx_unlock(&ring->irq_lock); if (_intel_wait_for(ring->dev, i915_seqno_passed(ring->get_seqno(ring), seqno) || atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "i915wrq") != 0) ret = -EBUSY; } ring->waiting_seqno = 0; CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, ret); } if (atomic_load_acq_int(&dev_priv->mm.wedged)) ret = -EAGAIN; /* Directly dispatch request retiring. While we have the work queue * to handle this, the waiter on a request often wants an associated * buffer to have made it to the inactive list, and we would need * a separate wait queue to handle that. */ if (ret == 0 && do_retire) i915_gem_retire_requests_ring(ring); return (ret); } static u32 i915_gem_get_seqno(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; u32 seqno = dev_priv->next_seqno; /* reserve 0 for non-seqno */ if (++dev_priv->next_seqno == 0) dev_priv->next_seqno = 1; return seqno; } u32 i915_gem_next_request_seqno(struct intel_ring_buffer *ring) { if (ring->outstanding_lazy_request == 0) ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); return ring->outstanding_lazy_request; } int i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, struct drm_i915_gem_request *request) { drm_i915_private_t *dev_priv; struct drm_i915_file_private *file_priv; uint32_t seqno; u32 request_ring_position; int was_empty; int ret; KASSERT(request != NULL, ("NULL request in add")); DRM_LOCK_ASSERT(ring->dev); dev_priv = ring->dev->dev_private; seqno = i915_gem_next_request_seqno(ring); request_ring_position = intel_ring_get_tail(ring); ret = ring->add_request(ring, &seqno); if (ret != 0) return ret; CTR2(KTR_DRM, "request_add %s %d", ring->name, seqno); request->seqno = seqno; request->ring = ring; request->tail = request_ring_position; request->emitted_jiffies = ticks; was_empty = list_empty(&ring->request_list); list_add_tail(&request->list, &ring->request_list); if (file != NULL) { file_priv = file->driver_priv; mtx_lock(&file_priv->mm.lck); request->file_priv = file_priv; list_add_tail(&request->client_list, &file_priv->mm.request_list); mtx_unlock(&file_priv->mm.lck); } ring->outstanding_lazy_request = 0; if (!dev_priv->mm.suspended) { if (i915_enable_hangcheck) { callout_schedule(&dev_priv->hangcheck_timer, DRM_I915_HANGCHECK_PERIOD); } if (was_empty) taskqueue_enqueue_timeout(dev_priv->tq, &dev_priv->mm.retire_task, hz); } return (0); } static inline void i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) { struct drm_i915_file_private *file_priv = request->file_priv; if (!file_priv) return; DRM_LOCK_ASSERT(request->ring->dev); mtx_lock(&file_priv->mm.lck); if (request->file_priv != NULL) { list_del(&request->client_list); request->file_priv = NULL; } mtx_unlock(&file_priv->mm.lck); } void i915_gem_release(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv; struct drm_i915_gem_request *request; file_priv = file->driver_priv; /* Clean up our request list when the client is going away, so that * later retire_requests won't dereference our soon-to-be-gone * file_priv. */ mtx_lock(&file_priv->mm.lck); while (!list_empty(&file_priv->mm.request_list)) { request = list_first_entry(&file_priv->mm.request_list, struct drm_i915_gem_request, client_list); list_del(&request->client_list); request->file_priv = NULL; } mtx_unlock(&file_priv->mm.lck); } static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, struct intel_ring_buffer *ring) { if (ring->dev != NULL) DRM_LOCK_ASSERT(ring->dev); while (!list_empty(&ring->request_list)) { struct drm_i915_gem_request *request; request = list_first_entry(&ring->request_list, struct drm_i915_gem_request, list); list_del(&request->list); i915_gem_request_remove_from_client(request); free(request, DRM_I915_GEM); } while (!list_empty(&ring->active_list)) { struct drm_i915_gem_object *obj; obj = list_first_entry(&ring->active_list, struct drm_i915_gem_object, ring_list); obj->base.write_domain = 0; list_del_init(&obj->gpu_write_list); i915_gem_object_move_to_inactive(obj); } } static void i915_gem_reset_fences(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int i; for (i = 0; i < dev_priv->num_fence_regs; i++) { struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; struct drm_i915_gem_object *obj = reg->obj; if (!obj) continue; if (obj->tiling_mode) i915_gem_release_mmap(obj); reg->obj->fence_reg = I915_FENCE_REG_NONE; reg->obj->fenced_gpu_access = false; reg->obj->last_fenced_seqno = 0; reg->obj->last_fenced_ring = NULL; i915_gem_clear_fence_reg(dev, reg); } } void i915_gem_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; int i; for (i = 0; i < I915_NUM_RINGS; i++) i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]); /* Remove anything from the flushing lists. The GPU cache is likely * to be lost on reset along with the data, so simply move the * lost bo to the inactive list. */ while (!list_empty(&dev_priv->mm.flushing_list)) { obj = list_first_entry(&dev_priv->mm.flushing_list, struct drm_i915_gem_object, mm_list); obj->base.write_domain = 0; list_del_init(&obj->gpu_write_list); i915_gem_object_move_to_inactive(obj); } /* Move everything out of the GPU domains to ensure we do any * necessary invalidation upon reuse. */ list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; } /* The fence registers are invalidated so clear them out */ i915_gem_reset_fences(dev); } /** * This function clears the request list as sequence numbers are passed. */ void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) { uint32_t seqno; int i; if (list_empty(&ring->request_list)) return; seqno = ring->get_seqno(ring); CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) if (seqno >= ring->sync_seqno[i]) ring->sync_seqno[i] = 0; while (!list_empty(&ring->request_list)) { struct drm_i915_gem_request *request; request = list_first_entry(&ring->request_list, struct drm_i915_gem_request, list); if (!i915_seqno_passed(seqno, request->seqno)) break; CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", ring->name, seqno); ring->last_retired_head = request->tail; list_del(&request->list); i915_gem_request_remove_from_client(request); free(request, DRM_I915_GEM); } /* Move any buffers on the active list that are no longer referenced * by the ringbuffer to the flushing/inactive lists as appropriate. */ while (!list_empty(&ring->active_list)) { struct drm_i915_gem_object *obj; obj = list_first_entry(&ring->active_list, struct drm_i915_gem_object, ring_list); if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) break; if (obj->base.write_domain != 0) i915_gem_object_move_to_flushing(obj); else i915_gem_object_move_to_inactive(obj); } if (ring->trace_irq_seqno && i915_seqno_passed(seqno, ring->trace_irq_seqno)) { mtx_lock(&ring->irq_lock); ring->irq_put(ring); mtx_unlock(&ring->irq_lock); ring->trace_irq_seqno = 0; } } void i915_gem_retire_requests(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj, *next; int i; if (!list_empty(&dev_priv->mm.deferred_free_list)) { list_for_each_entry_safe(obj, next, &dev_priv->mm.deferred_free_list, mm_list) i915_gem_free_object_tail(obj); } for (i = 0; i < I915_NUM_RINGS; i++) i915_gem_retire_requests_ring(&dev_priv->rings[i]); } static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, struct intel_ring_buffer *pipelined) { struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; u32 size = obj->gtt_space->size; int regnum = obj->fence_reg; uint64_t val; val = (uint64_t)((obj->gtt_offset + size - 4096) & 0xfffff000) << 32; val |= obj->gtt_offset & 0xfffff000; val |= (uint64_t)((obj->stride / 128) - 1) << SANDYBRIDGE_FENCE_PITCH_SHIFT; if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I965_FENCE_TILING_Y_SHIFT; val |= I965_FENCE_REG_VALID; if (pipelined) { int ret = intel_ring_begin(pipelined, 6); if (ret) return ret; intel_ring_emit(pipelined, MI_NOOP); intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); intel_ring_emit(pipelined, (u32)val); intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); intel_ring_emit(pipelined, (u32)(val >> 32)); intel_ring_advance(pipelined); } else I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); return 0; } static int i965_write_fence_reg(struct drm_i915_gem_object *obj, struct intel_ring_buffer *pipelined) { struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; u32 size = obj->gtt_space->size; int regnum = obj->fence_reg; uint64_t val; val = (uint64_t)((obj->gtt_offset + size - 4096) & 0xfffff000) << 32; val |= obj->gtt_offset & 0xfffff000; val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I965_FENCE_TILING_Y_SHIFT; val |= I965_FENCE_REG_VALID; if (pipelined) { int ret = intel_ring_begin(pipelined, 6); if (ret) return ret; intel_ring_emit(pipelined, MI_NOOP); intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); intel_ring_emit(pipelined, (u32)val); intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); intel_ring_emit(pipelined, (u32)(val >> 32)); intel_ring_advance(pipelined); } else I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); return 0; } static int i915_write_fence_reg(struct drm_i915_gem_object *obj, struct intel_ring_buffer *pipelined) { struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; u32 size = obj->gtt_space->size; u32 fence_reg, val, pitch_val; int tile_width; if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || (size & -size) != size || (obj->gtt_offset & (size - 1))) { printf( "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", obj->gtt_offset, obj->map_and_fenceable, size); return -EINVAL; } if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) tile_width = 128; else tile_width = 512; /* Note: pitch better be a power of two tile widths */ pitch_val = obj->stride / tile_width; pitch_val = ffs(pitch_val) - 1; val = obj->gtt_offset; if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I830_FENCE_TILING_Y_SHIFT; val |= I915_FENCE_SIZE_BITS(size); val |= pitch_val << I830_FENCE_PITCH_SHIFT; val |= I830_FENCE_REG_VALID; fence_reg = obj->fence_reg; if (fence_reg < 8) fence_reg = FENCE_REG_830_0 + fence_reg * 4; else fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; if (pipelined) { int ret = intel_ring_begin(pipelined, 4); if (ret) return ret; intel_ring_emit(pipelined, MI_NOOP); intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit(pipelined, fence_reg); intel_ring_emit(pipelined, val); intel_ring_advance(pipelined); } else I915_WRITE(fence_reg, val); return 0; } static int i830_write_fence_reg(struct drm_i915_gem_object *obj, struct intel_ring_buffer *pipelined) { struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; u32 size = obj->gtt_space->size; int regnum = obj->fence_reg; uint32_t val; uint32_t pitch_val; if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || (size & -size) != size || (obj->gtt_offset & (size - 1))) { printf( "object 0x%08x not 512K or pot-size 0x%08x aligned\n", obj->gtt_offset, size); return -EINVAL; } pitch_val = obj->stride / 128; pitch_val = ffs(pitch_val) - 1; val = obj->gtt_offset; if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I830_FENCE_TILING_Y_SHIFT; val |= I830_FENCE_SIZE_BITS(size); val |= pitch_val << I830_FENCE_PITCH_SHIFT; val |= I830_FENCE_REG_VALID; if (pipelined) { int ret = intel_ring_begin(pipelined, 4); if (ret) return ret; intel_ring_emit(pipelined, MI_NOOP); intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); intel_ring_emit(pipelined, val); intel_ring_advance(pipelined); } else I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); return 0; } static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) { return i915_seqno_passed(ring->get_seqno(ring), seqno); } static int i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, struct intel_ring_buffer *pipelined) { int ret; if (obj->fenced_gpu_access) { if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, obj->base.write_domain); if (ret) return ret; } obj->fenced_gpu_access = false; } if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { if (!ring_passed_seqno(obj->last_fenced_ring, obj->last_fenced_seqno)) { ret = i915_wait_request(obj->last_fenced_ring, obj->last_fenced_seqno, true); if (ret) return ret; } obj->last_fenced_seqno = 0; obj->last_fenced_ring = NULL; } /* Ensure that all CPU reads are completed before installing a fence * and all writes before removing the fence. */ if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) mb(); return 0; } int i915_gem_object_put_fence(struct drm_i915_gem_object *obj) { int ret; if (obj->tiling_mode) i915_gem_release_mmap(obj); ret = i915_gem_object_flush_fence(obj, NULL); if (ret) return ret; if (obj->fence_reg != I915_FENCE_REG_NONE) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) printf("%s: pin_count %d\n", __func__, dev_priv->fence_regs[obj->fence_reg].pin_count); i915_gem_clear_fence_reg(obj->base.dev, &dev_priv->fence_regs[obj->fence_reg]); obj->fence_reg = I915_FENCE_REG_NONE; } return 0; } static struct drm_i915_fence_reg * i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_fence_reg *reg, *first, *avail; int i; /* First try to find a free reg */ avail = NULL; for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { reg = &dev_priv->fence_regs[i]; if (!reg->obj) return reg; if (!reg->pin_count) avail = reg; } if (avail == NULL) return NULL; /* None available, try to steal one or wait for a user to finish */ avail = first = NULL; list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { if (reg->pin_count) continue; if (first == NULL) first = reg; if (!pipelined || !reg->obj->last_fenced_ring || reg->obj->last_fenced_ring == pipelined) { avail = reg; break; } } if (avail == NULL) avail = first; return avail; } int i915_gem_object_get_fence(struct drm_i915_gem_object *obj, struct intel_ring_buffer *pipelined) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_fence_reg *reg; int ret; pipelined = NULL; ret = 0; if (obj->fence_reg != I915_FENCE_REG_NONE) { reg = &dev_priv->fence_regs[obj->fence_reg]; list_move_tail(®->lru_list, &dev_priv->mm.fence_list); if (obj->tiling_changed) { ret = i915_gem_object_flush_fence(obj, pipelined); if (ret) return ret; if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) pipelined = NULL; if (pipelined) { reg->setup_seqno = i915_gem_next_request_seqno(pipelined); obj->last_fenced_seqno = reg->setup_seqno; obj->last_fenced_ring = pipelined; } goto update; } if (!pipelined) { if (reg->setup_seqno) { if (!ring_passed_seqno(obj->last_fenced_ring, reg->setup_seqno)) { ret = i915_wait_request( obj->last_fenced_ring, reg->setup_seqno, true); if (ret) return ret; } reg->setup_seqno = 0; } } else if (obj->last_fenced_ring && obj->last_fenced_ring != pipelined) { ret = i915_gem_object_flush_fence(obj, pipelined); if (ret) return ret; } if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) pipelined = NULL; KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); if (obj->tiling_changed) { if (pipelined) { reg->setup_seqno = i915_gem_next_request_seqno(pipelined); obj->last_fenced_seqno = reg->setup_seqno; obj->last_fenced_ring = pipelined; } goto update; } return 0; } reg = i915_find_fence_reg(dev, pipelined); if (reg == NULL) return -EDEADLK; ret = i915_gem_object_flush_fence(obj, pipelined); if (ret) return ret; if (reg->obj) { struct drm_i915_gem_object *old = reg->obj; drm_gem_object_reference(&old->base); if (old->tiling_mode) i915_gem_release_mmap(old); ret = i915_gem_object_flush_fence(old, pipelined); if (ret) { drm_gem_object_unreference(&old->base); return ret; } if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) pipelined = NULL; old->fence_reg = I915_FENCE_REG_NONE; old->last_fenced_ring = pipelined; old->last_fenced_seqno = pipelined ? i915_gem_next_request_seqno(pipelined) : 0; drm_gem_object_unreference(&old->base); } else if (obj->last_fenced_seqno == 0) pipelined = NULL; reg->obj = obj; list_move_tail(®->lru_list, &dev_priv->mm.fence_list); obj->fence_reg = reg - dev_priv->fence_regs; obj->last_fenced_ring = pipelined; reg->setup_seqno = pipelined ? i915_gem_next_request_seqno(pipelined) : 0; obj->last_fenced_seqno = reg->setup_seqno; update: obj->tiling_changed = false; switch (INTEL_INFO(dev)->gen) { case 7: case 6: ret = sandybridge_write_fence_reg(obj, pipelined); break; case 5: case 4: ret = i965_write_fence_reg(obj, pipelined); break; case 3: ret = i915_write_fence_reg(obj, pipelined); break; case 2: ret = i830_write_fence_reg(obj, pipelined); break; } return ret; } static void i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) { drm_i915_private_t *dev_priv = dev->dev_private; uint32_t fence_reg = reg - dev_priv->fence_regs; switch (INTEL_INFO(dev)->gen) { case 7: case 6: I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); break; case 5: case 4: I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); break; case 3: if (fence_reg >= 8) fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; else case 2: fence_reg = FENCE_REG_830_0 + fence_reg * 4; I915_WRITE(fence_reg, 0); break; } list_del_init(®->lru_list); reg->obj = NULL; reg->setup_seqno = 0; reg->pin_count = 0; } int i915_gem_init_object(struct drm_gem_object *obj) { printf("i915_gem_init_object called\n"); return (0); } static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) { return (obj->gtt_space && !obj->active && obj->pin_count == 0); } static void i915_gem_retire_task_handler(void *arg, int pending) { drm_i915_private_t *dev_priv; struct drm_device *dev; bool idle; int i; dev_priv = arg; dev = dev_priv->dev; /* Come back later if the device is busy... */ if (!sx_try_xlock(&dev->dev_struct_lock)) { taskqueue_enqueue_timeout(dev_priv->tq, &dev_priv->mm.retire_task, hz); return; } CTR0(KTR_DRM, "retire_task"); i915_gem_retire_requests(dev); /* Send a periodic flush down the ring so we don't hold onto GEM * objects indefinitely. */ idle = true; for (i = 0; i < I915_NUM_RINGS; i++) { struct intel_ring_buffer *ring = &dev_priv->rings[i]; if (!list_empty(&ring->gpu_write_list)) { struct drm_i915_gem_request *request; int ret; ret = i915_gem_flush_ring(ring, 0, I915_GEM_GPU_DOMAINS); request = malloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO); if (ret || request == NULL || i915_add_request(ring, NULL, request)) free(request, DRM_I915_GEM); } idle &= list_empty(&ring->request_list); } if (!dev_priv->mm.suspended && !idle) taskqueue_enqueue_timeout(dev_priv->tq, &dev_priv->mm.retire_task, hz); DRM_UNLOCK(dev); } void i915_gem_lastclose(struct drm_device *dev) { int ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return; ret = i915_gem_idle(dev); if (ret != 0) DRM_ERROR("failed to idle hardware: %d\n", ret); } static int i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) { drm_i915_private_t *dev_priv; struct drm_i915_gem_phys_object *phys_obj; int ret; dev_priv = dev->dev_private; if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) return (0); phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, M_WAITOK | M_ZERO); phys_obj->id = id; phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); if (phys_obj->handle == NULL) { ret = -ENOMEM; goto free_obj; } pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, size / PAGE_SIZE, PAT_WRITE_COMBINING); dev_priv->mm.phys_objs[id - 1] = phys_obj; return (0); free_obj: free(phys_obj, DRM_I915_GEM); return (ret); } static void i915_gem_free_phys_object(struct drm_device *dev, int id) { drm_i915_private_t *dev_priv; struct drm_i915_gem_phys_object *phys_obj; dev_priv = dev->dev_private; if (dev_priv->mm.phys_objs[id - 1] == NULL) return; phys_obj = dev_priv->mm.phys_objs[id - 1]; if (phys_obj->cur_obj != NULL) i915_gem_detach_phys_object(dev, phys_obj->cur_obj); drm_pci_free(dev, phys_obj->handle); free(phys_obj, DRM_I915_GEM); dev_priv->mm.phys_objs[id - 1] = NULL; } void i915_gem_free_all_phys_object(struct drm_device *dev) { int i; for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) i915_gem_free_phys_object(dev, i); } void i915_gem_detach_phys_object(struct drm_device *dev, struct drm_i915_gem_object *obj) { vm_page_t m; struct sf_buf *sf; char *vaddr, *dst; int i, page_count; if (obj->phys_obj == NULL) return; vaddr = obj->phys_obj->handle->vaddr; page_count = obj->base.size / PAGE_SIZE; VM_OBJECT_WLOCK(obj->base.vm_obj); for (i = 0; i < page_count; i++) { m = i915_gem_wire_page(obj->base.vm_obj, i); if (m == NULL) continue; /* XXX */ VM_OBJECT_WUNLOCK(obj->base.vm_obj); sf = sf_buf_alloc(m, 0); if (sf != NULL) { dst = (char *)sf_buf_kva(sf); memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); sf_buf_free(sf); } drm_clflush_pages(&m, 1); VM_OBJECT_WLOCK(obj->base.vm_obj); vm_page_reference(m); vm_page_lock(m); vm_page_dirty(m); vm_page_unwire(m, PQ_INACTIVE); vm_page_unlock(m); atomic_add_long(&i915_gem_wired_pages_cnt, -1); } VM_OBJECT_WUNLOCK(obj->base.vm_obj); intel_gtt_chipset_flush(); obj->phys_obj->cur_obj = NULL; obj->phys_obj = NULL; } int i915_gem_attach_phys_object(struct drm_device *dev, struct drm_i915_gem_object *obj, int id, int align) { drm_i915_private_t *dev_priv; vm_page_t m; struct sf_buf *sf; char *dst, *src; int i, page_count, ret; if (id > I915_MAX_PHYS_OBJECT) return (-EINVAL); if (obj->phys_obj != NULL) { if (obj->phys_obj->id == id) return (0); i915_gem_detach_phys_object(dev, obj); } dev_priv = dev->dev_private; if (dev_priv->mm.phys_objs[id - 1] == NULL) { ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); if (ret != 0) { DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->base.size); return (ret); } } /* bind to the object */ obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; obj->phys_obj->cur_obj = obj; page_count = obj->base.size / PAGE_SIZE; VM_OBJECT_WLOCK(obj->base.vm_obj); ret = 0; for (i = 0; i < page_count; i++) { m = i915_gem_wire_page(obj->base.vm_obj, i); if (m == NULL) { ret = -EIO; break; } VM_OBJECT_WUNLOCK(obj->base.vm_obj); sf = sf_buf_alloc(m, 0); src = (char *)sf_buf_kva(sf); dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); memcpy(dst, src, PAGE_SIZE); sf_buf_free(sf); VM_OBJECT_WLOCK(obj->base.vm_obj); vm_page_reference(m); vm_page_lock(m); vm_page_unwire(m, PQ_INACTIVE); vm_page_unlock(m); atomic_add_long(&i915_gem_wired_pages_cnt, -1); } VM_OBJECT_WUNLOCK(obj->base.vm_obj); return (0); } static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, uint64_t size, struct drm_file *file_priv) { char *user_data, *vaddr; int ret; vaddr = (char *)obj->phys_obj->handle->vaddr + offset; user_data = (char *)(uintptr_t)data_ptr; if (copyin_nofault(user_data, vaddr, size) != 0) { /* The physical object once assigned is fixed for the lifetime * of the obj, so we can safely drop the lock and continue * to access vaddr. */ DRM_UNLOCK(dev); ret = -copyin(user_data, vaddr, size); DRM_LOCK(dev); if (ret != 0) return (ret); } intel_gtt_chipset_flush(); return (0); } static int i915_gpu_is_active(struct drm_device *dev) { drm_i915_private_t *dev_priv; dev_priv = dev->dev_private; return (!list_empty(&dev_priv->mm.flushing_list) || !list_empty(&dev_priv->mm.active_list)); } static void i915_gem_lowmem(void *arg) { struct drm_device *dev; struct drm_i915_private *dev_priv; struct drm_i915_gem_object *obj, *next; int cnt, cnt_fail, cnt_total; dev = arg; dev_priv = dev->dev_private; if (!sx_try_xlock(&dev->dev_struct_lock)) return; CTR0(KTR_DRM, "gem_lowmem"); rescan: /* first scan for clean buffers */ i915_gem_retire_requests(dev); cnt_total = cnt_fail = cnt = 0; list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, mm_list) { if (i915_gem_object_is_purgeable(obj)) { if (i915_gem_object_unbind(obj) != 0) cnt_total++; } else cnt_total++; } /* second pass, evict/count anything still on the inactive list */ list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, mm_list) { if (i915_gem_object_unbind(obj) == 0) cnt++; else cnt_fail++; } if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { /* * We are desperate for pages, so as a last resort, wait * for the GPU to finish and discard whatever we can. * This has a dramatic impact to reduce the number of * OOM-killer events whilst running the GPU aggressively. */ if (i915_gpu_idle(dev, true) == 0) goto rescan; } DRM_UNLOCK(dev); } void i915_gem_unload(struct drm_device *dev) { struct drm_i915_private *dev_priv; dev_priv = dev->dev_private; EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); } Index: head/sys/dev/drm2/i915/i915_gem_context.c =================================================================== --- head/sys/dev/drm2/i915/i915_gem_context.c (nonexistent) +++ head/sys/dev/drm2/i915/i915_gem_context.c (revision 271705) @@ -0,0 +1,549 @@ +/* + * Copyright © 2011-2012 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Authors: + * Ben Widawsky + * + */ + +/* + * This file implements HW context support. On gen5+ a HW context consists of an + * opaque GPU object which is referenced at times of context saves and restores. + * With RC6 enabled, the context is also referenced as the GPU enters and exists + * from RC6 (GPU has it's own internal power context, except on gen5). Though + * something like a context does exist for the media ring, the code only + * supports contexts for the render ring. + * + * In software, there is a distinction between contexts created by the user, + * and the default HW context. The default HW context is used by GPU clients + * that do not request setup of their own hardware context. The default + * context's state is never restored to help prevent programming errors. This + * would happen if a client ran and piggy-backed off another clients GPU state. + * The default context only exists to give the GPU some offset to load as the + * current to invoke a save of the context we actually care about. In fact, the + * code could likely be constructed, albeit in a more complicated fashion, to + * never use the default context, though that limits the driver's ability to + * swap out, and/or destroy other contexts. + * + * All other contexts are created as a request by the GPU client. These contexts + * store GPU state, and thus allow GPU clients to not re-emit state (and + * potentially query certain state) at any time. The kernel driver makes + * certain that the appropriate commands are inserted. + * + * The context life cycle is semi-complicated in that context BOs may live + * longer than the context itself because of the way the hardware, and object + * tracking works. Below is a very crude representation of the state machine + * describing the context life. + * refcount pincount active + * S0: initial state 0 0 0 + * S1: context created 1 0 0 + * S2: context is currently running 2 1 X + * S3: GPU referenced, but not current 2 0 1 + * S4: context is current, but destroyed 1 1 0 + * S5: like S3, but destroyed 1 0 1 + * + * The most common (but not all) transitions: + * S0->S1: client creates a context + * S1->S2: client submits execbuf with context + * S2->S3: other clients submits execbuf with context + * S3->S1: context object was retired + * S3->S2: clients submits another execbuf + * S2->S4: context destroy called with current context + * S3->S5->S0: destroy path + * S4->S5->S0: destroy path on current context + * + * There are two confusing terms used above: + * The "current context" means the context which is currently running on the + * GPU. The GPU has loaded it's state already and has stored away the gtt + * offset of the BO. The GPU is not actively referencing the data at this + * offset, but it will on the next context switch. The only way to avoid this + * is to do a GPU reset. + * + * An "active context' is one which was previously the "current context" and is + * on the active list waiting for the next context switch to occur. Until this + * happens, the object must remain at the same gtt offset. It is therefore + * possible to destroy a context, but it is still active. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include "i915_drv.h" + +/* This is a HW constraint. The value below is the largest known requirement + * I've seen in a spec to date, and that was a workaround for a non-shipping + * part. It should be safe to decrease this, but it's more future proof as is. + */ +#define CONTEXT_ALIGN (64<<10) + +static struct i915_hw_context * +i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id); +static int do_switch(struct i915_hw_context *to); + +static int get_context_size(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + u32 reg; + + switch (INTEL_INFO(dev)->gen) { + case 6: + reg = I915_READ(CXT_SIZE); + ret = GEN6_CXT_TOTAL_SIZE(reg) * 64; + break; + case 7: + reg = I915_READ(GEN7_CXT_SIZE); +#ifdef FREEBSD_WIP + if (IS_HASWELL(dev)) + ret = HSW_CXT_TOTAL_SIZE(reg) * 64; + else +#endif + ret = GEN7_CXT_TOTAL_SIZE(reg) * 64; + break; + default: + panic("i915_gem_context: Unsupported Intel GPU generation %d", + INTEL_INFO(dev)->gen); + } + + return ret; +} + +static void do_destroy(struct i915_hw_context *ctx) +{ +#if defined(INVARIANTS) + struct drm_device *dev = ctx->obj->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; +#endif + + if (ctx->file_priv) + drm_gem_names_remove(&ctx->file_priv->context_idr, ctx->id); + else + KASSERT(ctx == dev_priv->rings[RCS].default_context, + ("i915_gem_context: ctx != default_context")); + + drm_gem_object_unreference(&ctx->obj->base); + free(ctx, DRM_I915_GEM); +} + +static int +create_hw_context(struct drm_device *dev, + struct drm_i915_file_private *file_priv, + struct i915_hw_context **ret_ctx) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct i915_hw_context *ctx; + int ret, id; + + ctx = malloc(sizeof(*ctx), DRM_I915_GEM, M_NOWAIT | M_ZERO); + if (ctx == NULL) + return (-ENOMEM); + + ctx->obj = i915_gem_alloc_object(dev, dev_priv->hw_context_size); + if (ctx->obj == NULL) { + free(ctx, DRM_I915_GEM); + DRM_DEBUG_DRIVER("Context object allocated failed\n"); + return (-ENOMEM); + } + + if (INTEL_INFO(dev)->gen >= 7) { + ret = i915_gem_object_set_cache_level(ctx->obj, + I915_CACHE_LLC_MLC); + if (ret) + goto err_out; + } + + /* The ring associated with the context object is handled by the normal + * object tracking code. We give an initial ring value simple to pass an + * assertion in the context switch code. + */ + ctx->ring = &dev_priv->rings[RCS]; + + /* Default context will never have a file_priv */ + if (file_priv == NULL) { + *ret_ctx = ctx; + return (0); + } + + ctx->file_priv = file_priv; + +again: + id = 0; + ret = drm_gem_name_create(&file_priv->context_idr, ctx, &id); + if (ret == 0) + ctx->id = id; + + if (ret == -EAGAIN) + goto again; + else if (ret) + goto err_out; + + *ret_ctx = ctx; + return (0); + +err_out: + do_destroy(ctx); + return (ret); +} + +static inline bool is_default_context(struct i915_hw_context *ctx) +{ + return (ctx == ctx->ring->default_context); +} + +/** + * The default context needs to exist per ring that uses contexts. It stores the + * context state of the GPU for applications that don't utilize HW contexts, as + * well as an idle case. + */ +static int create_default_context(struct drm_i915_private *dev_priv) +{ + struct i915_hw_context *ctx; + int ret; + + DRM_LOCK_ASSERT(dev_priv->dev); + + ret = create_hw_context(dev_priv->dev, NULL, &ctx); + if (ret != 0) + return (ret); + + /* We may need to do things with the shrinker which require us to + * immediately switch back to the default context. This can cause a + * problem as pinning the default context also requires GTT space which + * may not be available. To avoid this we always pin the + * default context. + */ + dev_priv->rings[RCS].default_context = ctx; + ret = i915_gem_object_pin(ctx->obj, CONTEXT_ALIGN, false); + if (ret) + goto err_destroy; + + ret = do_switch(ctx); + if (ret) + goto err_unpin; + + DRM_DEBUG_DRIVER("Default HW context loaded\n"); + return 0; + +err_unpin: + i915_gem_object_unpin(ctx->obj); +err_destroy: + do_destroy(ctx); + return ret; +} + +void i915_gem_context_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + uint32_t ctx_size; + + if (!HAS_HW_CONTEXTS(dev)) { + dev_priv->hw_contexts_disabled = true; + return; + } + + /* If called from reset, or thaw... we've been here already */ + if (dev_priv->hw_contexts_disabled || + dev_priv->rings[RCS].default_context) + return; + + ctx_size = get_context_size(dev); + dev_priv->hw_context_size = get_context_size(dev); + dev_priv->hw_context_size = roundup(dev_priv->hw_context_size, 4096); + + if (ctx_size <= 0 || ctx_size > (1<<20)) { + dev_priv->hw_contexts_disabled = true; + return; + } + + if (create_default_context(dev_priv)) { + dev_priv->hw_contexts_disabled = true; + return; + } + + DRM_DEBUG_DRIVER("HW context support initialized\n"); +} + +void i915_gem_context_fini(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (dev_priv->hw_contexts_disabled) + return; + + /* The only known way to stop the gpu from accessing the hw context is + * to reset it. Do this as the very last operation to avoid confusing + * other code, leading to spurious errors. */ + intel_gpu_reset(dev); + + i915_gem_object_unpin(dev_priv->rings[RCS].default_context->obj); + + do_destroy(dev_priv->rings[RCS].default_context); +} + +static int context_idr_cleanup(uint32_t id, void *p, void *data) +{ + struct i915_hw_context *ctx = p; + + KASSERT(id != DEFAULT_CONTEXT_ID, ("i915_gem_context: id == DEFAULT_CONTEXT_ID in cleanup")); + + do_destroy(ctx); + + return 0; +} + +void i915_gem_context_close(struct drm_device *dev, struct drm_file *file) +{ + struct drm_i915_file_private *file_priv = file->driver_priv; + + //DRM_LOCK(dev); /* Called from preclose(), the lock is already owned. */ + drm_gem_names_foreach(&file_priv->context_idr, context_idr_cleanup, NULL); + drm_gem_names_fini(&file_priv->context_idr); + //DRM_UNLOCK(dev); +} + +static struct i915_hw_context * +i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id) +{ + return (struct i915_hw_context *)drm_gem_find_ptr(&file_priv->context_idr, id); +} + +static inline int +mi_set_context(struct intel_ring_buffer *ring, + struct i915_hw_context *new_context, + u32 hw_flags) +{ + int ret; + + /* w/a: If Flush TLB Invalidation Mode is enabled, driver must do a TLB + * invalidation prior to MI_SET_CONTEXT. On GEN6 we don't set the value + * explicitly, so we rely on the value at ring init, stored in + * itlb_before_ctx_switch. + */ + if (IS_GEN6(ring->dev) && ring->itlb_before_ctx_switch) { + ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, 0); + if (ret) + return ret; + } + + ret = intel_ring_begin(ring, 6); + if (ret) + return ret; + + if (IS_GEN7(ring->dev)) + intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_DISABLE); + else + intel_ring_emit(ring, MI_NOOP); + + intel_ring_emit(ring, MI_NOOP); + intel_ring_emit(ring, MI_SET_CONTEXT); + intel_ring_emit(ring, new_context->obj->gtt_offset | + MI_MM_SPACE_GTT | + MI_SAVE_EXT_STATE_EN | + MI_RESTORE_EXT_STATE_EN | + hw_flags); + /* w/a: MI_SET_CONTEXT must always be followed by MI_NOOP */ + intel_ring_emit(ring, MI_NOOP); + + if (IS_GEN7(ring->dev)) + intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_ENABLE); + else + intel_ring_emit(ring, MI_NOOP); + + intel_ring_advance(ring); + + return ret; +} + +static int do_switch(struct i915_hw_context *to) +{ + struct intel_ring_buffer *ring = to->ring; + struct drm_i915_gem_object *from_obj = ring->last_context_obj; + u32 hw_flags = 0; + int ret; + + KASSERT(!(from_obj != NULL && from_obj->pin_count == 0), + ("i915_gem_context: invalid \"from\" context")); + + if (from_obj == to->obj) + return 0; + + ret = i915_gem_object_pin(to->obj, CONTEXT_ALIGN, false); + if (ret) + return ret; + + /* Clear this page out of any CPU caches for coherent swap-in/out. Note + * that thanks to write = false in this call and us not setting any gpu + * write domains when putting a context object onto the active list + * (when switching away from it), this won't block. + * XXX: We need a real interface to do this instead of trickery. */ + ret = i915_gem_object_set_to_gtt_domain(to->obj, false); + if (ret) { + i915_gem_object_unpin(to->obj); + return ret; + } + + if (!to->obj->has_global_gtt_mapping) + i915_gem_gtt_bind_object(to->obj); + + if (!to->is_initialized || is_default_context(to)) + hw_flags |= MI_RESTORE_INHIBIT; + else if (from_obj == to->obj) /* not yet expected */ + hw_flags |= MI_FORCE_RESTORE; + + ret = mi_set_context(ring, to, hw_flags); + if (ret) { + i915_gem_object_unpin(to->obj); + return ret; + } + + /* The backing object for the context is done after switching to the + * *next* context. Therefore we cannot retire the previous context until + * the next context has already started running. In fact, the below code + * is a bit suboptimal because the retiring can occur simply after the + * MI_SET_CONTEXT instead of when the next seqno has completed. + */ + if (from_obj != NULL) { + from_obj->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION; + i915_gem_object_move_to_active(from_obj, ring, + i915_gem_next_request_seqno(ring)); + /* As long as MI_SET_CONTEXT is serializing, ie. it flushes the + * whole damn pipeline, we don't need to explicitly mark the + * object dirty. The only exception is that the context must be + * correct in case the object gets swapped out. Ideally we'd be + * able to defer doing this until we know the object would be + * swapped, but there is no way to do that yet. + */ + from_obj->dirty = 1; + KASSERT(from_obj->ring == ring, ("i915_gem_context: from_ring != ring")); + i915_gem_object_unpin(from_obj); + + drm_gem_object_unreference(&from_obj->base); + } + + drm_gem_object_reference(&to->obj->base); + ring->last_context_obj = to->obj; + to->is_initialized = true; + + return 0; +} + +/** + * i915_switch_context() - perform a GPU context switch. + * @ring: ring for which we'll execute the context switch + * @file_priv: file_priv associated with the context, may be NULL + * @id: context id number + * @seqno: sequence number by which the new context will be switched to + * @flags: + * + * The context life cycle is simple. The context refcount is incremented and + * decremented by 1 and create and destroy. If the context is in use by the GPU, + * it will have a refoucnt > 1. This allows us to destroy the context abstract + * object while letting the normal object tracking destroy the backing BO. + */ +int i915_switch_context(struct intel_ring_buffer *ring, + struct drm_file *file, + int to_id) +{ + struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct i915_hw_context *to; + + if (dev_priv->hw_contexts_disabled) + return 0; + + if (ring != &dev_priv->rings[RCS]) + return 0; + + if (to_id == DEFAULT_CONTEXT_ID) { + to = ring->default_context; + } else { + if (file == NULL) + return -EINVAL; + + to = i915_gem_context_get(file->driver_priv, to_id); + if (to == NULL) + return -ENOENT; + } + + return do_switch(to); +} + +int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_context_create *args = data; + struct drm_i915_file_private *file_priv = file->driver_priv; + struct i915_hw_context *ctx; + int ret; + + if (!(dev->driver->driver_features & DRIVER_GEM)) + return -ENODEV; + + if (dev_priv->hw_contexts_disabled) + return -ENODEV; + + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ret; + + ret = create_hw_context(dev, file_priv, &ctx); + DRM_UNLOCK(dev); + if (ret != 0) + return (ret); + + args->ctx_id = ctx->id; + DRM_DEBUG_DRIVER("HW context %d created\n", args->ctx_id); + + return 0; +} + +int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct drm_i915_gem_context_destroy *args = data; + struct drm_i915_file_private *file_priv = file->driver_priv; + struct i915_hw_context *ctx; + int ret; + + if (!(dev->driver->driver_features & DRIVER_GEM)) + return -ENODEV; + + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ret; + + ctx = i915_gem_context_get(file_priv, args->ctx_id); + if (!ctx) { + DRM_UNLOCK(dev); + return -ENOENT; + } + + do_destroy(ctx); + + DRM_UNLOCK(dev); + + DRM_DEBUG_DRIVER("HW context %d destroyed\n", args->ctx_id); + return 0; +} Property changes on: head/sys/dev/drm2/i915/i915_gem_context.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/drm2/i915/i915_gem_execbuffer.c =================================================================== --- head/sys/dev/drm2/i915/i915_gem_execbuffer.c (revision 271704) +++ head/sys/dev/drm2/i915/i915_gem_execbuffer.c (revision 271705) @@ -1,1528 +1,1544 @@ /* * Copyright © 2008,2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Eric Anholt * Chris Wilson * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include struct change_domains { uint32_t invalidate_domains; uint32_t flush_domains; uint32_t flush_rings; uint32_t flips; }; /* * Set the next domain for the specified object. This * may not actually perform the necessary flushing/invaliding though, * as that may want to be batched with other set_domain operations * * This is (we hope) the only really tricky part of gem. The goal * is fairly simple -- track which caches hold bits of the object * and make sure they remain coherent. A few concrete examples may * help to explain how it works. For shorthand, we use the notation * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the * a pair of read and write domain masks. * * Case 1: the batch buffer * * 1. Allocated * 2. Written by CPU * 3. Mapped to GTT * 4. Read by GPU * 5. Unmapped from GTT * 6. Freed * * Let's take these a step at a time * * 1. Allocated * Pages allocated from the kernel may still have * cache contents, so we set them to (CPU, CPU) always. * 2. Written by CPU (using pwrite) * The pwrite function calls set_domain (CPU, CPU) and * this function does nothing (as nothing changes) * 3. Mapped by GTT * This function asserts that the object is not * currently in any GPU-based read or write domains * 4. Read by GPU * i915_gem_execbuffer calls set_domain (COMMAND, 0). * As write_domain is zero, this function adds in the * current read domains (CPU+COMMAND, 0). * flush_domains is set to CPU. * invalidate_domains is set to COMMAND * clflush is run to get data out of the CPU caches * then i915_dev_set_domain calls i915_gem_flush to * emit an MI_FLUSH and drm_agp_chipset_flush * 5. Unmapped from GTT * i915_gem_object_unbind calls set_domain (CPU, CPU) * flush_domains and invalidate_domains end up both zero * so no flushing/invalidating happens * 6. Freed * yay, done * * Case 2: The shared render buffer * * 1. Allocated * 2. Mapped to GTT * 3. Read/written by GPU * 4. set_domain to (CPU,CPU) * 5. Read/written by CPU * 6. Read/written by GPU * * 1. Allocated * Same as last example, (CPU, CPU) * 2. Mapped to GTT * Nothing changes (assertions find that it is not in the GPU) * 3. Read/written by GPU * execbuffer calls set_domain (RENDER, RENDER) * flush_domains gets CPU * invalidate_domains gets GPU * clflush (obj) * MI_FLUSH and drm_agp_chipset_flush * 4. set_domain (CPU, CPU) * flush_domains gets GPU * invalidate_domains gets CPU * wait_rendering (obj) to make sure all drawing is complete. * This will include an MI_FLUSH to get the data from GPU * to memory * clflush (obj) to invalidate the CPU cache * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?) * 5. Read/written by CPU * cache lines are loaded and dirtied * 6. Read written by GPU * Same as last GPU access * * Case 3: The constant buffer * * 1. Allocated * 2. Written by CPU * 3. Read by GPU * 4. Updated (written) by CPU again * 5. Read by GPU * * 1. Allocated * (CPU, CPU) * 2. Written by CPU * (CPU, CPU) * 3. Read by GPU * (CPU+RENDER, 0) * flush_domains = CPU * invalidate_domains = RENDER * clflush (obj) * MI_FLUSH * drm_agp_chipset_flush * 4. Updated (written) by CPU again * (CPU, CPU) * flush_domains = 0 (no previous write domain) * invalidate_domains = 0 (no new read domains) * 5. Read by GPU * (CPU+RENDER, 0) * flush_domains = CPU * invalidate_domains = RENDER * clflush (obj) * MI_FLUSH * drm_agp_chipset_flush */ static void i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj, struct intel_ring_buffer *ring, struct change_domains *cd) { uint32_t invalidate_domains = 0, flush_domains = 0; /* * If the object isn't moving to a new write domain, * let the object stay in multiple read domains */ if (obj->base.pending_write_domain == 0) obj->base.pending_read_domains |= obj->base.read_domains; /* * Flush the current write domain if * the new read domains don't match. Invalidate * any read domains which differ from the old * write domain */ if (obj->base.write_domain && (((obj->base.write_domain != obj->base.pending_read_domains || obj->ring != ring)) || (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) { flush_domains |= obj->base.write_domain; invalidate_domains |= obj->base.pending_read_domains & ~obj->base.write_domain; } /* * Invalidate any read caches which may have * stale data. That is, any new read domains. */ invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains; if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) i915_gem_clflush_object(obj); if (obj->base.pending_write_domain) cd->flips |= atomic_load_acq_int(&obj->pending_flip); /* The actual obj->write_domain will be updated with * pending_write_domain after we emit the accumulated flush for all * of our domain changes in execbuffers (which clears objects' * write_domains). So if we have a current write domain that we * aren't changing, set pending_write_domain to that. */ if (flush_domains == 0 && obj->base.pending_write_domain == 0) obj->base.pending_write_domain = obj->base.write_domain; cd->invalidate_domains |= invalidate_domains; cd->flush_domains |= flush_domains; if (flush_domains & I915_GEM_GPU_DOMAINS) cd->flush_rings |= intel_ring_flag(obj->ring); if (invalidate_domains & I915_GEM_GPU_DOMAINS) cd->flush_rings |= intel_ring_flag(ring); } struct eb_objects { u_long hashmask; LIST_HEAD(, drm_i915_gem_object) *buckets; }; static struct eb_objects * eb_create(int size) { struct eb_objects *eb; eb = malloc(sizeof(*eb), DRM_I915_GEM, M_WAITOK | M_ZERO); eb->buckets = hashinit(size, DRM_I915_GEM, &eb->hashmask); return (eb); } static void eb_reset(struct eb_objects *eb) { int i; for (i = 0; i <= eb->hashmask; i++) LIST_INIT(&eb->buckets[i]); } static void eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj) { LIST_INSERT_HEAD(&eb->buckets[obj->exec_handle & eb->hashmask], obj, exec_node); } static struct drm_i915_gem_object * eb_get_object(struct eb_objects *eb, unsigned long handle) { struct drm_i915_gem_object *obj; LIST_FOREACH(obj, &eb->buckets[handle & eb->hashmask], exec_node) { if (obj->exec_handle == handle) return (obj); } return (NULL); } static void eb_destroy(struct eb_objects *eb) { free(eb->buckets, DRM_I915_GEM); free(eb, DRM_I915_GEM); } static int i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, struct eb_objects *eb, struct drm_i915_gem_relocation_entry *reloc) { struct drm_device *dev = obj->base.dev; struct drm_gem_object *target_obj; uint32_t target_offset; int ret = -EINVAL; /* we've already hold a reference to all valid objects */ target_obj = &eb_get_object(eb, reloc->target_handle)->base; if (unlikely(target_obj == NULL)) return -ENOENT; target_offset = to_intel_bo(target_obj)->gtt_offset; #if WATCH_RELOC DRM_INFO("%s: obj %p offset %08x target %d " "read %08x write %08x gtt %08x " "presumed %08x delta %08x\n", __func__, obj, (int) reloc->offset, (int) reloc->target_handle, (int) reloc->read_domains, (int) reloc->write_domain, (int) target_offset, (int) reloc->presumed_offset, reloc->delta); #endif /* The target buffer should have appeared before us in the * exec_object list, so it should have a GTT space bound by now. */ if (unlikely(target_offset == 0)) { DRM_DEBUG("No GTT space found for object %d\n", reloc->target_handle); return ret; } /* Validate that the target is in a valid r/w GPU domain */ if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { DRM_DEBUG("reloc with multiple write domains: " "obj %p target %d offset %d " "read %08x write %08x", obj, reloc->target_handle, (int) reloc->offset, reloc->read_domains, reloc->write_domain); return ret; } if (unlikely((reloc->write_domain | reloc->read_domains) & ~I915_GEM_GPU_DOMAINS)) { DRM_DEBUG("reloc with read/write non-GPU domains: " "obj %p target %d offset %d " "read %08x write %08x", obj, reloc->target_handle, (int) reloc->offset, reloc->read_domains, reloc->write_domain); return ret; } if (unlikely(reloc->write_domain && target_obj->pending_write_domain && reloc->write_domain != target_obj->pending_write_domain)) { DRM_DEBUG("Write domain conflict: " "obj %p target %d offset %d " "new %08x old %08x\n", obj, reloc->target_handle, (int) reloc->offset, reloc->write_domain, target_obj->pending_write_domain); return ret; } target_obj->pending_read_domains |= reloc->read_domains; target_obj->pending_write_domain |= reloc->write_domain; /* If the relocation already has the right value in it, no * more work needs to be done. */ if (target_offset == reloc->presumed_offset) return 0; /* Check that the relocation address is valid... */ if (unlikely(reloc->offset > obj->base.size - 4)) { DRM_DEBUG("Relocation beyond object bounds: " "obj %p target %d offset %d size %d.\n", obj, reloc->target_handle, (int) reloc->offset, (int) obj->base.size); return ret; } if (unlikely(reloc->offset & 3)) { DRM_DEBUG("Relocation not 4-byte aligned: " "obj %p target %d offset %d.\n", obj, reloc->target_handle, (int) reloc->offset); return ret; } reloc->delta += target_offset; if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) { uint32_t page_offset = reloc->offset & PAGE_MASK; char *vaddr; struct sf_buf *sf; sf = sf_buf_alloc(obj->pages[OFF_TO_IDX(reloc->offset)], SFB_NOWAIT); if (sf == NULL) return (-ENOMEM); vaddr = (void *)sf_buf_kva(sf); *(uint32_t *)(vaddr + page_offset) = reloc->delta; sf_buf_free(sf); } else { uint32_t *reloc_entry; char *reloc_page; /* We can't wait for rendering with pagefaults disabled */ if (obj->active && (curthread->td_pflags & TDP_NOFAULTING) != 0) return (-EFAULT); ret = i915_gem_object_set_to_gtt_domain(obj, 1); if (ret) return ret; /* * Map the page containing the relocation we're going * to perform. */ reloc->offset += obj->gtt_offset; reloc_page = pmap_mapdev_attr(dev->agp->base + (reloc->offset & ~PAGE_MASK), PAGE_SIZE, PAT_WRITE_COMBINING); reloc_entry = (uint32_t *)(reloc_page + (reloc->offset & PAGE_MASK)); *(volatile uint32_t *)reloc_entry = reloc->delta; pmap_unmapdev((vm_offset_t)reloc_page, PAGE_SIZE); } /* and update the user's relocation entry */ reloc->presumed_offset = target_offset; return 0; } static int i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj, struct eb_objects *eb) { struct drm_i915_gem_relocation_entry *user_relocs; struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; struct drm_i915_gem_relocation_entry reloc; int i, ret; user_relocs = (void *)(uintptr_t)entry->relocs_ptr; for (i = 0; i < entry->relocation_count; i++) { ret = -copyin_nofault(user_relocs + i, &reloc, sizeof(reloc)); if (ret != 0) return (ret); ret = i915_gem_execbuffer_relocate_entry(obj, eb, &reloc); if (ret != 0) return (ret); ret = -copyout_nofault(&reloc.presumed_offset, &user_relocs[i].presumed_offset, sizeof(reloc.presumed_offset)); if (ret != 0) return (ret); } return (0); } static int i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj, struct eb_objects *eb, struct drm_i915_gem_relocation_entry *relocs) { const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; int i, ret; for (i = 0; i < entry->relocation_count; i++) { ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]); if (ret) return ret; } return 0; } static int i915_gem_execbuffer_relocate(struct drm_device *dev, struct eb_objects *eb, struct list_head *objects) { struct drm_i915_gem_object *obj; int ret, pflags; /* Try to move as many of the relocation targets off the active list * to avoid unnecessary fallbacks to the slow path, as we cannot wait * for the retirement with pagefaults disabled. */ i915_gem_retire_requests(dev); ret = 0; pflags = vm_fault_disable_pagefaults(); /* This is the fast path and we cannot handle a pagefault whilst * holding the device lock lest the user pass in the relocations * contained within a mmaped bo. For in such a case we, the page * fault handler would call i915_gem_fault() and we would try to * acquire the device lock again. Obviously this is bad. */ list_for_each_entry(obj, objects, exec_list) { ret = i915_gem_execbuffer_relocate_object(obj, eb); if (ret != 0) break; } vm_fault_enable_pagefaults(pflags); return (ret); } #define __EXEC_OBJECT_HAS_FENCE (1<<31) static int pin_and_fence_object(struct drm_i915_gem_object *obj, struct intel_ring_buffer *ring) { struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; bool need_fence, need_mappable; int ret; need_fence = has_fenced_gpu_access && entry->flags & EXEC_OBJECT_NEEDS_FENCE && obj->tiling_mode != I915_TILING_NONE; need_mappable = entry->relocation_count ? true : need_fence; ret = i915_gem_object_pin(obj, entry->alignment, need_mappable); if (ret) return ret; if (has_fenced_gpu_access) { if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) { if (obj->tiling_mode) { ret = i915_gem_object_get_fence(obj, ring); if (ret) goto err_unpin; entry->flags |= __EXEC_OBJECT_HAS_FENCE; i915_gem_object_pin_fence(obj); } else { ret = i915_gem_object_put_fence(obj); if (ret) goto err_unpin; } obj->pending_fenced_gpu_access = true; } } entry->offset = obj->gtt_offset; return 0; err_unpin: i915_gem_object_unpin(obj); return ret; } static int i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, struct drm_file *file, struct list_head *objects) { drm_i915_private_t *dev_priv; struct drm_i915_gem_object *obj; int ret, retry; bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; struct list_head ordered_objects; dev_priv = ring->dev->dev_private; INIT_LIST_HEAD(&ordered_objects); while (!list_empty(objects)) { struct drm_i915_gem_exec_object2 *entry; bool need_fence, need_mappable; obj = list_first_entry(objects, struct drm_i915_gem_object, exec_list); entry = obj->exec_entry; need_fence = has_fenced_gpu_access && entry->flags & EXEC_OBJECT_NEEDS_FENCE && obj->tiling_mode != I915_TILING_NONE; need_mappable = entry->relocation_count ? true : need_fence; if (need_mappable) list_move(&obj->exec_list, &ordered_objects); else list_move_tail(&obj->exec_list, &ordered_objects); obj->base.pending_read_domains = 0; obj->base.pending_write_domain = 0; } list_splice(&ordered_objects, objects); /* Attempt to pin all of the buffers into the GTT. * This is done in 3 phases: * * 1a. Unbind all objects that do not match the GTT constraints for * the execbuffer (fenceable, mappable, alignment etc). * 1b. Increment pin count for already bound objects and obtain * a fence register if required. * 2. Bind new objects. * 3. Decrement pin count. * * This avoid unnecessary unbinding of later objects in order to makr * room for the earlier objects *unless* we need to defragment. */ retry = 0; do { ret = 0; /* Unbind any ill-fitting objects or pin. */ list_for_each_entry(obj, objects, exec_list) { struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; bool need_fence, need_mappable; if (!obj->gtt_space) continue; need_fence = has_fenced_gpu_access && entry->flags & EXEC_OBJECT_NEEDS_FENCE && obj->tiling_mode != I915_TILING_NONE; need_mappable = entry->relocation_count ? true : need_fence; if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) || (need_mappable && !obj->map_and_fenceable)) ret = i915_gem_object_unbind(obj); else ret = pin_and_fence_object(obj, ring); if (ret) goto err; } /* Bind fresh objects */ list_for_each_entry(obj, objects, exec_list) { if (obj->gtt_space) continue; ret = pin_and_fence_object(obj, ring); if (ret) { int ret_ignore; /* This can potentially raise a harmless * -EINVAL if we failed to bind in the above * call. It cannot raise -EINTR since we know * that the bo is freshly bound and so will * not need to be flushed or waited upon. */ ret_ignore = i915_gem_object_unbind(obj); (void)ret_ignore; if (obj->gtt_space != NULL) printf("%s: gtt_space\n", __func__); break; } } /* Decrement pin count for bound objects */ list_for_each_entry(obj, objects, exec_list) { struct drm_i915_gem_exec_object2 *entry; if (!obj->gtt_space) continue; entry = obj->exec_entry; if (entry->flags & __EXEC_OBJECT_HAS_FENCE) { i915_gem_object_unpin_fence(obj); entry->flags &= ~__EXEC_OBJECT_HAS_FENCE; } i915_gem_object_unpin(obj); /* ... and ensure ppgtt mapping exist if needed. */ if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) { i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, obj, obj->cache_level); obj->has_aliasing_ppgtt_mapping = 1; } } if (ret != -ENOSPC || retry > 1) return ret; /* First attempt, just clear anything that is purgeable. * Second attempt, clear the entire GTT. */ ret = i915_gem_evict_everything(ring->dev, retry == 0); if (ret) return ret; retry++; } while (1); err: list_for_each_entry_continue_reverse(obj, objects, exec_list) { struct drm_i915_gem_exec_object2 *entry; if (!obj->gtt_space) continue; entry = obj->exec_entry; if (entry->flags & __EXEC_OBJECT_HAS_FENCE) { i915_gem_object_unpin_fence(obj); entry->flags &= ~__EXEC_OBJECT_HAS_FENCE; } i915_gem_object_unpin(obj); } return ret; } static int i915_gem_execbuffer_relocate_slow(struct drm_device *dev, struct drm_file *file, struct intel_ring_buffer *ring, struct list_head *objects, struct eb_objects *eb, struct drm_i915_gem_exec_object2 *exec, int count) { struct drm_i915_gem_relocation_entry *reloc; struct drm_i915_gem_object *obj; int *reloc_offset; int i, total, ret; /* We may process another execbuffer during the unlock... */ while (!list_empty(objects)) { obj = list_first_entry(objects, struct drm_i915_gem_object, exec_list); list_del_init(&obj->exec_list); drm_gem_object_unreference(&obj->base); } DRM_UNLOCK(dev); total = 0; for (i = 0; i < count; i++) total += exec[i].relocation_count; reloc_offset = malloc(count * sizeof(*reloc_offset), DRM_I915_GEM, M_WAITOK | M_ZERO); reloc = malloc(total * sizeof(*reloc), DRM_I915_GEM, M_WAITOK | M_ZERO); total = 0; for (i = 0; i < count; i++) { struct drm_i915_gem_relocation_entry *user_relocs; user_relocs = (void *)(uintptr_t)exec[i].relocs_ptr; ret = -copyin(user_relocs, reloc + total, exec[i].relocation_count * sizeof(*reloc)); if (ret != 0) { DRM_LOCK(dev); goto err; } reloc_offset[i] = total; total += exec[i].relocation_count; } ret = i915_mutex_lock_interruptible(dev); if (ret) { DRM_LOCK(dev); goto err; } /* reacquire the objects */ eb_reset(eb); for (i = 0; i < count; i++) { struct drm_i915_gem_object *obj; obj = to_intel_bo(drm_gem_object_lookup(dev, file, exec[i].handle)); if (&obj->base == NULL) { DRM_DEBUG("Invalid object handle %d at index %d\n", exec[i].handle, i); ret = -ENOENT; goto err; } list_add_tail(&obj->exec_list, objects); obj->exec_handle = exec[i].handle; obj->exec_entry = &exec[i]; eb_add_object(eb, obj); } ret = i915_gem_execbuffer_reserve(ring, file, objects); if (ret) goto err; list_for_each_entry(obj, objects, exec_list) { int offset = obj->exec_entry - exec; ret = i915_gem_execbuffer_relocate_object_slow(obj, eb, reloc + reloc_offset[offset]); if (ret) goto err; } /* Leave the user relocations as are, this is the painfully slow path, * and we want to avoid the complication of dropping the lock whilst * having buffers reserved in the aperture and so causing spurious * ENOSPC for random operations. */ err: free(reloc, DRM_I915_GEM); free(reloc_offset, DRM_I915_GEM); return ret; } static int i915_gem_execbuffer_flush(struct drm_device *dev, uint32_t invalidate_domains, uint32_t flush_domains, uint32_t flush_rings) { drm_i915_private_t *dev_priv = dev->dev_private; int i, ret; if (flush_domains & I915_GEM_DOMAIN_CPU) intel_gtt_chipset_flush(); if (flush_domains & I915_GEM_DOMAIN_GTT) wmb(); if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) { for (i = 0; i < I915_NUM_RINGS; i++) if (flush_rings & (1 << i)) { ret = i915_gem_flush_ring(&dev_priv->rings[i], invalidate_domains, flush_domains); if (ret) return ret; } } return 0; } static bool intel_enable_semaphores(struct drm_device *dev) { if (INTEL_INFO(dev)->gen < 6) return 0; if (i915_semaphores >= 0) return i915_semaphores; /* Enable semaphores on SNB when IO remapping is off */ if (INTEL_INFO(dev)->gen == 6) return !intel_iommu_enabled; return 1; } static int i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj, struct intel_ring_buffer *to) { struct intel_ring_buffer *from = obj->ring; u32 seqno; int ret, idx; if (from == NULL || to == from) return 0; /* XXX gpu semaphores are implicated in various hard hangs on SNB */ if (!intel_enable_semaphores(obj->base.dev)) return i915_gem_object_wait_rendering(obj); idx = intel_ring_sync_index(from, to); seqno = obj->last_rendering_seqno; if (seqno <= from->sync_seqno[idx]) return 0; if (seqno == from->outstanding_lazy_request) { struct drm_i915_gem_request *request; request = malloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO); ret = i915_add_request(from, NULL, request); if (ret) { free(request, DRM_I915_GEM); return ret; } seqno = request->seqno; } from->sync_seqno[idx] = seqno; return to->sync_to(to, from, seqno - 1); } static int i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips) { u32 plane, flip_mask; int ret; /* Check for any pending flips. As we only maintain a flip queue depth * of 1, we can simply insert a WAIT for the next display flip prior * to executing the batch and avoid stalling the CPU. */ for (plane = 0; flips >> plane; plane++) { if (((flips >> plane) & 1) == 0) continue; if (plane) flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; else flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; ret = intel_ring_begin(ring, 2); if (ret) return ret; intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); } return 0; } static int i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring, struct list_head *objects) { struct drm_i915_gem_object *obj; struct change_domains cd; int ret; memset(&cd, 0, sizeof(cd)); list_for_each_entry(obj, objects, exec_list) i915_gem_object_set_to_gpu_domain(obj, ring, &cd); if (cd.invalidate_domains | cd.flush_domains) { #if WATCH_EXEC DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n", __func__, cd.invalidate_domains, cd.flush_domains); #endif ret = i915_gem_execbuffer_flush(ring->dev, cd.invalidate_domains, cd.flush_domains, cd.flush_rings); if (ret) return ret; } if (cd.flips) { ret = i915_gem_execbuffer_wait_for_flips(ring, cd.flips); if (ret) return ret; } list_for_each_entry(obj, objects, exec_list) { ret = i915_gem_execbuffer_sync_rings(obj, ring); if (ret) return ret; } return 0; } static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) { return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0; } static int validate_exec_list(struct drm_i915_gem_exec_object2 *exec, int count, vm_page_t ***map) { vm_page_t *ma; int i, length, page_count; /* XXXKIB various limits checking is missing there */ *map = malloc(count * sizeof(*ma), DRM_I915_GEM, M_WAITOK | M_ZERO); for (i = 0; i < count; i++) { /* First check for malicious input causing overflow */ if (exec[i].relocation_count > INT_MAX / sizeof(struct drm_i915_gem_relocation_entry)) return -EINVAL; length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry); if (length == 0) { (*map)[i] = NULL; continue; } /* * Since both start and end of the relocation region * may be not aligned on the page boundary, be * conservative and request a page slot for each * partial page. Thus +2. */ page_count = howmany(length, PAGE_SIZE) + 2; ma = (*map)[i] = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | M_ZERO); if (vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, exec[i].relocs_ptr, length, VM_PROT_READ | VM_PROT_WRITE, ma, page_count) == -1) { free(ma, DRM_I915_GEM); (*map)[i] = NULL; return (-EFAULT); } } return 0; } static void i915_gem_execbuffer_move_to_active(struct list_head *objects, struct intel_ring_buffer *ring, u32 seqno) { struct drm_i915_gem_object *obj; uint32_t old_read, old_write; list_for_each_entry(obj, objects, exec_list) { old_read = obj->base.read_domains; old_write = obj->base.write_domain; obj->base.read_domains = obj->base.pending_read_domains; obj->base.write_domain = obj->base.pending_write_domain; obj->fenced_gpu_access = obj->pending_fenced_gpu_access; i915_gem_object_move_to_active(obj, ring, seqno); if (obj->base.write_domain) { obj->dirty = 1; obj->pending_gpu_write = true; list_move_tail(&obj->gpu_write_list, &ring->gpu_write_list); intel_mark_busy(ring->dev, obj); } CTR3(KTR_DRM, "object_change_domain move_to_active %p %x %x", obj, old_read, old_write); } } int i915_gem_sync_exec_requests; static void i915_gem_execbuffer_retire_commands(struct drm_device *dev, struct drm_file *file, struct intel_ring_buffer *ring) { struct drm_i915_gem_request *request; u32 invalidate; /* * Ensure that the commands in the batch buffer are * finished before the interrupt fires. * * The sampler always gets flushed on i965 (sigh). */ invalidate = I915_GEM_DOMAIN_COMMAND; if (INTEL_INFO(dev)->gen >= 4) invalidate |= I915_GEM_DOMAIN_SAMPLER; if (ring->flush(ring, invalidate, 0)) { i915_gem_next_request_seqno(ring); return; } /* Add a breadcrumb for the completion of the batch buffer */ request = malloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO); if (request == NULL || i915_add_request(ring, file, request)) { i915_gem_next_request_seqno(ring); free(request, DRM_I915_GEM); } else if (i915_gem_sync_exec_requests) i915_wait_request(ring, request->seqno, true); } static void i915_gem_fix_mi_batchbuffer_end(struct drm_i915_gem_object *batch_obj, uint32_t batch_start_offset, uint32_t batch_len) { char *mkva; uint64_t po_r, po_w; uint32_t cmd; po_r = batch_obj->base.dev->agp->base + batch_obj->gtt_offset + batch_start_offset + batch_len; if (batch_len > 0) po_r -= 4; mkva = pmap_mapdev_attr(trunc_page(po_r), 2 * PAGE_SIZE, PAT_WRITE_COMBINING); po_r &= PAGE_MASK; cmd = *(uint32_t *)(mkva + po_r); if (cmd != MI_BATCH_BUFFER_END) { /* * batch_len != 0 due to the check at the start of * i915_gem_do_execbuffer */ if (batch_obj->base.size > batch_start_offset + batch_len) { po_w = po_r + 4; /* DRM_DEBUG("batchbuffer does not end by MI_BATCH_BUFFER_END !\n"); */ } else { po_w = po_r; DRM_DEBUG("batchbuffer does not end by MI_BATCH_BUFFER_END, overwriting last bo cmd !\n"); } *(uint32_t *)(mkva + po_w) = MI_BATCH_BUFFER_END; } pmap_unmapdev((vm_offset_t)mkva, 2 * PAGE_SIZE); } int i915_fix_mi_batchbuffer_end = 0; static int i915_reset_gen7_sol_offsets(struct drm_device *dev, struct intel_ring_buffer *ring) { drm_i915_private_t *dev_priv = dev->dev_private; int ret, i; if (!IS_GEN7(dev) || ring != &dev_priv->rings[RCS]) return 0; ret = intel_ring_begin(ring, 4 * 3); if (ret) return ret; for (i = 0; i < 4; i++) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i)); intel_ring_emit(ring, 0); } intel_ring_advance(ring); return 0; } static int i915_gem_do_execbuffer(struct drm_device *dev, void *data, struct drm_file *file, struct drm_i915_gem_execbuffer2 *args, struct drm_i915_gem_exec_object2 *exec) { drm_i915_private_t *dev_priv = dev->dev_private; struct list_head objects; struct eb_objects *eb; struct drm_i915_gem_object *batch_obj; struct drm_clip_rect *cliprects = NULL; struct intel_ring_buffer *ring; vm_page_t **relocs_ma; + u32 ctx_id = i915_execbuffer2_get_context_id(*args); u32 exec_start, exec_len; u32 seqno; u32 mask; int ret, mode, i; if (!i915_gem_check_execbuffer(args)) { DRM_DEBUG("execbuf with invalid offset/length\n"); return -EINVAL; } if (args->batch_len == 0) return (0); ret = validate_exec_list(exec, args->buffer_count, &relocs_ma); if (ret != 0) goto pre_struct_lock_err; switch (args->flags & I915_EXEC_RING_MASK) { case I915_EXEC_DEFAULT: case I915_EXEC_RENDER: ring = &dev_priv->rings[RCS]; break; case I915_EXEC_BSD: if (!HAS_BSD(dev)) { DRM_DEBUG("execbuf with invalid ring (BSD)\n"); return -EINVAL; } ring = &dev_priv->rings[VCS]; + if (ctx_id != 0) { + DRM_DEBUG("Ring %s doesn't support contexts\n", + ring->name); + return -EPERM; + } break; case I915_EXEC_BLT: if (!HAS_BLT(dev)) { DRM_DEBUG("execbuf with invalid ring (BLT)\n"); return -EINVAL; } ring = &dev_priv->rings[BCS]; + if (ctx_id != 0) { + DRM_DEBUG("Ring %s doesn't support contexts\n", + ring->name); + return -EPERM; + } break; default: DRM_DEBUG("execbuf with unknown ring: %d\n", (int)(args->flags & I915_EXEC_RING_MASK)); ret = -EINVAL; goto pre_struct_lock_err; } mode = args->flags & I915_EXEC_CONSTANTS_MASK; mask = I915_EXEC_CONSTANTS_MASK; switch (mode) { case I915_EXEC_CONSTANTS_REL_GENERAL: case I915_EXEC_CONSTANTS_ABSOLUTE: case I915_EXEC_CONSTANTS_REL_SURFACE: if (ring == &dev_priv->rings[RCS] && mode != dev_priv->relative_constants_mode) { if (INTEL_INFO(dev)->gen < 4) { ret = -EINVAL; goto pre_struct_lock_err; } if (INTEL_INFO(dev)->gen > 5 && mode == I915_EXEC_CONSTANTS_REL_SURFACE) { ret = -EINVAL; goto pre_struct_lock_err; } /* The HW changed the meaning on this bit on gen6 */ if (INTEL_INFO(dev)->gen >= 6) mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE; } break; default: DRM_DEBUG("execbuf with unknown constants: %d\n", mode); ret = -EINVAL; goto pre_struct_lock_err; } if (args->buffer_count < 1) { DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count); ret = -EINVAL; goto pre_struct_lock_err; } if (args->num_cliprects != 0) { if (ring != &dev_priv->rings[RCS]) { DRM_DEBUG("clip rectangles are only valid with the render ring\n"); ret = -EINVAL; goto pre_struct_lock_err; } if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) { DRM_DEBUG("execbuf with %u cliprects\n", args->num_cliprects); ret = -EINVAL; goto pre_struct_lock_err; } cliprects = malloc( sizeof(*cliprects) * args->num_cliprects, DRM_I915_GEM, M_WAITOK | M_ZERO); ret = -copyin((void *)(uintptr_t)args->cliprects_ptr, cliprects, sizeof(*cliprects) * args->num_cliprects); if (ret != 0) goto pre_struct_lock_err; } ret = i915_mutex_lock_interruptible(dev); if (ret) goto pre_struct_lock_err; if (dev_priv->mm.suspended) { ret = -EBUSY; goto struct_lock_err; } eb = eb_create(args->buffer_count); if (eb == NULL) { ret = -ENOMEM; goto struct_lock_err; } /* Look up object handles */ INIT_LIST_HEAD(&objects); for (i = 0; i < args->buffer_count; i++) { struct drm_i915_gem_object *obj; obj = to_intel_bo(drm_gem_object_lookup(dev, file, exec[i].handle)); if (&obj->base == NULL) { DRM_DEBUG("Invalid object handle %d at index %d\n", exec[i].handle, i); /* prevent error path from reading uninitialized data */ ret = -ENOENT; goto err; } if (!list_empty(&obj->exec_list)) { DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n", obj, exec[i].handle, i); ret = -EINVAL; goto err; } list_add_tail(&obj->exec_list, &objects); obj->exec_handle = exec[i].handle; obj->exec_entry = &exec[i]; eb_add_object(eb, obj); } /* take note of the batch buffer before we might reorder the lists */ batch_obj = list_entry(objects.prev, struct drm_i915_gem_object, exec_list); /* Move the objects en-masse into the GTT, evicting if necessary. */ ret = i915_gem_execbuffer_reserve(ring, file, &objects); if (ret) goto err; /* The objects are in their final locations, apply the relocations. */ ret = i915_gem_execbuffer_relocate(dev, eb, &objects); if (ret) { if (ret == -EFAULT) { ret = i915_gem_execbuffer_relocate_slow(dev, file, ring, &objects, eb, exec, args->buffer_count); DRM_LOCK_ASSERT(dev); } if (ret) goto err; } /* Set the pending read domains for the batch buffer to COMMAND */ if (batch_obj->base.pending_write_domain) { DRM_DEBUG("Attempting to use self-modifying batch buffer\n"); ret = -EINVAL; goto err; } batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND; ret = i915_gem_execbuffer_move_to_gpu(ring, &objects); if (ret) goto err; + ret = i915_switch_context(ring, file, ctx_id); + if (ret) + goto err; + seqno = i915_gem_next_request_seqno(ring); for (i = 0; i < I915_NUM_RINGS - 1; i++) { if (seqno < ring->sync_seqno[i]) { /* The GPU can not handle its semaphore value wrapping, * so every billion or so execbuffers, we need to stall * the GPU in order to reset the counters. */ ret = i915_gpu_idle(dev, true); if (ret) goto err; KASSERT(ring->sync_seqno[i] == 0, ("Non-zero sync_seqno")); } } if (ring == &dev_priv->rings[RCS] && mode != dev_priv->relative_constants_mode) { ret = intel_ring_begin(ring, 4); if (ret) goto err; intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit(ring, INSTPM); intel_ring_emit(ring, mask << 16 | mode); intel_ring_advance(ring); dev_priv->relative_constants_mode = mode; } if (args->flags & I915_EXEC_GEN7_SOL_RESET) { ret = i915_reset_gen7_sol_offsets(dev, ring); if (ret) goto err; } exec_start = batch_obj->gtt_offset + args->batch_start_offset; exec_len = args->batch_len; if (i915_fix_mi_batchbuffer_end) { i915_gem_fix_mi_batchbuffer_end(batch_obj, args->batch_start_offset, args->batch_len); } CTR4(KTR_DRM, "ring_dispatch %s %d exec %x %x", ring->name, seqno, exec_start, exec_len); if (cliprects) { for (i = 0; i < args->num_cliprects; i++) { ret = i915_emit_box_p(dev, &cliprects[i], args->DR1, args->DR4); if (ret) goto err; ret = ring->dispatch_execbuffer(ring, exec_start, exec_len); if (ret) goto err; } } else { ret = ring->dispatch_execbuffer(ring, exec_start, exec_len); if (ret) goto err; } i915_gem_execbuffer_move_to_active(&objects, ring, seqno); i915_gem_execbuffer_retire_commands(dev, file, ring); err: eb_destroy(eb); while (!list_empty(&objects)) { struct drm_i915_gem_object *obj; obj = list_first_entry(&objects, struct drm_i915_gem_object, exec_list); list_del_init(&obj->exec_list); drm_gem_object_unreference(&obj->base); } struct_lock_err: DRM_UNLOCK(dev); pre_struct_lock_err: for (i = 0; i < args->buffer_count; i++) { if (relocs_ma[i] != NULL) { vm_page_unhold_pages(relocs_ma[i], howmany( exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry), PAGE_SIZE)); free(relocs_ma[i], DRM_I915_GEM); } } free(relocs_ma, DRM_I915_GEM); free(cliprects, DRM_I915_GEM); return ret; } /* * Legacy execbuffer just creates an exec2 list from the original exec object * list array and passes it to the real function. */ int i915_gem_execbuffer(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_execbuffer *args = data; struct drm_i915_gem_execbuffer2 exec2; struct drm_i915_gem_exec_object *exec_list = NULL; struct drm_i915_gem_exec_object2 *exec2_list = NULL; int ret, i; DRM_DEBUG("buffers_ptr %d buffer_count %d len %08x\n", (int) args->buffers_ptr, args->buffer_count, args->batch_len); if (args->buffer_count < 1) { DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count); return -EINVAL; } /* Copy in the exec list from userland */ /* XXXKIB user-controlled malloc size */ exec_list = malloc(sizeof(*exec_list) * args->buffer_count, DRM_I915_GEM, M_WAITOK); exec2_list = malloc(sizeof(*exec2_list) * args->buffer_count, DRM_I915_GEM, M_WAITOK); ret = -copyin((void *)(uintptr_t)args->buffers_ptr, exec_list, sizeof(*exec_list) * args->buffer_count); if (ret != 0) { DRM_DEBUG("copy %d exec entries failed %d\n", args->buffer_count, ret); free(exec_list, DRM_I915_GEM); free(exec2_list, DRM_I915_GEM); return (ret); } for (i = 0; i < args->buffer_count; i++) { exec2_list[i].handle = exec_list[i].handle; exec2_list[i].relocation_count = exec_list[i].relocation_count; exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; exec2_list[i].alignment = exec_list[i].alignment; exec2_list[i].offset = exec_list[i].offset; if (INTEL_INFO(dev)->gen < 4) exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; else exec2_list[i].flags = 0; } exec2.buffers_ptr = args->buffers_ptr; exec2.buffer_count = args->buffer_count; exec2.batch_start_offset = args->batch_start_offset; exec2.batch_len = args->batch_len; exec2.DR1 = args->DR1; exec2.DR4 = args->DR4; exec2.num_cliprects = args->num_cliprects; exec2.cliprects_ptr = args->cliprects_ptr; exec2.flags = I915_EXEC_RENDER; + i915_execbuffer2_set_context_id(exec2, 0); ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list); if (!ret) { /* Copy the new buffer offsets back to the user's exec list. */ for (i = 0; i < args->buffer_count; i++) exec_list[i].offset = exec2_list[i].offset; /* ... and back out to userspace */ ret = -copyout(exec_list, (void *)(uintptr_t)args->buffers_ptr, sizeof(*exec_list) * args->buffer_count); if (ret != 0) { DRM_DEBUG("failed to copy %d exec entries " "back to user (%d)\n", args->buffer_count, ret); } } free(exec_list, DRM_I915_GEM); free(exec2_list, DRM_I915_GEM); return ret; } int i915_gem_execbuffer2(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_execbuffer2 *args = data; struct drm_i915_gem_exec_object2 *exec2_list = NULL; int ret; DRM_DEBUG("buffers_ptr %jx buffer_count %d len %08x\n", (uintmax_t)args->buffers_ptr, args->buffer_count, args->batch_len); if (args->buffer_count < 1 || args->buffer_count > UINT_MAX / sizeof(*exec2_list)) { DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count); return -EINVAL; } /* XXXKIB user-controllable malloc size */ exec2_list = malloc(sizeof(*exec2_list) * args->buffer_count, DRM_I915_GEM, M_WAITOK); ret = -copyin((void *)(uintptr_t)args->buffers_ptr, exec2_list, sizeof(*exec2_list) * args->buffer_count); if (ret != 0) { DRM_DEBUG("copy %d exec entries failed %d\n", args->buffer_count, ret); free(exec2_list, DRM_I915_GEM); return (ret); } ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list); if (!ret) { /* Copy the new buffer offsets back to the user's exec list. */ ret = -copyout(exec2_list, (void *)(uintptr_t)args->buffers_ptr, sizeof(*exec2_list) * args->buffer_count); if (ret) { DRM_DEBUG("failed to copy %d exec entries " "back to user (%d)\n", args->buffer_count, ret); } } free(exec2_list, DRM_I915_GEM); return ret; } Index: head/sys/dev/drm2/i915/i915_gem_gtt.c =================================================================== --- head/sys/dev/drm2/i915/i915_gem_gtt.c (revision 271704) +++ head/sys/dev/drm2/i915/i915_gem_gtt.c (revision 271705) @@ -1,329 +1,334 @@ /* * Copyright © 2010 Daniel Vetter * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include /* PPGTT support for Sandybdrige/Gen6 and later */ static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, unsigned first_entry, unsigned num_entries) { uint32_t *pt_vaddr; uint32_t scratch_pte; struct sf_buf *sf; unsigned act_pd, first_pte, last_pte, i; act_pd = first_entry / I915_PPGTT_PT_ENTRIES; first_pte = first_entry % I915_PPGTT_PT_ENTRIES; scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr); scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC; while (num_entries) { last_pte = first_pte + num_entries; if (last_pte > I915_PPGTT_PT_ENTRIES) last_pte = I915_PPGTT_PT_ENTRIES; sched_pin(); sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE); pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf); for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte; sf_buf_free(sf); sched_unpin(); num_entries -= last_pte - first_pte; first_pte = 0; act_pd++; } } int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) { struct drm_i915_private *dev_priv; struct i915_hw_ppgtt *ppgtt; u_int first_pd_entry_in_global_pt, i; dev_priv = dev->dev_private; /* * ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 * entries. For aliasing ppgtt support we just steal them at the end for * now. */ first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; ppgtt = malloc(sizeof(*ppgtt), DRM_I915_GEM, M_WAITOK | M_ZERO); ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; ppgtt->pt_pages = malloc(sizeof(vm_page_t) * ppgtt->num_pd_entries, DRM_I915_GEM, M_WAITOK | M_ZERO); for (i = 0; i < ppgtt->num_pd_entries; i++) { ppgtt->pt_pages[i] = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (ppgtt->pt_pages[i] == NULL) { dev_priv->mm.aliasing_ppgtt = ppgtt; i915_gem_cleanup_aliasing_ppgtt(dev); return (-ENOMEM); } } ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt.scratch_page_dma; i915_ppgtt_clear_range(ppgtt, 0, ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES); ppgtt->pd_offset = (first_pd_entry_in_global_pt) * sizeof(uint32_t); dev_priv->mm.aliasing_ppgtt = ppgtt; return (0); } static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, unsigned first_entry, unsigned num_entries, vm_page_t *pages, uint32_t pte_flags) { uint32_t *pt_vaddr, pte; struct sf_buf *sf; unsigned act_pd, first_pte; unsigned last_pte, i; vm_paddr_t page_addr; act_pd = first_entry / I915_PPGTT_PT_ENTRIES; first_pte = first_entry % I915_PPGTT_PT_ENTRIES; while (num_entries) { last_pte = first_pte + num_entries; if (last_pte > I915_PPGTT_PT_ENTRIES) last_pte = I915_PPGTT_PT_ENTRIES; sched_pin(); sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE); pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf); for (i = first_pte; i < last_pte; i++) { page_addr = VM_PAGE_TO_PHYS(*pages); pte = GEN6_PTE_ADDR_ENCODE(page_addr); pt_vaddr[i] = pte | pte_flags; pages++; } sf_buf_free(sf); sched_unpin(); num_entries -= last_pte - first_pte; first_pte = 0; act_pd++; } } void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_object *obj, enum i915_cache_level cache_level) { struct drm_device *dev; struct drm_i915_private *dev_priv; uint32_t pte_flags; dev = obj->base.dev; dev_priv = dev->dev_private; pte_flags = GEN6_PTE_VALID; switch (cache_level) { case I915_CACHE_LLC_MLC: pte_flags |= GEN6_PTE_CACHE_LLC_MLC; break; case I915_CACHE_LLC: pte_flags |= GEN6_PTE_CACHE_LLC; break; case I915_CACHE_NONE: pte_flags |= GEN6_PTE_UNCACHED; break; default: panic("cache mode"); } i915_ppgtt_insert_pages(ppgtt, obj->gtt_space->start >> PAGE_SHIFT, obj->base.size >> PAGE_SHIFT, obj->pages, pte_flags); } void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_object *obj) { i915_ppgtt_clear_range(ppgtt, obj->gtt_space->start >> PAGE_SHIFT, obj->base.size >> PAGE_SHIFT); } void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) { struct drm_i915_private *dev_priv; struct i915_hw_ppgtt *ppgtt; vm_page_t m; int i; dev_priv = dev->dev_private; ppgtt = dev_priv->mm.aliasing_ppgtt; if (ppgtt == NULL) return; dev_priv->mm.aliasing_ppgtt = NULL; for (i = 0; i < ppgtt->num_pd_entries; i++) { m = ppgtt->pt_pages[i]; if (m != NULL) { vm_page_unwire(m, PQ_INACTIVE); vm_page_free(m); } } free(ppgtt->pt_pages, DRM_I915_GEM); free(ppgtt, DRM_I915_GEM); } static unsigned int cache_level_to_agp_type(struct drm_device *dev, enum i915_cache_level cache_level) { switch (cache_level) { case I915_CACHE_LLC_MLC: if (INTEL_INFO(dev)->gen >= 6) return (AGP_USER_CACHED_MEMORY_LLC_MLC); /* * Older chipsets do not have this extra level of CPU * cacheing, so fallthrough and request the PTE simply * as cached. */ case I915_CACHE_LLC: return (AGP_USER_CACHED_MEMORY); default: case I915_CACHE_NONE: return (AGP_USER_MEMORY); } } static bool do_idling(struct drm_i915_private *dev_priv) { bool ret = dev_priv->mm.interruptible; if (dev_priv->mm.gtt.do_idle_maps) { dev_priv->mm.interruptible = false; if (i915_gpu_idle(dev_priv->dev, false)) { DRM_ERROR("Couldn't idle GPU\n"); /* Wait a bit, in hopes it avoids the hang */ DELAY(10); } } return ret; } static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) { if (dev_priv->mm.gtt.do_idle_maps) dev_priv->mm.interruptible = interruptible; } void i915_gem_restore_gtt_mappings(struct drm_device *dev) { struct drm_i915_private *dev_priv; struct drm_i915_gem_object *obj; dev_priv = dev->dev_private; /* First fill our portion of the GTT with scratch pages */ intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE, (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { i915_gem_clflush_object(obj); i915_gem_gtt_rebind_object(obj, obj->cache_level); } intel_gtt_chipset_flush(); } int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj) { unsigned int agp_type; agp_type = cache_level_to_agp_type(obj->base.dev, obj->cache_level); intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, obj->base.size >> PAGE_SHIFT, obj->pages, agp_type); + + obj->has_global_gtt_mapping = 1; + return (0); } void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level) { struct drm_device *dev; struct drm_i915_private *dev_priv; unsigned int agp_type; dev = obj->base.dev; dev_priv = dev->dev_private; agp_type = cache_level_to_agp_type(dev, cache_level); intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, obj->base.size >> PAGE_SHIFT, obj->pages, agp_type); + + obj->has_global_gtt_mapping = 0; } void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; bool interruptible; dev = obj->base.dev; dev_priv = dev->dev_private; interruptible = do_idling(dev_priv); intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT, obj->base.size >> PAGE_SHIFT); undo_idling(dev_priv, interruptible); } Index: head/sys/dev/drm2/i915/i915_irq.c =================================================================== --- head/sys/dev/drm2/i915/i915_irq.c (revision 271704) +++ head/sys/dev/drm2/i915/i915_irq.c (revision 271705) @@ -1,2262 +1,2262 @@ /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*- */ /*- * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include static void i915_capture_error_state(struct drm_device *dev); static u32 ring_last_seqno(struct intel_ring_buffer *ring); /** * Interrupts that are always left unmasked. * * Since pipe events are edge-triggered from the PIPESTAT register to IIR, * we leave them always unmasked in IMR and then control enabling them through * PIPESTAT alone. */ #define I915_INTERRUPT_ENABLE_FIX \ (I915_ASLE_INTERRUPT | \ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | \ I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | \ I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT | \ I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) /** Interrupts that we mask and unmask at runtime. */ #define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT | I915_BSD_USER_INTERRUPT) #define I915_PIPE_VBLANK_STATUS (PIPE_START_VBLANK_INTERRUPT_STATUS |\ PIPE_VBLANK_INTERRUPT_STATUS) #define I915_PIPE_VBLANK_ENABLE (PIPE_START_VBLANK_INTERRUPT_ENABLE |\ PIPE_VBLANK_INTERRUPT_ENABLE) #define DRM_I915_VBLANK_PIPE_ALL (DRM_I915_VBLANK_PIPE_A | \ DRM_I915_VBLANK_PIPE_B) /* For display hotplug interrupt */ static void ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask) { if ((dev_priv->irq_mask & mask) != 0) { dev_priv->irq_mask &= ~mask; I915_WRITE(DEIMR, dev_priv->irq_mask); POSTING_READ(DEIMR); } } static inline void ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask) { if ((dev_priv->irq_mask & mask) != mask) { dev_priv->irq_mask |= mask; I915_WRITE(DEIMR, dev_priv->irq_mask); POSTING_READ(DEIMR); } } void i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask) { if ((dev_priv->pipestat[pipe] & mask) != mask) { u32 reg = PIPESTAT(pipe); dev_priv->pipestat[pipe] |= mask; /* Enable the interrupt, clear any pending status */ I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16)); POSTING_READ(reg); } } void i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask) { if ((dev_priv->pipestat[pipe] & mask) != 0) { u32 reg = PIPESTAT(pipe); dev_priv->pipestat[pipe] &= ~mask; I915_WRITE(reg, dev_priv->pipestat[pipe]); POSTING_READ(reg); } } /** * intel_enable_asle - enable ASLE interrupt for OpRegion */ void intel_enable_asle(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; mtx_lock(&dev_priv->irq_lock); if (HAS_PCH_SPLIT(dev)) ironlake_enable_display_irq(dev_priv, DE_GSE); else { i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE); if (INTEL_INFO(dev)->gen >= 4) i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE); } mtx_unlock(&dev_priv->irq_lock); } /** * i915_pipe_enabled - check if a pipe is enabled * @dev: DRM device * @pipe: pipe to check * * Reading certain registers when the pipe is disabled can hang the chip. * Use this routine to make sure the PLL is running and the pipe is active * before reading such registers if unsure. */ static int i915_pipe_enabled(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE; } /* Called from drm generic code, passed a 'crtc', which * we use as a pipe index */ static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; unsigned long high_frame; unsigned long low_frame; u32 high1, high2, low; if (!i915_pipe_enabled(dev, pipe)) { DRM_DEBUG("trying to get vblank count for disabled " "pipe %c\n", pipe_name(pipe)); return 0; } high_frame = PIPEFRAME(pipe); low_frame = PIPEFRAMEPIXEL(pipe); /* * High & low register fields aren't synchronized, so make sure * we get a low value that's stable across two reads of the high * register. */ do { high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK; low = I915_READ(low_frame) & PIPE_FRAME_LOW_MASK; high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK; } while (high1 != high2); high1 >>= PIPE_FRAME_HIGH_SHIFT; low >>= PIPE_FRAME_LOW_SHIFT; return (high1 << 8) | low; } static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; int reg = PIPE_FRMCOUNT_GM45(pipe); if (!i915_pipe_enabled(dev, pipe)) { DRM_DEBUG("i915: trying to get vblank count for disabled " "pipe %c\n", pipe_name(pipe)); return 0; } return I915_READ(reg); } static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, int *vpos, int *hpos) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; u32 vbl = 0, position = 0; int vbl_start, vbl_end, htotal, vtotal; bool in_vbl = true; int ret = 0; if (!i915_pipe_enabled(dev, pipe)) { DRM_DEBUG("i915: trying to get scanoutpos for disabled " "pipe %c\n", pipe_name(pipe)); return 0; } /* Get vtotal. */ vtotal = 1 + ((I915_READ(VTOTAL(pipe)) >> 16) & 0x1fff); if (INTEL_INFO(dev)->gen >= 4) { /* No obvious pixelcount register. Only query vertical * scanout position from Display scan line register. */ position = I915_READ(PIPEDSL(pipe)); /* Decode into vertical scanout position. Don't have * horizontal scanout position. */ *vpos = position & 0x1fff; *hpos = 0; } else { /* Have access to pixelcount since start of frame. * We can split this into vertical and horizontal * scanout position. */ position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; htotal = 1 + ((I915_READ(HTOTAL(pipe)) >> 16) & 0x1fff); *vpos = position / htotal; *hpos = position - (*vpos * htotal); } /* Query vblank area. */ vbl = I915_READ(VBLANK(pipe)); /* Test position against vblank region. */ vbl_start = vbl & 0x1fff; vbl_end = (vbl >> 16) & 0x1fff; if ((*vpos < vbl_start) || (*vpos > vbl_end)) in_vbl = false; /* Inside "upper part" of vblank area? Apply corrective offset: */ if (in_vbl && (*vpos >= vbl_start)) *vpos = *vpos - vtotal; /* Readouts valid? */ if (vbl > 0) ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE; /* In vblank? */ if (in_vbl) ret |= DRM_SCANOUTPOS_INVBL; return ret; } static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe, int *max_error, struct timeval *vblank_time, unsigned flags) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc; if (pipe < 0 || pipe >= dev_priv->num_pipe) { DRM_ERROR("Invalid crtc %d\n", pipe); return -EINVAL; } /* Get drm_crtc to timestamp: */ crtc = intel_get_crtc_for_pipe(dev, pipe); if (crtc == NULL) { DRM_ERROR("Invalid crtc %d\n", pipe); return -EINVAL; } if (!crtc->enabled) { #if 0 DRM_DEBUG("crtc %d is disabled\n", pipe); #endif return -EBUSY; } /* Helper routine in DRM core does all the work: */ return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, vblank_time, flags, crtc); } /* * Handle hotplug events outside the interrupt handler proper. */ static void i915_hotplug_work_func(void *context, int pending) { drm_i915_private_t *dev_priv = context; struct drm_device *dev = dev_priv->dev; struct drm_mode_config *mode_config; struct intel_encoder *encoder; DRM_DEBUG("running encoder hotplug functions\n"); dev_priv = context; dev = dev_priv->dev; mode_config = &dev->mode_config; sx_xlock(&mode_config->mutex); DRM_DEBUG_KMS("running encoder hotplug functions\n"); list_for_each_entry(encoder, &mode_config->encoder_list, base.head) if (encoder->hot_plug) encoder->hot_plug(encoder); sx_xunlock(&mode_config->mutex); /* Just fire off a uevent and let userspace tell us what to do */ #if 0 drm_helper_hpd_irq_event(dev); #endif } static void i915_handle_rps_change(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; u32 busy_up, busy_down, max_avg, min_avg; u8 new_delay = dev_priv->cur_delay; I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG); busy_up = I915_READ(RCPREVBSYTUPAVG); busy_down = I915_READ(RCPREVBSYTDNAVG); max_avg = I915_READ(RCBMAXAVG); min_avg = I915_READ(RCBMINAVG); /* Handle RCS change request from hw */ if (busy_up > max_avg) { if (dev_priv->cur_delay != dev_priv->max_delay) new_delay = dev_priv->cur_delay - 1; if (new_delay < dev_priv->max_delay) new_delay = dev_priv->max_delay; } else if (busy_down < min_avg) { if (dev_priv->cur_delay != dev_priv->min_delay) new_delay = dev_priv->cur_delay + 1; if (new_delay > dev_priv->min_delay) new_delay = dev_priv->min_delay; } if (ironlake_set_drps(dev, new_delay)) dev_priv->cur_delay = new_delay; return; } static void notify_ring(struct drm_device *dev, struct intel_ring_buffer *ring) { struct drm_i915_private *dev_priv = dev->dev_private; u32 seqno; if (ring->obj == NULL) return; seqno = ring->get_seqno(ring); CTR2(KTR_DRM, "request_complete %s %d", ring->name, seqno); mtx_lock(&ring->irq_lock); ring->irq_seqno = seqno; wakeup(ring); mtx_unlock(&ring->irq_lock); if (i915_enable_hangcheck) { dev_priv->hangcheck_count = 0; callout_schedule(&dev_priv->hangcheck_timer, DRM_I915_HANGCHECK_PERIOD); } } static void gen6_pm_rps_work_func(void *arg, int pending) { struct drm_device *dev; drm_i915_private_t *dev_priv; u8 new_delay; u32 pm_iir, pm_imr; dev_priv = (drm_i915_private_t *)arg; dev = dev_priv->dev; new_delay = dev_priv->cur_delay; mtx_lock(&dev_priv->rps_lock); pm_iir = dev_priv->pm_iir; dev_priv->pm_iir = 0; pm_imr = I915_READ(GEN6_PMIMR); I915_WRITE(GEN6_PMIMR, 0); mtx_unlock(&dev_priv->rps_lock); if (!pm_iir) return; DRM_LOCK(dev); if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { if (dev_priv->cur_delay != dev_priv->max_delay) new_delay = dev_priv->cur_delay + 1; if (new_delay > dev_priv->max_delay) new_delay = dev_priv->max_delay; } else if (pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT)) { gen6_gt_force_wake_get(dev_priv); if (dev_priv->cur_delay != dev_priv->min_delay) new_delay = dev_priv->cur_delay - 1; if (new_delay < dev_priv->min_delay) { new_delay = dev_priv->min_delay; I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, I915_READ(GEN6_RP_INTERRUPT_LIMITS) | ((new_delay << 16) & 0x3f0000)); } else { /* Make sure we continue to get down interrupts * until we hit the minimum frequency */ I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, I915_READ(GEN6_RP_INTERRUPT_LIMITS) & ~0x3f0000); } gen6_gt_force_wake_put(dev_priv); } gen6_set_rps(dev, new_delay); dev_priv->cur_delay = new_delay; /* * rps_lock not held here because clearing is non-destructive. There is * an *extremely* unlikely race with gen6_rps_enable() that is prevented * by holding struct_mutex for the duration of the write. */ DRM_UNLOCK(dev); } static void pch_irq_handler(struct drm_device *dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; u32 pch_iir; int pipe; pch_iir = I915_READ(SDEIIR); if (pch_iir & SDE_AUDIO_POWER_MASK) DRM_DEBUG("i915: PCH audio power change on port %d\n", (pch_iir & SDE_AUDIO_POWER_MASK) >> SDE_AUDIO_POWER_SHIFT); if (pch_iir & SDE_GMBUS) DRM_DEBUG("i915: PCH GMBUS interrupt\n"); if (pch_iir & SDE_AUDIO_HDCP_MASK) DRM_DEBUG("i915: PCH HDCP audio interrupt\n"); if (pch_iir & SDE_AUDIO_TRANS_MASK) DRM_DEBUG("i915: PCH transcoder audio interrupt\n"); if (pch_iir & SDE_POISON) DRM_ERROR("i915: PCH poison interrupt\n"); if (pch_iir & SDE_FDI_MASK) for_each_pipe(pipe) DRM_DEBUG(" pipe %c FDI IIR: 0x%08x\n", pipe_name(pipe), I915_READ(FDI_RX_IIR(pipe))); if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE)) DRM_DEBUG("i915: PCH transcoder CRC done interrupt\n"); if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR)) DRM_DEBUG("i915: PCH transcoder CRC error interrupt\n"); if (pch_iir & SDE_TRANSB_FIFO_UNDER) DRM_DEBUG("i915: PCH transcoder B underrun interrupt\n"); if (pch_iir & SDE_TRANSA_FIFO_UNDER) DRM_DEBUG("PCH transcoder A underrun interrupt\n"); } static void ivybridge_irq_handler(void *arg) { struct drm_device *dev = (struct drm_device *) arg; drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir; #if 0 struct drm_i915_master_private *master_priv; #endif atomic_inc(&dev_priv->irq_received); /* disable master interrupt before clearing iir */ de_ier = I915_READ(DEIER); I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); POSTING_READ(DEIER); de_iir = I915_READ(DEIIR); gt_iir = I915_READ(GTIIR); pch_iir = I915_READ(SDEIIR); pm_iir = I915_READ(GEN6_PMIIR); CTR4(KTR_DRM, "ivybridge_irq de %x gt %x pch %x pm %x", de_iir, gt_iir, pch_iir, pm_iir); if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 && pm_iir == 0) goto done; #if 0 if (dev->primary->master) { master_priv = dev->primary->master->driver_priv; if (master_priv->sarea_priv) master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); } #else if (dev_priv->sarea_priv) dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); #endif if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY)) notify_ring(dev, &dev_priv->rings[RCS]); if (gt_iir & GT_GEN6_BSD_USER_INTERRUPT) notify_ring(dev, &dev_priv->rings[VCS]); if (gt_iir & GT_BLT_USER_INTERRUPT) notify_ring(dev, &dev_priv->rings[BCS]); if (de_iir & DE_GSE_IVB) { intel_opregion_gse_intr(dev); } if (de_iir & DE_PLANEA_FLIP_DONE_IVB) { intel_prepare_page_flip(dev, 0); intel_finish_page_flip_plane(dev, 0); } if (de_iir & DE_PLANEB_FLIP_DONE_IVB) { intel_prepare_page_flip(dev, 1); intel_finish_page_flip_plane(dev, 1); } if (de_iir & DE_PIPEA_VBLANK_IVB) drm_handle_vblank(dev, 0); if (de_iir & DE_PIPEB_VBLANK_IVB) drm_handle_vblank(dev, 1); /* check event from PCH */ if (de_iir & DE_PCH_EVENT_IVB) { if (pch_iir & SDE_HOTPLUG_MASK_CPT) taskqueue_enqueue(dev_priv->tq, &dev_priv->hotplug_task); pch_irq_handler(dev); } if (pm_iir & GEN6_PM_DEFERRED_EVENTS) { mtx_lock(&dev_priv->rps_lock); if ((dev_priv->pm_iir & pm_iir) != 0) printf("Missed a PM interrupt\n"); dev_priv->pm_iir |= pm_iir; I915_WRITE(GEN6_PMIMR, dev_priv->pm_iir); POSTING_READ(GEN6_PMIMR); mtx_unlock(&dev_priv->rps_lock); taskqueue_enqueue(dev_priv->tq, &dev_priv->rps_task); } /* should clear PCH hotplug event before clear CPU irq */ I915_WRITE(SDEIIR, pch_iir); I915_WRITE(GTIIR, gt_iir); I915_WRITE(DEIIR, de_iir); I915_WRITE(GEN6_PMIIR, pm_iir); done: I915_WRITE(DEIER, de_ier); POSTING_READ(DEIER); } static void ironlake_irq_handler(void *arg) { struct drm_device *dev = arg; drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir; u32 hotplug_mask; #if 0 struct drm_i915_master_private *master_priv; #endif u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT; atomic_inc(&dev_priv->irq_received); if (IS_GEN6(dev)) bsd_usr_interrupt = GT_GEN6_BSD_USER_INTERRUPT; /* disable master interrupt before clearing iir */ de_ier = I915_READ(DEIER); I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); POSTING_READ(DEIER); de_iir = I915_READ(DEIIR); gt_iir = I915_READ(GTIIR); pch_iir = I915_READ(SDEIIR); pm_iir = I915_READ(GEN6_PMIIR); CTR4(KTR_DRM, "ironlake_irq de %x gt %x pch %x pm %x", de_iir, gt_iir, pch_iir, pm_iir); if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 && (!IS_GEN6(dev) || pm_iir == 0)) goto done; if (HAS_PCH_CPT(dev)) hotplug_mask = SDE_HOTPLUG_MASK_CPT; else hotplug_mask = SDE_HOTPLUG_MASK; #if 0 if (dev->primary->master) { master_priv = dev->primary->master->driver_priv; if (master_priv->sarea_priv) master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); } #else if (dev_priv->sarea_priv) dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); #endif if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY)) notify_ring(dev, &dev_priv->rings[RCS]); if (gt_iir & bsd_usr_interrupt) notify_ring(dev, &dev_priv->rings[VCS]); if (gt_iir & GT_BLT_USER_INTERRUPT) notify_ring(dev, &dev_priv->rings[BCS]); if (de_iir & DE_GSE) { intel_opregion_gse_intr(dev); } if (de_iir & DE_PLANEA_FLIP_DONE) { intel_prepare_page_flip(dev, 0); intel_finish_page_flip_plane(dev, 0); } if (de_iir & DE_PLANEB_FLIP_DONE) { intel_prepare_page_flip(dev, 1); intel_finish_page_flip_plane(dev, 1); } if (de_iir & DE_PIPEA_VBLANK) drm_handle_vblank(dev, 0); if (de_iir & DE_PIPEB_VBLANK) drm_handle_vblank(dev, 1); /* check event from PCH */ if (de_iir & DE_PCH_EVENT) { if (pch_iir & hotplug_mask) taskqueue_enqueue(dev_priv->tq, &dev_priv->hotplug_task); pch_irq_handler(dev); } if (de_iir & DE_PCU_EVENT) { I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS)); i915_handle_rps_change(dev); } if (pm_iir & GEN6_PM_DEFERRED_EVENTS) { mtx_lock(&dev_priv->rps_lock); if ((dev_priv->pm_iir & pm_iir) != 0) printf("Missed a PM interrupt\n"); dev_priv->pm_iir |= pm_iir; I915_WRITE(GEN6_PMIMR, dev_priv->pm_iir); POSTING_READ(GEN6_PMIMR); mtx_unlock(&dev_priv->rps_lock); taskqueue_enqueue(dev_priv->tq, &dev_priv->rps_task); } /* should clear PCH hotplug event before clear CPU irq */ I915_WRITE(SDEIIR, pch_iir); I915_WRITE(GTIIR, gt_iir); I915_WRITE(DEIIR, de_iir); I915_WRITE(GEN6_PMIIR, pm_iir); done: I915_WRITE(DEIER, de_ier); POSTING_READ(DEIER); } /** * i915_error_work_func - do process context error handling work * @work: work struct * * Fire an error uevent so userspace can see that a hang or error * was detected. */ static void i915_error_work_func(void *context, int pending) { drm_i915_private_t *dev_priv = context; struct drm_device *dev = dev_priv->dev; /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event); */ if (atomic_load_acq_int(&dev_priv->mm.wedged)) { DRM_DEBUG("i915: resetting chip\n"); /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event); */ - if (!i915_reset(dev, GRDOM_RENDER)) { + if (!i915_reset(dev)) { atomic_store_rel_int(&dev_priv->mm.wedged, 0); /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event); */ } mtx_lock(&dev_priv->error_completion_lock); dev_priv->error_completion++; wakeup(&dev_priv->error_completion); mtx_unlock(&dev_priv->error_completion_lock); } } static void i915_report_and_clear_eir(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; u32 eir = I915_READ(EIR); int pipe; if (!eir) return; printf("i915: render error detected, EIR: 0x%08x\n", eir); if (IS_G4X(dev)) { if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) { u32 ipeir = I915_READ(IPEIR_I965); printf(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965)); printf(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965)); printf(" INSTDONE: 0x%08x\n", I915_READ(INSTDONE_I965)); printf(" INSTPS: 0x%08x\n", I915_READ(INSTPS)); printf(" INSTDONE1: 0x%08x\n", I915_READ(INSTDONE1)); printf(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965)); I915_WRITE(IPEIR_I965, ipeir); POSTING_READ(IPEIR_I965); } if (eir & GM45_ERROR_PAGE_TABLE) { u32 pgtbl_err = I915_READ(PGTBL_ER); printf("page table error\n"); printf(" PGTBL_ER: 0x%08x\n", pgtbl_err); I915_WRITE(PGTBL_ER, pgtbl_err); POSTING_READ(PGTBL_ER); } } if (!IS_GEN2(dev)) { if (eir & I915_ERROR_PAGE_TABLE) { u32 pgtbl_err = I915_READ(PGTBL_ER); printf("page table error\n"); printf(" PGTBL_ER: 0x%08x\n", pgtbl_err); I915_WRITE(PGTBL_ER, pgtbl_err); POSTING_READ(PGTBL_ER); } } if (eir & I915_ERROR_MEMORY_REFRESH) { printf("memory refresh error:\n"); for_each_pipe(pipe) printf("pipe %c stat: 0x%08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); /* pipestat has already been acked */ } if (eir & I915_ERROR_INSTRUCTION) { printf("instruction error\n"); printf(" INSTPM: 0x%08x\n", I915_READ(INSTPM)); if (INTEL_INFO(dev)->gen < 4) { u32 ipeir = I915_READ(IPEIR); printf(" IPEIR: 0x%08x\n", I915_READ(IPEIR)); printf(" IPEHR: 0x%08x\n", I915_READ(IPEHR)); printf(" INSTDONE: 0x%08x\n", I915_READ(INSTDONE)); printf(" ACTHD: 0x%08x\n", I915_READ(ACTHD)); I915_WRITE(IPEIR, ipeir); POSTING_READ(IPEIR); } else { u32 ipeir = I915_READ(IPEIR_I965); printf(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965)); printf(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965)); printf(" INSTDONE: 0x%08x\n", I915_READ(INSTDONE_I965)); printf(" INSTPS: 0x%08x\n", I915_READ(INSTPS)); printf(" INSTDONE1: 0x%08x\n", I915_READ(INSTDONE1)); printf(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965)); I915_WRITE(IPEIR_I965, ipeir); POSTING_READ(IPEIR_I965); } } I915_WRITE(EIR, eir); POSTING_READ(EIR); eir = I915_READ(EIR); if (eir) { /* * some errors might have become stuck, * mask them. */ DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir); I915_WRITE(EMR, I915_READ(EMR) | eir); I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); } } /** * i915_handle_error - handle an error interrupt * @dev: drm device * * Do some basic checking of regsiter state at error interrupt time and * dump it to the syslog. Also call i915_capture_error_state() to make * sure we get a record and make it available in debugfs. Fire a uevent * so userspace knows something bad happened (should trigger collection * of a ring dump etc.). */ void i915_handle_error(struct drm_device *dev, bool wedged) { struct drm_i915_private *dev_priv = dev->dev_private; i915_capture_error_state(dev); i915_report_and_clear_eir(dev); if (wedged) { mtx_lock(&dev_priv->error_completion_lock); dev_priv->error_completion = 0; dev_priv->mm.wedged = 1; /* unlock acts as rel barrier for store to wedged */ mtx_unlock(&dev_priv->error_completion_lock); /* * Wakeup waiting processes so they don't hang */ mtx_lock(&dev_priv->rings[RCS].irq_lock); wakeup(&dev_priv->rings[RCS]); mtx_unlock(&dev_priv->rings[RCS].irq_lock); if (HAS_BSD(dev)) { mtx_lock(&dev_priv->rings[VCS].irq_lock); wakeup(&dev_priv->rings[VCS]); mtx_unlock(&dev_priv->rings[VCS].irq_lock); } if (HAS_BLT(dev)) { mtx_lock(&dev_priv->rings[BCS].irq_lock); wakeup(&dev_priv->rings[BCS]); mtx_unlock(&dev_priv->rings[BCS].irq_lock); } } taskqueue_enqueue(dev_priv->tq, &dev_priv->error_task); } static void i915_pageflip_stall_check(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_i915_gem_object *obj; struct intel_unpin_work *work; bool stall_detected; /* Ignore early vblank irqs */ if (intel_crtc == NULL) return; mtx_lock(&dev->event_lock); work = intel_crtc->unpin_work; if (work == NULL || work->pending || !work->enable_stall_check) { /* Either the pending flip IRQ arrived, or we're too early. Don't check */ mtx_unlock(&dev->event_lock); return; } /* Potential stall - if we see that the flip has happened, assume a missed interrupt */ obj = work->pending_flip_obj; if (INTEL_INFO(dev)->gen >= 4) { int dspsurf = DSPSURF(intel_crtc->plane); stall_detected = I915_READ(dspsurf) == obj->gtt_offset; } else { int dspaddr = DSPADDR(intel_crtc->plane); stall_detected = I915_READ(dspaddr) == (obj->gtt_offset + crtc->y * crtc->fb->pitches[0] + crtc->x * crtc->fb->bits_per_pixel/8); } mtx_unlock(&dev->event_lock); if (stall_detected) { DRM_DEBUG("Pageflip stall detected\n"); intel_prepare_page_flip(dev, intel_crtc->plane); } } static void i915_driver_irq_handler(void *arg) { struct drm_device *dev = (struct drm_device *)arg; drm_i915_private_t *dev_priv = (drm_i915_private_t *)dev->dev_private; #if 0 struct drm_i915_master_private *master_priv; #endif u32 iir, new_iir; u32 pipe_stats[I915_MAX_PIPES]; u32 vblank_status; int vblank = 0; int irq_received; int pipe; bool blc_event = false; atomic_inc(&dev_priv->irq_received); iir = I915_READ(IIR); CTR1(KTR_DRM, "driver_irq_handler %x", iir); if (INTEL_INFO(dev)->gen >= 4) vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS; else vblank_status = PIPE_VBLANK_INTERRUPT_STATUS; for (;;) { irq_received = iir != 0; /* Can't rely on pipestat interrupt bit in iir as it might * have been cleared after the pipestat interrupt was received. * It doesn't set the bit in iir again, but it still produces * interrupts (for non-MSI). */ mtx_lock(&dev_priv->irq_lock); if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) i915_handle_error(dev, false); for_each_pipe(pipe) { int reg = PIPESTAT(pipe); pipe_stats[pipe] = I915_READ(reg); /* * Clear the PIPE*STAT regs before the IIR */ if (pipe_stats[pipe] & 0x8000ffff) { if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) DRM_DEBUG("pipe %c underrun\n", pipe_name(pipe)); I915_WRITE(reg, pipe_stats[pipe]); irq_received = 1; } } mtx_unlock(&dev_priv->irq_lock); if (!irq_received) break; /* Consume port. Then clear IIR or we'll miss events */ if ((I915_HAS_HOTPLUG(dev)) && (iir & I915_DISPLAY_PORT_INTERRUPT)) { u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); DRM_DEBUG("i915: hotplug event received, stat 0x%08x\n", hotplug_status); if (hotplug_status & dev_priv->hotplug_supported_mask) taskqueue_enqueue(dev_priv->tq, &dev_priv->hotplug_task); I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); I915_READ(PORT_HOTPLUG_STAT); } I915_WRITE(IIR, iir); new_iir = I915_READ(IIR); /* Flush posted writes */ #if 0 if (dev->primary->master) { master_priv = dev->primary->master->driver_priv; if (master_priv->sarea_priv) master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); } #else if (dev_priv->sarea_priv) dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); #endif if (iir & I915_USER_INTERRUPT) notify_ring(dev, &dev_priv->rings[RCS]); if (iir & I915_BSD_USER_INTERRUPT) notify_ring(dev, &dev_priv->rings[VCS]); if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) { intel_prepare_page_flip(dev, 0); if (dev_priv->flip_pending_is_done) intel_finish_page_flip_plane(dev, 0); } if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT) { intel_prepare_page_flip(dev, 1); if (dev_priv->flip_pending_is_done) intel_finish_page_flip_plane(dev, 1); } for_each_pipe(pipe) { if (pipe_stats[pipe] & vblank_status && drm_handle_vblank(dev, pipe)) { vblank++; if (!dev_priv->flip_pending_is_done) { i915_pageflip_stall_check(dev, pipe); intel_finish_page_flip(dev, pipe); } } if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) blc_event = true; } if (blc_event || (iir & I915_ASLE_INTERRUPT)) { intel_opregion_asle_intr(dev); } /* With MSI, interrupts are only generated when iir * transitions from zero to nonzero. If another bit got * set while we were handling the existing iir bits, then * we would never get another interrupt. * * This is fine on non-MSI as well, as if we hit this path * we avoid exiting the interrupt handler only to generate * another one. * * Note that for MSI this could cause a stray interrupt report * if an interrupt landed in the time between writing IIR and * the posting read. This should be rare enough to never * trigger the 99% of 100,000 interrupts test for disabling * stray interrupts. */ iir = new_iir; } } static int i915_emit_irq(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; #if 0 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; #endif i915_kernel_lost_context(dev); DRM_DEBUG("i915: emit_irq\n"); dev_priv->counter++; if (dev_priv->counter > 0x7FFFFFFFUL) dev_priv->counter = 1; #if 0 if (master_priv->sarea_priv) master_priv->sarea_priv->last_enqueue = dev_priv->counter; #else if (dev_priv->sarea_priv) dev_priv->sarea_priv->last_enqueue = dev_priv->counter; #endif if (BEGIN_LP_RING(4) == 0) { OUT_RING(MI_STORE_DWORD_INDEX); OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); OUT_RING(dev_priv->counter); OUT_RING(MI_USER_INTERRUPT); ADVANCE_LP_RING(); } return dev_priv->counter; } static int i915_wait_irq(struct drm_device * dev, int irq_nr) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; #if 0 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; #endif int ret; struct intel_ring_buffer *ring = LP_RING(dev_priv); DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr, READ_BREADCRUMB(dev_priv)); #if 0 if (READ_BREADCRUMB(dev_priv) >= irq_nr) { if (master_priv->sarea_priv) master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); return 0; } if (master_priv->sarea_priv) master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; #else if (READ_BREADCRUMB(dev_priv) >= irq_nr) { if (dev_priv->sarea_priv) { dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); } return 0; } if (dev_priv->sarea_priv) dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; #endif ret = 0; mtx_lock(&ring->irq_lock); if (ring->irq_get(ring)) { DRM_UNLOCK(dev); while (ret == 0 && READ_BREADCRUMB(dev_priv) < irq_nr) { ret = -msleep(ring, &ring->irq_lock, PCATCH, "915wtq", 3 * hz); } ring->irq_put(ring); mtx_unlock(&ring->irq_lock); DRM_LOCK(dev); } else { mtx_unlock(&ring->irq_lock); if (_intel_wait_for(dev, READ_BREADCRUMB(dev_priv) >= irq_nr, 3000, 1, "915wir")) ret = -EBUSY; } if (ret == -EBUSY) { DRM_ERROR("EBUSY -- rec: %d emitted: %d\n", READ_BREADCRUMB(dev_priv), (int)dev_priv->counter); } return ret; } /* Needs the lock as it touches the ring. */ int i915_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_irq_emit_t *emit = data; int result; if (!dev_priv || !LP_RING(dev_priv)->virtual_start) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } RING_LOCK_TEST_WITH_RETURN(dev, file_priv); DRM_LOCK(dev); result = i915_emit_irq(dev); DRM_UNLOCK(dev); if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) { DRM_ERROR("copy_to_user\n"); return -EFAULT; } return 0; } /* Doesn't need the hardware lock. */ int i915_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_irq_wait_t *irqwait = data; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } return i915_wait_irq(dev, irqwait->irq_seq); } /* Called from drm generic code, passed 'crtc' which * we use as a pipe index */ static int i915_enable_vblank(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; if (!i915_pipe_enabled(dev, pipe)) return -EINVAL; mtx_lock(&dev_priv->irq_lock); if (INTEL_INFO(dev)->gen >= 4) i915_enable_pipestat(dev_priv, pipe, PIPE_START_VBLANK_INTERRUPT_ENABLE); else i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_ENABLE); /* maintain vblank delivery even in deep C-states */ if (dev_priv->info->gen == 3) I915_WRITE(INSTPM, INSTPM_AGPBUSY_DIS << 16); mtx_unlock(&dev_priv->irq_lock); CTR1(KTR_DRM, "i915_enable_vblank %d", pipe); return 0; } static int ironlake_enable_vblank(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; if (!i915_pipe_enabled(dev, pipe)) return -EINVAL; mtx_lock(&dev_priv->irq_lock); ironlake_enable_display_irq(dev_priv, (pipe == 0) ? DE_PIPEA_VBLANK : DE_PIPEB_VBLANK); mtx_unlock(&dev_priv->irq_lock); CTR1(KTR_DRM, "ironlake_enable_vblank %d", pipe); return 0; } static int ivybridge_enable_vblank(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; if (!i915_pipe_enabled(dev, pipe)) return -EINVAL; mtx_lock(&dev_priv->irq_lock); ironlake_enable_display_irq(dev_priv, (pipe == 0) ? DE_PIPEA_VBLANK_IVB : DE_PIPEB_VBLANK_IVB); mtx_unlock(&dev_priv->irq_lock); CTR1(KTR_DRM, "ivybridge_enable_vblank %d", pipe); return 0; } /* Called from drm generic code, passed 'crtc' which * we use as a pipe index */ static void i915_disable_vblank(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; mtx_lock(&dev_priv->irq_lock); if (dev_priv->info->gen == 3) I915_WRITE(INSTPM, INSTPM_AGPBUSY_DIS << 16 | INSTPM_AGPBUSY_DIS); i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_ENABLE | PIPE_START_VBLANK_INTERRUPT_ENABLE); mtx_unlock(&dev_priv->irq_lock); CTR1(KTR_DRM, "i915_disable_vblank %d", pipe); } static void ironlake_disable_vblank(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; mtx_lock(&dev_priv->irq_lock); ironlake_disable_display_irq(dev_priv, (pipe == 0) ? DE_PIPEA_VBLANK : DE_PIPEB_VBLANK); mtx_unlock(&dev_priv->irq_lock); CTR1(KTR_DRM, "ironlake_disable_vblank %d", pipe); } static void ivybridge_disable_vblank(struct drm_device *dev, int pipe) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; mtx_lock(&dev_priv->irq_lock); ironlake_disable_display_irq(dev_priv, (pipe == 0) ? DE_PIPEA_VBLANK_IVB : DE_PIPEB_VBLANK_IVB); mtx_unlock(&dev_priv->irq_lock); CTR1(KTR_DRM, "ivybridge_disable_vblank %d", pipe); } /* Set the vblank monitor pipe */ int i915_vblank_pipe_set(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } return 0; } int i915_vblank_pipe_get(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_vblank_pipe_t *pipe = data; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; return 0; } /** * Schedule buffer swap at given vertical blank. */ int i915_vblank_swap(struct drm_device *dev, void *data, struct drm_file *file_priv) { /* The delayed swap mechanism was fundamentally racy, and has been * removed. The model was that the client requested a delayed flip/swap * from the kernel, then waited for vblank before continuing to perform * rendering. The problem was that the kernel might wake the client * up before it dispatched the vblank swap (since the lock has to be * held while touching the ringbuffer), in which case the client would * clear and start the next frame before the swap occurred, and * flicker would occur in addition to likely missing the vblank. * * In the absence of this ioctl, userland falls back to a correct path * of waiting for a vblank, then dispatching the swap on its own. * Context switching to userland and back is plenty fast enough for * meeting the requirements of vblank swapping. */ return -EINVAL; } static u32 ring_last_seqno(struct intel_ring_buffer *ring) { if (list_empty(&ring->request_list)) return (0); else return (list_entry(ring->request_list.prev, struct drm_i915_gem_request, list)->seqno); } static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err) { if (list_empty(&ring->request_list) || i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) { /* Issue a wake-up to catch stuck h/w. */ if (ring->waiting_seqno) { DRM_ERROR( "Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n", ring->name, ring->waiting_seqno, ring->get_seqno(ring)); wakeup(ring); *err = true; } return true; } return false; } static bool kick_ring(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 tmp = I915_READ_CTL(ring); if (tmp & RING_WAIT) { DRM_ERROR("Kicking stuck wait on %s\n", ring->name); I915_WRITE_CTL(ring, tmp); return true; } return false; } /** * This is called when the chip hasn't reported back with completed * batchbuffers in a long time. The first time this is called we simply record * ACTHD. If ACTHD hasn't changed by the time the hangcheck timer elapses * again, we assume the chip is wedged and try to fix it. */ void i915_hangcheck_elapsed(void *context) { struct drm_device *dev = (struct drm_device *)context; drm_i915_private_t *dev_priv = dev->dev_private; uint32_t acthd, instdone, instdone1, acthd_bsd, acthd_blt; bool err = false; if (!i915_enable_hangcheck) return; /* If all work is done then ACTHD clearly hasn't advanced. */ if (i915_hangcheck_ring_idle(&dev_priv->rings[RCS], &err) && i915_hangcheck_ring_idle(&dev_priv->rings[VCS], &err) && i915_hangcheck_ring_idle(&dev_priv->rings[BCS], &err)) { dev_priv->hangcheck_count = 0; if (err) goto repeat; return; } if (INTEL_INFO(dev)->gen < 4) { instdone = I915_READ(INSTDONE); instdone1 = 0; } else { instdone = I915_READ(INSTDONE_I965); instdone1 = I915_READ(INSTDONE1); } acthd = intel_ring_get_active_head(&dev_priv->rings[RCS]); acthd_bsd = HAS_BSD(dev) ? intel_ring_get_active_head(&dev_priv->rings[VCS]) : 0; acthd_blt = HAS_BLT(dev) ? intel_ring_get_active_head(&dev_priv->rings[BCS]) : 0; if (dev_priv->last_acthd == acthd && dev_priv->last_acthd_bsd == acthd_bsd && dev_priv->last_acthd_blt == acthd_blt && dev_priv->last_instdone == instdone && dev_priv->last_instdone1 == instdone1) { if (dev_priv->hangcheck_count++ > 1) { DRM_ERROR("Hangcheck timer elapsed... GPU hung\n"); i915_handle_error(dev, true); if (!IS_GEN2(dev)) { /* Is the chip hanging on a WAIT_FOR_EVENT? * If so we can simply poke the RB_WAIT bit * and break the hang. This should work on * all but the second generation chipsets. */ if (kick_ring(&dev_priv->rings[RCS])) goto repeat; if (HAS_BSD(dev) && kick_ring(&dev_priv->rings[VCS])) goto repeat; if (HAS_BLT(dev) && kick_ring(&dev_priv->rings[BCS])) goto repeat; } return; } } else { dev_priv->hangcheck_count = 0; dev_priv->last_acthd = acthd; dev_priv->last_acthd_bsd = acthd_bsd; dev_priv->last_acthd_blt = acthd_blt; dev_priv->last_instdone = instdone; dev_priv->last_instdone1 = instdone1; } repeat: /* Reset timer case chip hangs without another request being added */ callout_schedule(&dev_priv->hangcheck_timer, DRM_I915_HANGCHECK_PERIOD); } /* drm_dma.h hooks */ static void ironlake_irq_preinstall(struct drm_device *dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; atomic_set(&dev_priv->irq_received, 0); TASK_INIT(&dev_priv->hotplug_task, 0, i915_hotplug_work_func, dev->dev_private); TASK_INIT(&dev_priv->error_task, 0, i915_error_work_func, dev->dev_private); TASK_INIT(&dev_priv->rps_task, 0, gen6_pm_rps_work_func, dev->dev_private); I915_WRITE(HWSTAM, 0xeffe); /* XXX hotplug from PCH */ I915_WRITE(DEIMR, 0xffffffff); I915_WRITE(DEIER, 0x0); POSTING_READ(DEIER); /* and GT */ I915_WRITE(GTIMR, 0xffffffff); I915_WRITE(GTIER, 0x0); POSTING_READ(GTIER); /* south display irq */ I915_WRITE(SDEIMR, 0xffffffff); I915_WRITE(SDEIER, 0x0); POSTING_READ(SDEIER); } /* * Enable digital hotplug on the PCH, and configure the DP short pulse * duration to 2ms (which is the minimum in the Display Port spec) * * This register is the same on all known PCH chips. */ static void ironlake_enable_pch_hotplug(struct drm_device *dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; u32 hotplug; hotplug = I915_READ(PCH_PORT_HOTPLUG); hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK); hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms; hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms; hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms; I915_WRITE(PCH_PORT_HOTPLUG, hotplug); } static int ironlake_irq_postinstall(struct drm_device *dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; /* enable kind of interrupts always enabled */ u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE; u32 render_irqs; u32 hotplug_mask; dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; dev_priv->irq_mask = ~display_mask; /* should always can generate irq */ I915_WRITE(DEIIR, I915_READ(DEIIR)); I915_WRITE(DEIMR, dev_priv->irq_mask); I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK); POSTING_READ(DEIER); dev_priv->gt_irq_mask = ~0; I915_WRITE(GTIIR, I915_READ(GTIIR)); I915_WRITE(GTIMR, dev_priv->gt_irq_mask); if (IS_GEN6(dev)) render_irqs = GT_USER_INTERRUPT | GT_GEN6_BSD_USER_INTERRUPT | GT_BLT_USER_INTERRUPT; else render_irqs = GT_USER_INTERRUPT | GT_PIPE_NOTIFY | GT_BSD_USER_INTERRUPT; I915_WRITE(GTIER, render_irqs); POSTING_READ(GTIER); if (HAS_PCH_CPT(dev)) { hotplug_mask = (SDE_CRT_HOTPLUG_CPT | SDE_PORTB_HOTPLUG_CPT | SDE_PORTC_HOTPLUG_CPT | SDE_PORTD_HOTPLUG_CPT); } else { hotplug_mask = (SDE_CRT_HOTPLUG | SDE_PORTB_HOTPLUG | SDE_PORTC_HOTPLUG | SDE_PORTD_HOTPLUG | SDE_AUX_MASK); } dev_priv->pch_irq_mask = ~hotplug_mask; I915_WRITE(SDEIIR, I915_READ(SDEIIR)); I915_WRITE(SDEIMR, dev_priv->pch_irq_mask); I915_WRITE(SDEIER, hotplug_mask); POSTING_READ(SDEIER); ironlake_enable_pch_hotplug(dev); if (IS_IRONLAKE_M(dev)) { /* Clear & enable PCU event interrupts */ I915_WRITE(DEIIR, DE_PCU_EVENT); I915_WRITE(DEIER, I915_READ(DEIER) | DE_PCU_EVENT); ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT); } return 0; } static int ivybridge_irq_postinstall(struct drm_device *dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; /* enable kind of interrupts always enabled */ u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB | DE_PLANEA_FLIP_DONE_IVB | DE_PLANEB_FLIP_DONE_IVB; u32 render_irqs; u32 hotplug_mask; dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; dev_priv->irq_mask = ~display_mask; /* should always can generate irq */ I915_WRITE(DEIIR, I915_READ(DEIIR)); I915_WRITE(DEIMR, dev_priv->irq_mask); I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK_IVB | DE_PIPEB_VBLANK_IVB); POSTING_READ(DEIER); dev_priv->gt_irq_mask = ~0; I915_WRITE(GTIIR, I915_READ(GTIIR)); I915_WRITE(GTIMR, dev_priv->gt_irq_mask); render_irqs = GT_USER_INTERRUPT | GT_GEN6_BSD_USER_INTERRUPT | GT_BLT_USER_INTERRUPT; I915_WRITE(GTIER, render_irqs); POSTING_READ(GTIER); hotplug_mask = (SDE_CRT_HOTPLUG_CPT | SDE_PORTB_HOTPLUG_CPT | SDE_PORTC_HOTPLUG_CPT | SDE_PORTD_HOTPLUG_CPT); dev_priv->pch_irq_mask = ~hotplug_mask; I915_WRITE(SDEIIR, I915_READ(SDEIIR)); I915_WRITE(SDEIMR, dev_priv->pch_irq_mask); I915_WRITE(SDEIER, hotplug_mask); POSTING_READ(SDEIER); ironlake_enable_pch_hotplug(dev); return 0; } static void i915_driver_irq_preinstall(struct drm_device * dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; int pipe; atomic_set(&dev_priv->irq_received, 0); TASK_INIT(&dev_priv->hotplug_task, 0, i915_hotplug_work_func, dev->dev_private); TASK_INIT(&dev_priv->error_task, 0, i915_error_work_func, dev->dev_private); TASK_INIT(&dev_priv->rps_task, 0, gen6_pm_rps_work_func, dev->dev_private); if (I915_HAS_HOTPLUG(dev)) { I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); } I915_WRITE(HWSTAM, 0xeffe); for_each_pipe(pipe) I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE(IMR, 0xffffffff); I915_WRITE(IER, 0x0); POSTING_READ(IER); } /* * Must be called after intel_modeset_init or hotplug interrupts won't be * enabled correctly. */ static int i915_driver_irq_postinstall(struct drm_device *dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR; u32 error_mask; dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; /* Unmask the interrupts that we always want on. */ dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX; dev_priv->pipestat[0] = 0; dev_priv->pipestat[1] = 0; if (I915_HAS_HOTPLUG(dev)) { /* Enable in IER... */ enable_mask |= I915_DISPLAY_PORT_INTERRUPT; /* and unmask in IMR */ dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT; } /* * Enable some error detection, note the instruction error mask * bit is reserved, so we leave it masked. */ if (IS_G4X(dev)) { error_mask = ~(GM45_ERROR_PAGE_TABLE | GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV | I915_ERROR_MEMORY_REFRESH); } else { error_mask = ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH); } I915_WRITE(EMR, error_mask); I915_WRITE(IMR, dev_priv->irq_mask); I915_WRITE(IER, enable_mask); POSTING_READ(IER); if (I915_HAS_HOTPLUG(dev)) { u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN); /* Note HDMI and DP share bits */ if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS) hotplug_en |= HDMIB_HOTPLUG_INT_EN; if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS) hotplug_en |= HDMIC_HOTPLUG_INT_EN; if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS) hotplug_en |= HDMID_HOTPLUG_INT_EN; if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS) hotplug_en |= SDVOC_HOTPLUG_INT_EN; if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS) hotplug_en |= SDVOB_HOTPLUG_INT_EN; if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) { hotplug_en |= CRT_HOTPLUG_INT_EN; /* Programming the CRT detection parameters tends to generate a spurious hotplug event about three seconds later. So just do it once. */ if (IS_G4X(dev)) hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64; hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; } /* Ignore TV since it's buggy */ I915_WRITE(PORT_HOTPLUG_EN, hotplug_en); } intel_opregion_enable_asle(dev); return 0; } static void ironlake_irq_uninstall(struct drm_device *dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; if (dev_priv == NULL) return; dev_priv->vblank_pipe = 0; I915_WRITE(HWSTAM, 0xffffffff); I915_WRITE(DEIMR, 0xffffffff); I915_WRITE(DEIER, 0x0); I915_WRITE(DEIIR, I915_READ(DEIIR)); I915_WRITE(GTIMR, 0xffffffff); I915_WRITE(GTIER, 0x0); I915_WRITE(GTIIR, I915_READ(GTIIR)); I915_WRITE(SDEIMR, 0xffffffff); I915_WRITE(SDEIER, 0x0); I915_WRITE(SDEIIR, I915_READ(SDEIIR)); taskqueue_drain(dev_priv->tq, &dev_priv->hotplug_task); taskqueue_drain(dev_priv->tq, &dev_priv->error_task); taskqueue_drain(dev_priv->tq, &dev_priv->rps_task); } static void i915_driver_irq_uninstall(struct drm_device * dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; int pipe; if (!dev_priv) return; dev_priv->vblank_pipe = 0; if (I915_HAS_HOTPLUG(dev)) { I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); } I915_WRITE(HWSTAM, 0xffffffff); for_each_pipe(pipe) I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE(IMR, 0xffffffff); I915_WRITE(IER, 0x0); for_each_pipe(pipe) I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)) & 0x8000ffff); I915_WRITE(IIR, I915_READ(IIR)); taskqueue_drain(dev_priv->tq, &dev_priv->hotplug_task); taskqueue_drain(dev_priv->tq, &dev_priv->error_task); taskqueue_drain(dev_priv->tq, &dev_priv->rps_task); } void intel_irq_init(struct drm_device *dev) { dev->driver->get_vblank_counter = i915_get_vblank_counter; dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */ if (IS_G4X(dev) || IS_GEN5(dev) || IS_GEN6(dev) || IS_IVYBRIDGE(dev)) { dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */ dev->driver->get_vblank_counter = gm45_get_vblank_counter; } if (drm_core_check_feature(dev, DRIVER_MODESET)) dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp; else dev->driver->get_vblank_timestamp = NULL; dev->driver->get_scanout_position = i915_get_crtc_scanoutpos; if (IS_IVYBRIDGE(dev)) { /* Share pre & uninstall handlers with ILK/SNB */ dev->driver->irq_handler = ivybridge_irq_handler; dev->driver->irq_preinstall = ironlake_irq_preinstall; dev->driver->irq_postinstall = ivybridge_irq_postinstall; dev->driver->irq_uninstall = ironlake_irq_uninstall; dev->driver->enable_vblank = ivybridge_enable_vblank; dev->driver->disable_vblank = ivybridge_disable_vblank; } else if (HAS_PCH_SPLIT(dev)) { dev->driver->irq_handler = ironlake_irq_handler; dev->driver->irq_preinstall = ironlake_irq_preinstall; dev->driver->irq_postinstall = ironlake_irq_postinstall; dev->driver->irq_uninstall = ironlake_irq_uninstall; dev->driver->enable_vblank = ironlake_enable_vblank; dev->driver->disable_vblank = ironlake_disable_vblank; } else { dev->driver->irq_preinstall = i915_driver_irq_preinstall; dev->driver->irq_postinstall = i915_driver_irq_postinstall; dev->driver->irq_uninstall = i915_driver_irq_uninstall; dev->driver->irq_handler = i915_driver_irq_handler; dev->driver->enable_vblank = i915_enable_vblank; dev->driver->disable_vblank = i915_disable_vblank; } } static struct drm_i915_error_object * i915_error_object_create(struct drm_i915_private *dev_priv, struct drm_i915_gem_object *src) { struct drm_i915_error_object *dst; struct sf_buf *sf; void *d, *s; int page, page_count; u32 reloc_offset; if (src == NULL || src->pages == NULL) return NULL; page_count = src->base.size / PAGE_SIZE; dst = malloc(sizeof(*dst) + page_count * sizeof(u32 *), DRM_I915_GEM, M_NOWAIT); if (dst == NULL) return (NULL); reloc_offset = src->gtt_offset; for (page = 0; page < page_count; page++) { d = malloc(PAGE_SIZE, DRM_I915_GEM, M_NOWAIT); if (d == NULL) goto unwind; if (reloc_offset < dev_priv->mm.gtt_mappable_end) { /* Simply ignore tiling or any overlapping fence. * It's part of the error state, and this hopefully * captures what the GPU read. */ s = pmap_mapdev_attr(src->base.dev->agp->base + reloc_offset, PAGE_SIZE, PAT_WRITE_COMBINING); memcpy(d, s, PAGE_SIZE); pmap_unmapdev((vm_offset_t)s, PAGE_SIZE); } else { drm_clflush_pages(&src->pages[page], 1); sched_pin(); sf = sf_buf_alloc(src->pages[page], SFB_CPUPRIVATE | SFB_NOWAIT); if (sf != NULL) { s = (void *)(uintptr_t)sf_buf_kva(sf); memcpy(d, s, PAGE_SIZE); sf_buf_free(sf); } else { bzero(d, PAGE_SIZE); strcpy(d, "XXXKIB"); } sched_unpin(); drm_clflush_pages(&src->pages[page], 1); } dst->pages[page] = d; reloc_offset += PAGE_SIZE; } dst->page_count = page_count; dst->gtt_offset = src->gtt_offset; return (dst); unwind: while (page--) free(dst->pages[page], DRM_I915_GEM); free(dst, DRM_I915_GEM); return (NULL); } static void i915_error_object_free(struct drm_i915_error_object *obj) { int page; if (obj == NULL) return; for (page = 0; page < obj->page_count; page++) free(obj->pages[page], DRM_I915_GEM); free(obj, DRM_I915_GEM); } static void i915_error_state_free(struct drm_device *dev, struct drm_i915_error_state *error) { int i; for (i = 0; i < DRM_ARRAY_SIZE(error->ring); i++) { i915_error_object_free(error->ring[i].batchbuffer); i915_error_object_free(error->ring[i].ringbuffer); free(error->ring[i].requests, DRM_I915_GEM); } free(error->active_bo, DRM_I915_GEM); free(error->overlay, DRM_I915_GEM); free(error, DRM_I915_GEM); } static u32 capture_bo_list(struct drm_i915_error_buffer *err, int count, struct list_head *head) { struct drm_i915_gem_object *obj; int i = 0; list_for_each_entry(obj, head, mm_list) { err->size = obj->base.size; err->name = obj->base.name; err->seqno = obj->last_rendering_seqno; err->gtt_offset = obj->gtt_offset; err->read_domains = obj->base.read_domains; err->write_domain = obj->base.write_domain; err->fence_reg = obj->fence_reg; err->pinned = 0; if (obj->pin_count > 0) err->pinned = 1; if (obj->user_pin_count > 0) err->pinned = -1; err->tiling = obj->tiling_mode; err->dirty = obj->dirty; err->purgeable = obj->madv != I915_MADV_WILLNEED; err->ring = obj->ring ? obj->ring->id : -1; err->cache_level = obj->cache_level; if (++i == count) break; err++; } return (i); } static void i915_gem_record_fences(struct drm_device *dev, struct drm_i915_error_state *error) { struct drm_i915_private *dev_priv = dev->dev_private; int i; /* Fences */ switch (INTEL_INFO(dev)->gen) { case 7: case 6: for (i = 0; i < 16; i++) error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8)); break; case 5: case 4: for (i = 0; i < 16; i++) error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8)); break; case 3: if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) for (i = 0; i < 8; i++) error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4)); case 2: for (i = 0; i < 8; i++) error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4)); break; } } static struct drm_i915_error_object * i915_error_first_batchbuffer(struct drm_i915_private *dev_priv, struct intel_ring_buffer *ring) { struct drm_i915_gem_object *obj; u32 seqno; if (!ring->get_seqno) return (NULL); seqno = ring->get_seqno(ring); list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) { if (obj->ring != ring) continue; if (i915_seqno_passed(seqno, obj->last_rendering_seqno)) continue; if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0) continue; /* We need to copy these to an anonymous buffer as the simplest * method to avoid being overwritten by userspace. */ return (i915_error_object_create(dev_priv, obj)); } return NULL; } static void i915_record_ring_state(struct drm_device *dev, struct drm_i915_error_state *error, struct intel_ring_buffer *ring) { struct drm_i915_private *dev_priv = dev->dev_private; if (INTEL_INFO(dev)->gen >= 6) { error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base)); error->fault_reg[ring->id] = I915_READ(RING_FAULT_REG(ring)); error->semaphore_mboxes[ring->id][0] = I915_READ(RING_SYNC_0(ring->mmio_base)); error->semaphore_mboxes[ring->id][1] = I915_READ(RING_SYNC_1(ring->mmio_base)); } if (INTEL_INFO(dev)->gen >= 4) { error->ipeir[ring->id] = I915_READ(RING_IPEIR(ring->mmio_base)); error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base)); error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base)); error->instps[ring->id] = I915_READ(RING_INSTPS(ring->mmio_base)); if (ring->id == RCS) { error->instdone1 = I915_READ(INSTDONE1); error->bbaddr = I915_READ64(BB_ADDR); } } else { error->ipeir[ring->id] = I915_READ(IPEIR); error->ipehr[ring->id] = I915_READ(IPEHR); error->instdone[ring->id] = I915_READ(INSTDONE); } error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base)); error->seqno[ring->id] = ring->get_seqno(ring); error->acthd[ring->id] = intel_ring_get_active_head(ring); error->head[ring->id] = I915_READ_HEAD(ring); error->tail[ring->id] = I915_READ_TAIL(ring); error->cpu_ring_head[ring->id] = ring->head; error->cpu_ring_tail[ring->id] = ring->tail; } static void i915_gem_record_rings(struct drm_device *dev, struct drm_i915_error_state *error) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_request *request; int i, count; for (i = 0; i < I915_NUM_RINGS; i++) { struct intel_ring_buffer *ring = &dev_priv->rings[i]; if (ring->obj == NULL) continue; i915_record_ring_state(dev, error, ring); error->ring[i].batchbuffer = i915_error_first_batchbuffer(dev_priv, ring); error->ring[i].ringbuffer = i915_error_object_create(dev_priv, ring->obj); count = 0; list_for_each_entry(request, &ring->request_list, list) count++; error->ring[i].num_requests = count; error->ring[i].requests = malloc(count * sizeof(struct drm_i915_error_request), DRM_I915_GEM, M_WAITOK); if (error->ring[i].requests == NULL) { error->ring[i].num_requests = 0; continue; } count = 0; list_for_each_entry(request, &ring->request_list, list) { struct drm_i915_error_request *erq; erq = &error->ring[i].requests[count++]; erq->seqno = request->seqno; erq->jiffies = request->emitted_jiffies; erq->tail = request->tail; } } } static void i915_capture_error_state(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; struct drm_i915_error_state *error; int i, pipe; mtx_lock(&dev_priv->error_lock); error = dev_priv->first_error; mtx_unlock(&dev_priv->error_lock); if (error != NULL) return; /* Account for pipe specific data like PIPE*STAT */ error = malloc(sizeof(*error), DRM_I915_GEM, M_NOWAIT | M_ZERO); if (error == NULL) { DRM_DEBUG("out of memory, not capturing error state\n"); return; } DRM_INFO("capturing error event; look for more information in " "sysctl hw.dri.%d.info.i915_error_state\n", dev->sysctl_node_idx); error->eir = I915_READ(EIR); error->pgtbl_er = I915_READ(PGTBL_ER); for_each_pipe(pipe) error->pipestat[pipe] = I915_READ(PIPESTAT(pipe)); if (INTEL_INFO(dev)->gen >= 6) { error->error = I915_READ(ERROR_GEN6); error->done_reg = I915_READ(DONE_REG); } i915_gem_record_fences(dev, error); i915_gem_record_rings(dev, error); /* Record buffers on the active and pinned lists. */ error->active_bo = NULL; error->pinned_bo = NULL; i = 0; list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) i++; error->active_bo_count = i; list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) i++; error->pinned_bo_count = i - error->active_bo_count; error->active_bo = NULL; error->pinned_bo = NULL; if (i) { error->active_bo = malloc(sizeof(*error->active_bo) * i, DRM_I915_GEM, M_NOWAIT); if (error->active_bo) error->pinned_bo = error->active_bo + error->active_bo_count; } if (error->active_bo) error->active_bo_count = capture_bo_list(error->active_bo, error->active_bo_count, &dev_priv->mm.active_list); if (error->pinned_bo) error->pinned_bo_count = capture_bo_list(error->pinned_bo, error->pinned_bo_count, &dev_priv->mm.pinned_list); microtime(&error->time); error->overlay = intel_overlay_capture_error_state(dev); error->display = intel_display_capture_error_state(dev); mtx_lock(&dev_priv->error_lock); if (dev_priv->first_error == NULL) { dev_priv->first_error = error; error = NULL; } mtx_unlock(&dev_priv->error_lock); if (error != NULL) i915_error_state_free(dev, error); } void i915_destroy_error_state(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_error_state *error; mtx_lock(&dev_priv->error_lock); error = dev_priv->first_error; dev_priv->first_error = NULL; mtx_unlock(&dev_priv->error_lock); if (error != NULL) i915_error_state_free(dev, error); } Index: head/sys/dev/drm2/i915/i915_reg.h =================================================================== --- head/sys/dev/drm2/i915/i915_reg.h (revision 271704) +++ head/sys/dev/drm2/i915/i915_reg.h (revision 271705) @@ -1,3876 +1,3906 @@ /* Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #ifndef _I915_REG_H_ #define _I915_REG_H_ #define _PIPE(pipe, a, b) ((a) + (pipe)*((b)-(a))) /* * The Bridge device's PCI config space has information about the * fb aperture size and the amount of pre-reserved memory. * This is all handled in the intel-gtt.ko module. i915.ko only * cares about the vga bit for the vga rbiter. */ #define INTEL_GMCH_CTRL 0x52 #define INTEL_GMCH_VGA_DISABLE (1 << 1) /* PCI config space */ #define HPLLCC 0xc0 /* 855 only */ #define GC_CLOCK_CONTROL_MASK (0xf << 0) #define GC_CLOCK_133_200 (0 << 0) #define GC_CLOCK_100_200 (1 << 0) #define GC_CLOCK_100_133 (2 << 0) #define GC_CLOCK_166_250 (3 << 0) #define GCFGC2 0xda #define GCFGC 0xf0 /* 915+ only */ #define GC_LOW_FREQUENCY_ENABLE (1 << 7) #define GC_DISPLAY_CLOCK_190_200_MHZ (0 << 4) #define GC_DISPLAY_CLOCK_333_MHZ (4 << 4) #define GC_DISPLAY_CLOCK_MASK (7 << 4) #define GM45_GC_RENDER_CLOCK_MASK (0xf << 0) #define GM45_GC_RENDER_CLOCK_266_MHZ (8 << 0) #define GM45_GC_RENDER_CLOCK_320_MHZ (9 << 0) #define GM45_GC_RENDER_CLOCK_400_MHZ (0xb << 0) #define GM45_GC_RENDER_CLOCK_533_MHZ (0xc << 0) #define I965_GC_RENDER_CLOCK_MASK (0xf << 0) #define I965_GC_RENDER_CLOCK_267_MHZ (2 << 0) #define I965_GC_RENDER_CLOCK_333_MHZ (3 << 0) #define I965_GC_RENDER_CLOCK_444_MHZ (4 << 0) #define I965_GC_RENDER_CLOCK_533_MHZ (5 << 0) #define I945_GC_RENDER_CLOCK_MASK (7 << 0) #define I945_GC_RENDER_CLOCK_166_MHZ (0 << 0) #define I945_GC_RENDER_CLOCK_200_MHZ (1 << 0) #define I945_GC_RENDER_CLOCK_250_MHZ (3 << 0) #define I945_GC_RENDER_CLOCK_400_MHZ (5 << 0) #define I915_GC_RENDER_CLOCK_MASK (7 << 0) #define I915_GC_RENDER_CLOCK_166_MHZ (0 << 0) #define I915_GC_RENDER_CLOCK_200_MHZ (1 << 0) #define I915_GC_RENDER_CLOCK_333_MHZ (4 << 0) #define LBB 0xf4 /* Graphics reset regs */ #define I965_GDRST 0xc0 /* PCI config register */ #define ILK_GDSR 0x2ca4 /* MCHBAR offset */ #define GRDOM_FULL (0<<2) #define GRDOM_RENDER (1<<2) #define GRDOM_MEDIA (3<<2) +#define GRDOM_RESET_ENABLE (1<<0) #define GEN6_MBCUNIT_SNPCR 0x900c /* for LLC config */ #define GEN6_MBC_SNPCR_SHIFT 21 #define GEN6_MBC_SNPCR_MASK (3<<21) #define GEN6_MBC_SNPCR_MAX (0<<21) #define GEN6_MBC_SNPCR_MED (1<<21) #define GEN6_MBC_SNPCR_LOW (2<<21) #define GEN6_MBC_SNPCR_MIN (3<<21) /* only 1/16th of the cache is shared */ #define GEN6_MBCTL 0x0907c #define GEN6_MBCTL_ENABLE_BOOT_FETCH (1 << 4) #define GEN6_MBCTL_CTX_FETCH_NEEDED (1 << 3) #define GEN6_MBCTL_BME_UPDATE_ENABLE (1 << 2) #define GEN6_MBCTL_MAE_UPDATE_ENABLE (1 << 1) #define GEN6_MBCTL_BOOT_FETCH_MECH (1 << 0) #define GEN6_GDRST 0x941c #define GEN6_GRDOM_FULL (1 << 0) #define GEN6_GRDOM_RENDER (1 << 1) #define GEN6_GRDOM_MEDIA (1 << 2) #define GEN6_GRDOM_BLT (1 << 3) /* PPGTT stuff */ #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) #define GEN6_PDE_VALID (1 << 0) #define GEN6_PDE_LARGE_PAGE (2 << 0) /* use 32kb pages */ /* gen6+ has bit 11-4 for physical addr bit 39-32 */ #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) #define GEN6_PTE_VALID (1 << 0) #define GEN6_PTE_UNCACHED (1 << 1) #define GEN6_PTE_CACHE_LLC (2 << 1) #define GEN6_PTE_CACHE_LLC_MLC (3 << 1) #define GEN6_PTE_CACHE_BITS (3 << 1) #define GEN6_PTE_GFDT (1 << 3) #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) #define RING_PP_DIR_BASE(ring) ((ring)->mmio_base+0x228) #define RING_PP_DIR_BASE_READ(ring) ((ring)->mmio_base+0x518) #define RING_PP_DIR_DCLV(ring) ((ring)->mmio_base+0x220) #define PP_DIR_DCLV_2G 0xffffffff #define GAM_ECOCHK 0x4090 #define ECOCHK_SNB_BIT (1<<10) #define ECOCHK_PPGTT_CACHE64B (0x3<<3) #define ECOCHK_PPGTT_CACHE4B (0x0<<3) /* VGA stuff */ #define VGA_ST01_MDA 0x3ba #define VGA_ST01_CGA 0x3da #define VGA_MSR_WRITE 0x3c2 #define VGA_MSR_READ 0x3cc #define VGA_MSR_MEM_EN (1<<1) #define VGA_MSR_CGA_MODE (1<<0) #define VGA_SR_INDEX 0x3c4 #define VGA_SR_DATA 0x3c5 #define VGA_AR_INDEX 0x3c0 #define VGA_AR_VID_EN (1<<5) #define VGA_AR_DATA_WRITE 0x3c0 #define VGA_AR_DATA_READ 0x3c1 #define VGA_GR_INDEX 0x3ce #define VGA_GR_DATA 0x3cf /* GR05 */ #define VGA_GR_MEM_READ_MODE_SHIFT 3 #define VGA_GR_MEM_READ_MODE_PLANE 1 /* GR06 */ #define VGA_GR_MEM_MODE_MASK 0xc #define VGA_GR_MEM_MODE_SHIFT 2 #define VGA_GR_MEM_A0000_AFFFF 0 #define VGA_GR_MEM_A0000_BFFFF 1 #define VGA_GR_MEM_B0000_B7FFF 2 #define VGA_GR_MEM_B0000_BFFFF 3 #define VGA_DACMASK 0x3c6 #define VGA_DACRX 0x3c7 #define VGA_DACWX 0x3c8 #define VGA_DACDATA 0x3c9 #define VGA_CR_INDEX_MDA 0x3b4 #define VGA_CR_DATA_MDA 0x3b5 #define VGA_CR_INDEX_CGA 0x3d4 #define VGA_CR_DATA_CGA 0x3d5 /* * Memory interface instructions used by the kernel */ #define MI_INSTR(opcode, flags) (((opcode) << 23) | (flags)) #define MI_NOOP MI_INSTR(0, 0) #define MI_USER_INTERRUPT MI_INSTR(0x02, 0) #define MI_WAIT_FOR_EVENT MI_INSTR(0x03, 0) #define MI_WAIT_FOR_OVERLAY_FLIP (1<<16) #define MI_WAIT_FOR_PLANE_B_FLIP (1<<6) #define MI_WAIT_FOR_PLANE_A_FLIP (1<<2) #define MI_WAIT_FOR_PLANE_A_SCANLINES (1<<1) #define MI_FLUSH MI_INSTR(0x04, 0) #define MI_READ_FLUSH (1 << 0) #define MI_EXE_FLUSH (1 << 1) #define MI_NO_WRITE_FLUSH (1 << 2) #define MI_SCENE_COUNT (1 << 3) /* just increment scene count */ #define MI_END_SCENE (1 << 4) /* flush binner and incr scene count */ #define MI_INVALIDATE_ISP (1 << 5) /* invalidate indirect state pointers */ #define MI_BATCH_BUFFER_END MI_INSTR(0x0a, 0) #define MI_SUSPEND_FLUSH MI_INSTR(0x0b, 0) #define MI_SUSPEND_FLUSH_EN (1<<0) #define MI_REPORT_HEAD MI_INSTR(0x07, 0) #define MI_OVERLAY_FLIP MI_INSTR(0x11, 0) #define MI_OVERLAY_CONTINUE (0x0<<21) #define MI_OVERLAY_ON (0x1<<21) #define MI_OVERLAY_OFF (0x2<<21) #define MI_LOAD_SCAN_LINES_INCL MI_INSTR(0x12, 0) #define MI_DISPLAY_FLIP MI_INSTR(0x14, 2) #define MI_DISPLAY_FLIP_I915 MI_INSTR(0x14, 1) #define MI_DISPLAY_FLIP_PLANE(n) ((n) << 20) +#define MI_ARB_ON_OFF MI_INSTR(0x08, 0) +#define MI_ARB_ENABLE (1<<0) +#define MI_ARB_DISABLE (0<<0) + #define MI_SET_CONTEXT MI_INSTR(0x18, 0) #define MI_MM_SPACE_GTT (1<<8) #define MI_MM_SPACE_PHYSICAL (0<<8) #define MI_SAVE_EXT_STATE_EN (1<<3) #define MI_RESTORE_EXT_STATE_EN (1<<2) #define MI_FORCE_RESTORE (1<<1) #define MI_RESTORE_INHIBIT (1<<0) #define MI_STORE_DWORD_IMM MI_INSTR(0x20, 1) #define MI_MEM_VIRTUAL (1 << 22) /* 965+ only */ #define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1) #define MI_STORE_DWORD_INDEX_SHIFT 2 /* Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM: * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw * simply ignores the register load under certain conditions. * - One can actually load arbitrary many arbitrary registers: Simply issue x * address/value pairs. Don't overdue it, though, x <= 2^4 must hold! */ #define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*x-1) #define MI_FLUSH_DW MI_INSTR(0x26, 1) /* for GEN6 */ #define MI_INVALIDATE_TLB (1<<18) #define MI_INVALIDATE_BSD (1<<7) #define MI_BATCH_BUFFER MI_INSTR(0x30, 1) #define MI_BATCH_NON_SECURE (1) #define MI_BATCH_NON_SECURE_I965 (1<<8) #define MI_BATCH_BUFFER_START MI_INSTR(0x31, 0) #define MI_SEMAPHORE_MBOX MI_INSTR(0x16, 1) /* gen6+ */ #define MI_SEMAPHORE_GLOBAL_GTT (1<<22) #define MI_SEMAPHORE_UPDATE (1<<21) #define MI_SEMAPHORE_COMPARE (1<<20) #define MI_SEMAPHORE_REGISTER (1<<18) #define MI_SEMAPHORE_SYNC_RV (2<<16) #define MI_SEMAPHORE_SYNC_RB (0<<16) #define MI_SEMAPHORE_SYNC_VR (0<<16) #define MI_SEMAPHORE_SYNC_VB (2<<16) #define MI_SEMAPHORE_SYNC_BR (2<<16) #define MI_SEMAPHORE_SYNC_BV (0<<16) #define MI_SEMAPHORE_SYNC_INVALID (1<<0) /* * 3D instructions used by the kernel */ #define GFX_INSTR(opcode, flags) ((0x3 << 29) | ((opcode) << 24) | (flags)) #define GFX_OP_RASTER_RULES ((0x3<<29)|(0x7<<24)) #define GFX_OP_SCISSOR ((0x3<<29)|(0x1c<<24)|(0x10<<19)) #define SC_UPDATE_SCISSOR (0x1<<1) #define SC_ENABLE_MASK (0x1<<0) #define SC_ENABLE (0x1<<0) #define GFX_OP_LOAD_INDIRECT ((0x3<<29)|(0x1d<<24)|(0x7<<16)) #define GFX_OP_SCISSOR_INFO ((0x3<<29)|(0x1d<<24)|(0x81<<16)|(0x1)) #define SCI_YMIN_MASK (0xffff<<16) #define SCI_XMIN_MASK (0xffff<<0) #define SCI_YMAX_MASK (0xffff<<16) #define SCI_XMAX_MASK (0xffff<<0) #define GFX_OP_SCISSOR_ENABLE ((0x3<<29)|(0x1c<<24)|(0x10<<19)) #define GFX_OP_SCISSOR_RECT ((0x3<<29)|(0x1d<<24)|(0x81<<16)|1) #define GFX_OP_COLOR_FACTOR ((0x3<<29)|(0x1d<<24)|(0x1<<16)|0x0) #define GFX_OP_STIPPLE ((0x3<<29)|(0x1d<<24)|(0x83<<16)) #define GFX_OP_MAP_INFO ((0x3<<29)|(0x1d<<24)|0x4) #define GFX_OP_DESTBUFFER_VARS ((0x3<<29)|(0x1d<<24)|(0x85<<16)|0x0) #define GFX_OP_DESTBUFFER_INFO ((0x3<<29)|(0x1d<<24)|(0x8e<<16)|1) #define GFX_OP_DRAWRECT_INFO ((0x3<<29)|(0x1d<<24)|(0x80<<16)|(0x3)) #define GFX_OP_DRAWRECT_INFO_I965 ((0x7900<<16)|0x2) #define SRC_COPY_BLT_CMD ((2<<29)|(0x43<<22)|4) #define XY_SRC_COPY_BLT_CMD ((2<<29)|(0x53<<22)|6) #define XY_MONO_SRC_COPY_IMM_BLT ((2<<29)|(0x71<<22)|5) #define XY_SRC_COPY_BLT_WRITE_ALPHA (1<<21) #define XY_SRC_COPY_BLT_WRITE_RGB (1<<20) #define BLT_DEPTH_8 (0<<24) #define BLT_DEPTH_16_565 (1<<24) #define BLT_DEPTH_16_1555 (2<<24) #define BLT_DEPTH_32 (3<<24) #define BLT_ROP_GXCOPY (0xcc<<16) #define XY_SRC_COPY_BLT_SRC_TILED (1<<15) /* 965+ only */ #define XY_SRC_COPY_BLT_DST_TILED (1<<11) /* 965+ only */ #define CMD_OP_DISPLAYBUFFER_INFO ((0x0<<29)|(0x14<<23)|2) #define ASYNC_FLIP (1<<22) #define DISPLAY_PLANE_A (0<<20) #define DISPLAY_PLANE_B (1<<20) #define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|(len-2)) #define PIPE_CONTROL_CS_STALL (1<<20) #define PIPE_CONTROL_QW_WRITE (1<<14) #define PIPE_CONTROL_DEPTH_STALL (1<<13) #define PIPE_CONTROL_WRITE_FLUSH (1<<12) #define PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH (1<<12) /* gen6+ */ #define PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE (1<<11) /* MBZ on Ironlake */ #define PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE (1<<10) /* GM45+ only */ #define PIPE_CONTROL_INDIRECT_STATE_DISABLE (1<<9) #define PIPE_CONTROL_NOTIFY (1<<8) #define PIPE_CONTROL_VF_CACHE_INVALIDATE (1<<4) #define PIPE_CONTROL_CONST_CACHE_INVALIDATE (1<<3) #define PIPE_CONTROL_STATE_CACHE_INVALIDATE (1<<2) #define PIPE_CONTROL_STALL_AT_SCOREBOARD (1<<1) #define PIPE_CONTROL_DEPTH_CACHE_FLUSH (1<<0) #define PIPE_CONTROL_GLOBAL_GTT (1<<2) /* in addr dword */ /* * Reset registers */ #define DEBUG_RESET_I830 0x6070 #define DEBUG_RESET_FULL (1<<7) #define DEBUG_RESET_RENDER (1<<8) #define DEBUG_RESET_DISPLAY (1<<9) /* * Fence registers */ #define FENCE_REG_830_0 0x2000 #define FENCE_REG_945_8 0x3000 #define I830_FENCE_START_MASK 0x07f80000 #define I830_FENCE_TILING_Y_SHIFT 12 #define I830_FENCE_SIZE_BITS(size) ((ffs((size) >> 19) - 1) << 8) #define I830_FENCE_PITCH_SHIFT 4 #define I830_FENCE_REG_VALID (1<<0) #define I915_FENCE_MAX_PITCH_VAL 4 #define I830_FENCE_MAX_PITCH_VAL 6 #define I830_FENCE_MAX_SIZE_VAL (1<<8) #define I915_FENCE_START_MASK 0x0ff00000 #define I915_FENCE_SIZE_BITS(size) ((ffs((size) >> 20) - 1) << 8) #define FENCE_REG_965_0 0x03000 #define I965_FENCE_PITCH_SHIFT 2 #define I965_FENCE_TILING_Y_SHIFT 1 #define I965_FENCE_REG_VALID (1<<0) #define I965_FENCE_MAX_PITCH_VAL 0x0400 #define FENCE_REG_SANDYBRIDGE_0 0x100000 #define SANDYBRIDGE_FENCE_PITCH_SHIFT 32 /* control register for cpu gtt access */ #define TILECTL 0x101000 #define TILECTL_SWZCTL (1 << 0) #define TILECTL_TLB_PREFETCH_DIS (1 << 2) #define TILECTL_BACKSNOOP_DIS (1 << 3) /* * Instruction and interrupt control regs */ #define PGTBL_ER 0x02024 #define RENDER_RING_BASE 0x02000 #define BSD_RING_BASE 0x04000 #define GEN6_BSD_RING_BASE 0x12000 #define BLT_RING_BASE 0x22000 #define RING_TAIL(base) ((base)+0x30) #define RING_HEAD(base) ((base)+0x34) #define RING_START(base) ((base)+0x38) #define RING_CTL(base) ((base)+0x3c) #define RING_SYNC_0(base) ((base)+0x40) #define RING_SYNC_1(base) ((base)+0x44) #define GEN6_RVSYNC (RING_SYNC_0(RENDER_RING_BASE)) #define GEN6_RBSYNC (RING_SYNC_1(RENDER_RING_BASE)) #define GEN6_VRSYNC (RING_SYNC_1(GEN6_BSD_RING_BASE)) #define GEN6_VBSYNC (RING_SYNC_0(GEN6_BSD_RING_BASE)) #define GEN6_BRSYNC (RING_SYNC_0(BLT_RING_BASE)) #define GEN6_BVSYNC (RING_SYNC_1(BLT_RING_BASE)) #define RING_MAX_IDLE(base) ((base)+0x54) #define RING_HWS_PGA(base) ((base)+0x80) #define RING_HWS_PGA_GEN6(base) ((base)+0x2080) #define ARB_MODE 0x04030 #define ARB_MODE_SWIZZLE_SNB (1<<4) #define ARB_MODE_SWIZZLE_IVB (1<<5) #define ARB_MODE_ENABLE(x) GFX_MODE_ENABLE(x) #define ARB_MODE_DISABLE(x) GFX_MODE_DISABLE(x) #define RENDER_HWS_PGA_GEN7 (0x04080) #define RING_FAULT_REG(ring) (0x4094 + 0x100*(ring)->id) #define DONE_REG 0x40b0 #define BSD_HWS_PGA_GEN7 (0x04180) #define BLT_HWS_PGA_GEN7 (0x04280) #define RING_ACTHD(base) ((base)+0x74) #define RING_NOPID(base) ((base)+0x94) #define RING_IMR(base) ((base)+0xa8) #define TAIL_ADDR 0x001FFFF8 #define HEAD_WRAP_COUNT 0xFFE00000 #define HEAD_WRAP_ONE 0x00200000 #define HEAD_ADDR 0x001FFFFC #define RING_NR_PAGES 0x001FF000 #define RING_REPORT_MASK 0x00000006 #define RING_REPORT_64K 0x00000002 #define RING_REPORT_128K 0x00000004 #define RING_NO_REPORT 0x00000000 #define RING_VALID_MASK 0x00000001 #define RING_VALID 0x00000001 #define RING_INVALID 0x00000000 #define RING_WAIT_I8XX (1<<0) /* gen2, PRBx_HEAD */ #define RING_WAIT (1<<11) /* gen3+, PRBx_CTL */ #define RING_WAIT_SEMAPHORE (1<<10) /* gen6+ */ #if 0 #define PRB0_TAIL 0x02030 #define PRB0_HEAD 0x02034 #define PRB0_START 0x02038 #define PRB0_CTL 0x0203c #define PRB1_TAIL 0x02040 /* 915+ only */ #define PRB1_HEAD 0x02044 /* 915+ only */ #define PRB1_START 0x02048 /* 915+ only */ #define PRB1_CTL 0x0204c /* 915+ only */ #endif #define IPEIR_I965 0x02064 #define IPEHR_I965 0x02068 #define INSTDONE_I965 0x0206c #define RING_IPEIR(base) ((base)+0x64) #define RING_IPEHR(base) ((base)+0x68) #define RING_INSTDONE(base) ((base)+0x6c) #define RING_INSTPS(base) ((base)+0x70) #define RING_DMA_FADD(base) ((base)+0x78) #define RING_INSTPM(base) ((base)+0xc0) #define INSTPS 0x02070 /* 965+ only */ #define INSTDONE1 0x0207c /* 965+ only */ #define ACTHD_I965 0x02074 #define HWS_PGA 0x02080 #define HWS_ADDRESS_MASK 0xfffff000 #define HWS_START_ADDRESS_SHIFT 4 #define PWRCTXA 0x2088 /* 965GM+ only */ #define PWRCTX_EN (1<<0) #define IPEIR 0x02088 #define IPEHR 0x0208c #define INSTDONE 0x02090 #define NOPID 0x02094 #define HWSTAM 0x02098 #define ERROR_GEN6 0x040a0 /* GM45+ chicken bits -- debug workaround bits that may be required * for various sorts of correct behavior. The top 16 bits of each are * the enables for writing to the corresponding low bit. */ #define _3D_CHICKEN 0x02084 #define _3D_CHICKEN2 0x0208c /* Disables pipelining of read flushes past the SF-WIZ interface. * Required on all Ironlake steppings according to the B-Spec, but the * particular danger of not doing so is not specified. */ # define _3D_CHICKEN2_WM_READ_PIPELINED (1 << 14) #define _3D_CHICKEN3 0x02090 #define MI_MODE 0x0209c # define VS_TIMER_DISPATCH (1 << 6) # define MI_FLUSH_ENABLE (1 << 12) #define GFX_MODE 0x02520 #define GFX_MODE_GEN7 0x0229c #define RING_MODE_GEN7(ring) ((ring)->mmio_base+0x29c) #define GFX_RUN_LIST_ENABLE (1<<15) #define GFX_TLB_INVALIDATE_ALWAYS (1<<13) #define GFX_SURFACE_FAULT_ENABLE (1<<12) #define GFX_REPLAY_MODE (1<<11) #define GFX_PSMI_GRANULARITY (1<<10) #define GFX_PPGTT_ENABLE (1<<9) #define GFX_MODE_ENABLE(bit) (((bit) << 16) | (bit)) #define GFX_MODE_DISABLE(bit) (((bit) << 16) | (0)) #define SCPD0 0x0209c /* 915+ only */ #define IER 0x020a0 #define IIR 0x020a4 #define IMR 0x020a8 #define ISR 0x020ac #define I915_PIPE_CONTROL_NOTIFY_INTERRUPT (1<<18) #define I915_DISPLAY_PORT_INTERRUPT (1<<17) #define I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT (1<<15) #define I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT (1<<14) /* p-state */ #define I915_HWB_OOM_INTERRUPT (1<<13) #define I915_SYNC_STATUS_INTERRUPT (1<<12) #define I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT (1<<11) #define I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT (1<<10) #define I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT (1<<9) #define I915_DISPLAY_PLANE_C_FLIP_PENDING_INTERRUPT (1<<8) #define I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT (1<<7) #define I915_DISPLAY_PIPE_A_EVENT_INTERRUPT (1<<6) #define I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT (1<<5) #define I915_DISPLAY_PIPE_B_EVENT_INTERRUPT (1<<4) #define I915_DEBUG_INTERRUPT (1<<2) #define I915_USER_INTERRUPT (1<<1) #define I915_ASLE_INTERRUPT (1<<0) #define I915_BSD_USER_INTERRUPT (1<<25) #define EIR 0x020b0 #define EMR 0x020b4 #define ESR 0x020b8 #define GM45_ERROR_PAGE_TABLE (1<<5) #define GM45_ERROR_MEM_PRIV (1<<4) #define I915_ERROR_PAGE_TABLE (1<<4) #define GM45_ERROR_CP_PRIV (1<<3) #define I915_ERROR_MEMORY_REFRESH (1<<1) #define I915_ERROR_INSTRUCTION (1<<0) #define INSTPM 0x020c0 #define INSTPM_SELF_EN (1<<12) /* 915GM only */ #define INSTPM_AGPBUSY_DIS (1<<11) /* gen3: when disabled, pending interrupts will not assert AGPBUSY# and will only be delivered when out of C3. */ #define INSTPM_FORCE_ORDERING (1<<7) /* GEN6+ */ #define ACTHD 0x020c8 #define FW_BLC 0x020d8 #define FW_BLC2 0x020dc #define FW_BLC_SELF 0x020e0 /* 915+ only */ #define FW_BLC_SELF_EN_MASK (1<<31) #define FW_BLC_SELF_FIFO_MASK (1<<16) /* 945 only */ #define FW_BLC_SELF_EN (1<<15) /* 945 only */ #define MM_BURST_LENGTH 0x00700000 #define MM_FIFO_WATERMARK 0x0001F000 #define LM_BURST_LENGTH 0x00000700 #define LM_FIFO_WATERMARK 0x0000001F #define MI_ARB_STATE 0x020e4 /* 915+ only */ #define MI_ARB_MASK_SHIFT 16 /* shift for enable bits */ /* Make render/texture TLB fetches lower priorty than associated data * fetches. This is not turned on by default */ #define MI_ARB_RENDER_TLB_LOW_PRIORITY (1 << 15) /* Isoch request wait on GTT enable (Display A/B/C streams). * Make isoch requests stall on the TLB update. May cause * display underruns (test mode only) */ #define MI_ARB_ISOCH_WAIT_GTT (1 << 14) /* Block grant count for isoch requests when block count is * set to a finite value. */ #define MI_ARB_BLOCK_GRANT_MASK (3 << 12) #define MI_ARB_BLOCK_GRANT_8 (0 << 12) /* for 3 display planes */ #define MI_ARB_BLOCK_GRANT_4 (1 << 12) /* for 2 display planes */ #define MI_ARB_BLOCK_GRANT_2 (2 << 12) /* for 1 display plane */ #define MI_ARB_BLOCK_GRANT_0 (3 << 12) /* don't use */ /* Enable render writes to complete in C2/C3/C4 power states. * If this isn't enabled, render writes are prevented in low * power states. That seems bad to me. */ #define MI_ARB_C3_LP_WRITE_ENABLE (1 << 11) /* This acknowledges an async flip immediately instead * of waiting for 2TLB fetches. */ #define MI_ARB_ASYNC_FLIP_ACK_IMMEDIATE (1 << 10) /* Enables non-sequential data reads through arbiter */ #define MI_ARB_DUAL_DATA_PHASE_DISABLE (1 << 9) /* Disable FSB snooping of cacheable write cycles from binner/render * command stream */ #define MI_ARB_CACHE_SNOOP_DISABLE (1 << 8) /* Arbiter time slice for non-isoch streams */ #define MI_ARB_TIME_SLICE_MASK (7 << 5) #define MI_ARB_TIME_SLICE_1 (0 << 5) #define MI_ARB_TIME_SLICE_2 (1 << 5) #define MI_ARB_TIME_SLICE_4 (2 << 5) #define MI_ARB_TIME_SLICE_6 (3 << 5) #define MI_ARB_TIME_SLICE_8 (4 << 5) #define MI_ARB_TIME_SLICE_10 (5 << 5) #define MI_ARB_TIME_SLICE_14 (6 << 5) #define MI_ARB_TIME_SLICE_16 (7 << 5) /* Low priority grace period page size */ #define MI_ARB_LOW_PRIORITY_GRACE_4KB (0 << 4) /* default */ #define MI_ARB_LOW_PRIORITY_GRACE_8KB (1 << 4) /* Disable display A/B trickle feed */ #define MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE (1 << 2) /* Set display plane priority */ #define MI_ARB_DISPLAY_PRIORITY_A_B (0 << 0) /* display A > display B */ #define MI_ARB_DISPLAY_PRIORITY_B_A (1 << 0) /* display B > display A */ #define CACHE_MODE_0 0x02120 /* 915+ only */ #define CM0_MASK_SHIFT 16 #define CM0_IZ_OPT_DISABLE (1<<6) #define CM0_ZR_OPT_DISABLE (1<<5) #define CM0_STC_EVICT_DISABLE_LRA_SNB (1<<5) #define CM0_DEPTH_EVICT_DISABLE (1<<4) #define CM0_COLOR_EVICT_DISABLE (1<<3) #define CM0_DEPTH_WRITE_DISABLE (1<<1) #define CM0_RC_OP_FLUSH_DISABLE (1<<0) #define BB_ADDR 0x02140 /* 8 bytes */ #define GFX_FLSH_CNTL 0x02170 /* 915+ only */ #define ECOSKPD 0x021d0 #define ECO_GATING_CX_ONLY (1<<3) #define ECO_FLIP_DONE (1<<0) /* GEN6 interrupt control */ #define GEN6_RENDER_HWSTAM 0x2098 #define GEN6_RENDER_IMR 0x20a8 #define GEN6_RENDER_CONTEXT_SWITCH_INTERRUPT (1 << 8) #define GEN6_RENDER_PPGTT_PAGE_FAULT (1 << 7) #define GEN6_RENDER_TIMEOUT_COUNTER_EXPIRED (1 << 6) #define GEN6_RENDER_L3_PARITY_ERROR (1 << 5) #define GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT (1 << 4) #define GEN6_RENDER_COMMAND_PARSER_MASTER_ERROR (1 << 3) #define GEN6_RENDER_SYNC_STATUS (1 << 2) #define GEN6_RENDER_DEBUG_INTERRUPT (1 << 1) #define GEN6_RENDER_USER_INTERRUPT (1 << 0) #define GEN6_BLITTER_HWSTAM 0x22098 #define GEN6_BLITTER_IMR 0x220a8 #define GEN6_BLITTER_MI_FLUSH_DW_NOTIFY_INTERRUPT (1 << 26) #define GEN6_BLITTER_COMMAND_PARSER_MASTER_ERROR (1 << 25) #define GEN6_BLITTER_SYNC_STATUS (1 << 24) #define GEN6_BLITTER_USER_INTERRUPT (1 << 22) #define GEN6_BLITTER_ECOSKPD 0x221d0 #define GEN6_BLITTER_LOCK_SHIFT 16 #define GEN6_BLITTER_FBC_NOTIFY (1<<3) #define GEN6_BSD_SLEEP_PSMI_CONTROL 0x12050 #define GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK (1 << 16) #define GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE (1 << 0) #define GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE 0 #define GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR (1 << 3) #define GEN6_BSD_HWSTAM 0x12098 #define GEN6_BSD_IMR 0x120a8 #define GEN6_BSD_USER_INTERRUPT (1 << 12) #define GEN6_BSD_RNCID 0x12198 /* * Framebuffer compression (915+ only) */ #define FBC_CFB_BASE 0x03200 /* 4k page aligned */ #define FBC_LL_BASE 0x03204 /* 4k page aligned */ #define FBC_CONTROL 0x03208 #define FBC_CTL_EN (1<<31) #define FBC_CTL_PERIODIC (1<<30) #define FBC_CTL_INTERVAL_SHIFT (16) #define FBC_CTL_UNCOMPRESSIBLE (1<<14) #define FBC_CTL_C3_IDLE (1<<13) #define FBC_CTL_STRIDE_SHIFT (5) #define FBC_CTL_FENCENO (1<<0) #define FBC_COMMAND 0x0320c #define FBC_CMD_COMPRESS (1<<0) #define FBC_STATUS 0x03210 #define FBC_STAT_COMPRESSING (1<<31) #define FBC_STAT_COMPRESSED (1<<30) #define FBC_STAT_MODIFIED (1<<29) #define FBC_STAT_CURRENT_LINE (1<<0) #define FBC_CONTROL2 0x03214 #define FBC_CTL_FENCE_DBL (0<<4) #define FBC_CTL_IDLE_IMM (0<<2) #define FBC_CTL_IDLE_FULL (1<<2) #define FBC_CTL_IDLE_LINE (2<<2) #define FBC_CTL_IDLE_DEBUG (3<<2) #define FBC_CTL_CPU_FENCE (1<<1) #define FBC_CTL_PLANEA (0<<0) #define FBC_CTL_PLANEB (1<<0) #define FBC_FENCE_OFF 0x0321b #define FBC_TAG 0x03300 #define FBC_LL_SIZE (1536) /* Framebuffer compression for GM45+ */ #define DPFC_CB_BASE 0x3200 #define DPFC_CONTROL 0x3208 #define DPFC_CTL_EN (1<<31) #define DPFC_CTL_PLANEA (0<<30) #define DPFC_CTL_PLANEB (1<<30) #define DPFC_CTL_FENCE_EN (1<<29) #define DPFC_CTL_PERSISTENT_MODE (1<<25) #define DPFC_SR_EN (1<<10) #define DPFC_CTL_LIMIT_1X (0<<6) #define DPFC_CTL_LIMIT_2X (1<<6) #define DPFC_CTL_LIMIT_4X (2<<6) #define DPFC_RECOMP_CTL 0x320c #define DPFC_RECOMP_STALL_EN (1<<27) #define DPFC_RECOMP_STALL_WM_SHIFT (16) #define DPFC_RECOMP_STALL_WM_MASK (0x07ff0000) #define DPFC_RECOMP_TIMER_COUNT_SHIFT (0) #define DPFC_RECOMP_TIMER_COUNT_MASK (0x0000003f) #define DPFC_STATUS 0x3210 #define DPFC_INVAL_SEG_SHIFT (16) #define DPFC_INVAL_SEG_MASK (0x07ff0000) #define DPFC_COMP_SEG_SHIFT (0) #define DPFC_COMP_SEG_MASK (0x000003ff) #define DPFC_STATUS2 0x3214 #define DPFC_FENCE_YOFF 0x3218 #define DPFC_CHICKEN 0x3224 #define DPFC_HT_MODIFY (1<<31) /* Framebuffer compression for Ironlake */ #define ILK_DPFC_CB_BASE 0x43200 #define ILK_DPFC_CONTROL 0x43208 /* The bit 28-8 is reserved */ #define DPFC_RESERVED (0x1FFFFF00) #define ILK_DPFC_RECOMP_CTL 0x4320c #define ILK_DPFC_STATUS 0x43210 #define ILK_DPFC_FENCE_YOFF 0x43218 #define ILK_DPFC_CHICKEN 0x43224 #define ILK_FBC_RT_BASE 0x2128 #define ILK_FBC_RT_VALID (1<<0) #define ILK_DISPLAY_CHICKEN1 0x42000 #define ILK_FBCQ_DIS (1<<22) #define ILK_PABSTRETCH_DIS (1<<21) /* * Framebuffer compression for Sandybridge * * The following two registers are of type GTTMMADR */ #define SNB_DPFC_CTL_SA 0x100100 #define SNB_CPU_FENCE_ENABLE (1<<29) #define DPFC_CPU_FENCE_OFFSET 0x100104 /* * GPIO regs */ #define GPIOA 0x5010 #define GPIOB 0x5014 #define GPIOC 0x5018 #define GPIOD 0x501c #define GPIOE 0x5020 #define GPIOF 0x5024 #define GPIOG 0x5028 #define GPIOH 0x502c # define GPIO_CLOCK_DIR_MASK (1 << 0) # define GPIO_CLOCK_DIR_IN (0 << 1) # define GPIO_CLOCK_DIR_OUT (1 << 1) # define GPIO_CLOCK_VAL_MASK (1 << 2) # define GPIO_CLOCK_VAL_OUT (1 << 3) # define GPIO_CLOCK_VAL_IN (1 << 4) # define GPIO_CLOCK_PULLUP_DISABLE (1 << 5) # define GPIO_DATA_DIR_MASK (1 << 8) # define GPIO_DATA_DIR_IN (0 << 9) # define GPIO_DATA_DIR_OUT (1 << 9) # define GPIO_DATA_VAL_MASK (1 << 10) # define GPIO_DATA_VAL_OUT (1 << 11) # define GPIO_DATA_VAL_IN (1 << 12) # define GPIO_DATA_PULLUP_DISABLE (1 << 13) #define GMBUS0 0x5100 /* clock/port select */ #define GMBUS_RATE_100KHZ (0<<8) #define GMBUS_RATE_50KHZ (1<<8) #define GMBUS_RATE_400KHZ (2<<8) /* reserved on Pineview */ #define GMBUS_RATE_1MHZ (3<<8) /* reserved on Pineview */ #define GMBUS_HOLD_EXT (1<<7) /* 300ns hold time, rsvd on Pineview */ #define GMBUS_PORT_DISABLED 0 #define GMBUS_PORT_SSC 1 #define GMBUS_PORT_VGADDC 2 #define GMBUS_PORT_PANEL 3 #define GMBUS_PORT_DPC 4 /* HDMIC */ #define GMBUS_PORT_DPB 5 /* SDVO, HDMIB */ /* 6 reserved */ #define GMBUS_PORT_DPD 7 /* HDMID */ #define GMBUS_NUM_PORTS 8 #define GMBUS1 0x5104 /* command/status */ #define GMBUS_SW_CLR_INT (1<<31) #define GMBUS_SW_RDY (1<<30) #define GMBUS_ENT (1<<29) /* enable timeout */ #define GMBUS_CYCLE_NONE (0<<25) #define GMBUS_CYCLE_WAIT (1<<25) #define GMBUS_CYCLE_INDEX (2<<25) #define GMBUS_CYCLE_STOP (4<<25) #define GMBUS_BYTE_COUNT_SHIFT 16 #define GMBUS_SLAVE_INDEX_SHIFT 8 #define GMBUS_SLAVE_ADDR_SHIFT 1 #define GMBUS_SLAVE_READ (1<<0) #define GMBUS_SLAVE_WRITE (0<<0) #define GMBUS2 0x5108 /* status */ #define GMBUS_INUSE (1<<15) #define GMBUS_HW_WAIT_PHASE (1<<14) #define GMBUS_STALL_TIMEOUT (1<<13) #define GMBUS_INT (1<<12) #define GMBUS_HW_RDY (1<<11) #define GMBUS_SATOER (1<<10) #define GMBUS_ACTIVE (1<<9) #define GMBUS3 0x510c /* data buffer bytes 3-0 */ #define GMBUS4 0x5110 /* interrupt mask (Pineview+) */ #define GMBUS_SLAVE_TIMEOUT_EN (1<<4) #define GMBUS_NAK_EN (1<<3) #define GMBUS_IDLE_EN (1<<2) #define GMBUS_HW_WAIT_EN (1<<1) #define GMBUS_HW_RDY_EN (1<<0) #define GMBUS5 0x5120 /* byte index */ #define GMBUS_2BYTE_INDEX_EN (1<<31) /* * Clock control & power management */ #define VGA0 0x6000 #define VGA1 0x6004 #define VGA_PD 0x6010 #define VGA0_PD_P2_DIV_4 (1 << 7) #define VGA0_PD_P1_DIV_2 (1 << 5) #define VGA0_PD_P1_SHIFT 0 #define VGA0_PD_P1_MASK (0x1f << 0) #define VGA1_PD_P2_DIV_4 (1 << 15) #define VGA1_PD_P1_DIV_2 (1 << 13) #define VGA1_PD_P1_SHIFT 8 #define VGA1_PD_P1_MASK (0x1f << 8) #define _DPLL_A 0x06014 #define _DPLL_B 0x06018 #define DPLL(pipe) _PIPE(pipe, _DPLL_A, _DPLL_B) #define DPLL_VCO_ENABLE (1 << 31) #define DPLL_DVO_HIGH_SPEED (1 << 30) #define DPLL_SYNCLOCK_ENABLE (1 << 29) #define DPLL_VGA_MODE_DIS (1 << 28) #define DPLLB_MODE_DAC_SERIAL (1 << 26) /* i915 */ #define DPLLB_MODE_LVDS (2 << 26) /* i915 */ #define DPLL_MODE_MASK (3 << 26) #define DPLL_DAC_SERIAL_P2_CLOCK_DIV_10 (0 << 24) /* i915 */ #define DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 (1 << 24) /* i915 */ #define DPLLB_LVDS_P2_CLOCK_DIV_14 (0 << 24) /* i915 */ #define DPLLB_LVDS_P2_CLOCK_DIV_7 (1 << 24) /* i915 */ #define DPLL_P2_CLOCK_DIV_MASK 0x03000000 /* i915 */ #define DPLL_FPA01_P1_POST_DIV_MASK 0x00ff0000 /* i915 */ #define DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW 0x00ff8000 /* Pineview */ #define SRX_INDEX 0x3c4 #define SRX_DATA 0x3c5 #define SR01 1 #define SR01_SCREEN_OFF (1<<5) #define PPCR 0x61204 #define PPCR_ON (1<<0) #define DVOB 0x61140 #define DVOB_ON (1<<31) #define DVOC 0x61160 #define DVOC_ON (1<<31) #define LVDS 0x61180 #define LVDS_ON (1<<31) /* Scratch pad debug 0 reg: */ #define DPLL_FPA01_P1_POST_DIV_MASK_I830 0x001f0000 /* * The i830 generation, in LVDS mode, defines P1 as the bit number set within * this field (only one bit may be set). */ #define DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS 0x003f0000 #define DPLL_FPA01_P1_POST_DIV_SHIFT 16 #define DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW 15 /* i830, required in DVO non-gang */ #define PLL_P2_DIVIDE_BY_4 (1 << 23) #define PLL_P1_DIVIDE_BY_TWO (1 << 21) /* i830 */ #define PLL_REF_INPUT_DREFCLK (0 << 13) #define PLL_REF_INPUT_TVCLKINA (1 << 13) /* i830 */ #define PLL_REF_INPUT_TVCLKINBC (2 << 13) /* SDVO TVCLKIN */ #define PLLB_REF_INPUT_SPREADSPECTRUMIN (3 << 13) #define PLL_REF_INPUT_MASK (3 << 13) #define PLL_LOAD_PULSE_PHASE_SHIFT 9 /* Ironlake */ # define PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT 9 # define PLL_REF_SDVO_HDMI_MULTIPLIER_MASK (7 << 9) # define PLL_REF_SDVO_HDMI_MULTIPLIER(x) (((x)-1) << 9) # define DPLL_FPA1_P1_POST_DIV_SHIFT 0 # define DPLL_FPA1_P1_POST_DIV_MASK 0xff /* * Parallel to Serial Load Pulse phase selection. * Selects the phase for the 10X DPLL clock for the PCIe * digital display port. The range is 4 to 13; 10 or more * is just a flip delay. The default is 6 */ #define PLL_LOAD_PULSE_PHASE_MASK (0xf << PLL_LOAD_PULSE_PHASE_SHIFT) #define DISPLAY_RATE_SELECT_FPA1 (1 << 8) /* * SDVO multiplier for 945G/GM. Not used on 965. */ #define SDVO_MULTIPLIER_MASK 0x000000ff #define SDVO_MULTIPLIER_SHIFT_HIRES 4 #define SDVO_MULTIPLIER_SHIFT_VGA 0 #define _DPLL_A_MD 0x0601c /* 965+ only */ /* * UDI pixel divider, controlling how many pixels are stuffed into a packet. * * Value is pixels minus 1. Must be set to 1 pixel for SDVO. */ #define DPLL_MD_UDI_DIVIDER_MASK 0x3f000000 #define DPLL_MD_UDI_DIVIDER_SHIFT 24 /* UDI pixel divider for VGA, same as DPLL_MD_UDI_DIVIDER_MASK. */ #define DPLL_MD_VGA_UDI_DIVIDER_MASK 0x003f0000 #define DPLL_MD_VGA_UDI_DIVIDER_SHIFT 16 /* * SDVO/UDI pixel multiplier. * * SDVO requires that the bus clock rate be between 1 and 2 Ghz, and the bus * clock rate is 10 times the DPLL clock. At low resolution/refresh rate * modes, the bus rate would be below the limits, so SDVO allows for stuffing * dummy bytes in the datastream at an increased clock rate, with both sides of * the link knowing how many bytes are fill. * * So, for a mode with a dotclock of 65Mhz, we would want to double the clock * rate to 130Mhz to get a bus rate of 1.30Ghz. The DPLL clock rate would be * set to 130Mhz, and the SDVO multiplier set to 2x in this register and * through an SDVO command. * * This register field has values of multiplication factor minus 1, with * a maximum multiplier of 5 for SDVO. */ #define DPLL_MD_UDI_MULTIPLIER_MASK 0x00003f00 #define DPLL_MD_UDI_MULTIPLIER_SHIFT 8 /* * SDVO/UDI pixel multiplier for VGA, same as DPLL_MD_UDI_MULTIPLIER_MASK. * This best be set to the default value (3) or the CRT won't work. No, * I don't entirely understand what this does... */ #define DPLL_MD_VGA_UDI_MULTIPLIER_MASK 0x0000003f #define DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT 0 #define _DPLL_B_MD 0x06020 /* 965+ only */ #define DPLL_MD(pipe) _PIPE(pipe, _DPLL_A_MD, _DPLL_B_MD) #define _FPA0 0x06040 #define _FPA1 0x06044 #define _FPB0 0x06048 #define _FPB1 0x0604c #define FP0(pipe) _PIPE(pipe, _FPA0, _FPB0) #define FP1(pipe) _PIPE(pipe, _FPA1, _FPB1) #define FP_N_DIV_MASK 0x003f0000 #define FP_N_PINEVIEW_DIV_MASK 0x00ff0000 #define FP_N_DIV_SHIFT 16 #define FP_M1_DIV_MASK 0x00003f00 #define FP_M1_DIV_SHIFT 8 #define FP_M2_DIV_MASK 0x0000003f #define FP_M2_PINEVIEW_DIV_MASK 0x000000ff #define FP_M2_DIV_SHIFT 0 #define DPLL_TEST 0x606c #define DPLLB_TEST_SDVO_DIV_1 (0 << 22) #define DPLLB_TEST_SDVO_DIV_2 (1 << 22) #define DPLLB_TEST_SDVO_DIV_4 (2 << 22) #define DPLLB_TEST_SDVO_DIV_MASK (3 << 22) #define DPLLB_TEST_N_BYPASS (1 << 19) #define DPLLB_TEST_M_BYPASS (1 << 18) #define DPLLB_INPUT_BUFFER_ENABLE (1 << 16) #define DPLLA_TEST_N_BYPASS (1 << 3) #define DPLLA_TEST_M_BYPASS (1 << 2) #define DPLLA_INPUT_BUFFER_ENABLE (1 << 0) #define D_STATE 0x6104 #define DSTATE_GFX_RESET_I830 (1<<6) #define DSTATE_PLL_D3_OFF (1<<3) #define DSTATE_GFX_CLOCK_GATING (1<<1) #define DSTATE_DOT_CLOCK_GATING (1<<0) #define DSPCLK_GATE_D 0x6200 # define DPUNIT_B_CLOCK_GATE_DISABLE (1 << 30) /* 965 */ # define VSUNIT_CLOCK_GATE_DISABLE (1 << 29) /* 965 */ # define VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) /* 965 */ # define VRDUNIT_CLOCK_GATE_DISABLE (1 << 27) /* 965 */ # define AUDUNIT_CLOCK_GATE_DISABLE (1 << 26) /* 965 */ # define DPUNIT_A_CLOCK_GATE_DISABLE (1 << 25) /* 965 */ # define DPCUNIT_CLOCK_GATE_DISABLE (1 << 24) /* 965 */ # define TVRUNIT_CLOCK_GATE_DISABLE (1 << 23) /* 915-945 */ # define TVCUNIT_CLOCK_GATE_DISABLE (1 << 22) /* 915-945 */ # define TVFUNIT_CLOCK_GATE_DISABLE (1 << 21) /* 915-945 */ # define TVEUNIT_CLOCK_GATE_DISABLE (1 << 20) /* 915-945 */ # define DVSUNIT_CLOCK_GATE_DISABLE (1 << 19) /* 915-945 */ # define DSSUNIT_CLOCK_GATE_DISABLE (1 << 18) /* 915-945 */ # define DDBUNIT_CLOCK_GATE_DISABLE (1 << 17) /* 915-945 */ # define DPRUNIT_CLOCK_GATE_DISABLE (1 << 16) /* 915-945 */ # define DPFUNIT_CLOCK_GATE_DISABLE (1 << 15) /* 915-945 */ # define DPBMUNIT_CLOCK_GATE_DISABLE (1 << 14) /* 915-945 */ # define DPLSUNIT_CLOCK_GATE_DISABLE (1 << 13) /* 915-945 */ # define DPLUNIT_CLOCK_GATE_DISABLE (1 << 12) /* 915-945 */ # define DPOUNIT_CLOCK_GATE_DISABLE (1 << 11) # define DPBUNIT_CLOCK_GATE_DISABLE (1 << 10) # define DCUNIT_CLOCK_GATE_DISABLE (1 << 9) # define DPUNIT_CLOCK_GATE_DISABLE (1 << 8) # define VRUNIT_CLOCK_GATE_DISABLE (1 << 7) /* 915+: reserved */ # define OVHUNIT_CLOCK_GATE_DISABLE (1 << 6) /* 830-865 */ # define DPIOUNIT_CLOCK_GATE_DISABLE (1 << 6) /* 915-945 */ # define OVFUNIT_CLOCK_GATE_DISABLE (1 << 5) # define OVBUNIT_CLOCK_GATE_DISABLE (1 << 4) /** * This bit must be set on the 830 to prevent hangs when turning off the * overlay scaler. */ # define OVRUNIT_CLOCK_GATE_DISABLE (1 << 3) # define OVCUNIT_CLOCK_GATE_DISABLE (1 << 2) # define OVUUNIT_CLOCK_GATE_DISABLE (1 << 1) # define ZVUNIT_CLOCK_GATE_DISABLE (1 << 0) /* 830 */ # define OVLUNIT_CLOCK_GATE_DISABLE (1 << 0) /* 845,865 */ #define RENCLK_GATE_D1 0x6204 # define BLITTER_CLOCK_GATE_DISABLE (1 << 13) /* 945GM only */ # define MPEG_CLOCK_GATE_DISABLE (1 << 12) /* 945GM only */ # define PC_FE_CLOCK_GATE_DISABLE (1 << 11) # define PC_BE_CLOCK_GATE_DISABLE (1 << 10) # define WINDOWER_CLOCK_GATE_DISABLE (1 << 9) # define INTERPOLATOR_CLOCK_GATE_DISABLE (1 << 8) # define COLOR_CALCULATOR_CLOCK_GATE_DISABLE (1 << 7) # define MOTION_COMP_CLOCK_GATE_DISABLE (1 << 6) # define MAG_CLOCK_GATE_DISABLE (1 << 5) /** This bit must be unset on 855,865 */ # define MECI_CLOCK_GATE_DISABLE (1 << 4) # define DCMP_CLOCK_GATE_DISABLE (1 << 3) # define MEC_CLOCK_GATE_DISABLE (1 << 2) # define MECO_CLOCK_GATE_DISABLE (1 << 1) /** This bit must be set on 855,865. */ # define SV_CLOCK_GATE_DISABLE (1 << 0) # define I915_MPEG_CLOCK_GATE_DISABLE (1 << 16) # define I915_VLD_IP_PR_CLOCK_GATE_DISABLE (1 << 15) # define I915_MOTION_COMP_CLOCK_GATE_DISABLE (1 << 14) # define I915_BD_BF_CLOCK_GATE_DISABLE (1 << 13) # define I915_SF_SE_CLOCK_GATE_DISABLE (1 << 12) # define I915_WM_CLOCK_GATE_DISABLE (1 << 11) # define I915_IZ_CLOCK_GATE_DISABLE (1 << 10) # define I915_PI_CLOCK_GATE_DISABLE (1 << 9) # define I915_DI_CLOCK_GATE_DISABLE (1 << 8) # define I915_SH_SV_CLOCK_GATE_DISABLE (1 << 7) # define I915_PL_DG_QC_FT_CLOCK_GATE_DISABLE (1 << 6) # define I915_SC_CLOCK_GATE_DISABLE (1 << 5) # define I915_FL_CLOCK_GATE_DISABLE (1 << 4) # define I915_DM_CLOCK_GATE_DISABLE (1 << 3) # define I915_PS_CLOCK_GATE_DISABLE (1 << 2) # define I915_CC_CLOCK_GATE_DISABLE (1 << 1) # define I915_BY_CLOCK_GATE_DISABLE (1 << 0) # define I965_RCZ_CLOCK_GATE_DISABLE (1 << 30) /** This bit must always be set on 965G/965GM */ # define I965_RCC_CLOCK_GATE_DISABLE (1 << 29) # define I965_RCPB_CLOCK_GATE_DISABLE (1 << 28) # define I965_DAP_CLOCK_GATE_DISABLE (1 << 27) # define I965_ROC_CLOCK_GATE_DISABLE (1 << 26) # define I965_GW_CLOCK_GATE_DISABLE (1 << 25) # define I965_TD_CLOCK_GATE_DISABLE (1 << 24) /** This bit must always be set on 965G */ # define I965_ISC_CLOCK_GATE_DISABLE (1 << 23) # define I965_IC_CLOCK_GATE_DISABLE (1 << 22) # define I965_EU_CLOCK_GATE_DISABLE (1 << 21) # define I965_IF_CLOCK_GATE_DISABLE (1 << 20) # define I965_TC_CLOCK_GATE_DISABLE (1 << 19) # define I965_SO_CLOCK_GATE_DISABLE (1 << 17) # define I965_FBC_CLOCK_GATE_DISABLE (1 << 16) # define I965_MARI_CLOCK_GATE_DISABLE (1 << 15) # define I965_MASF_CLOCK_GATE_DISABLE (1 << 14) # define I965_MAWB_CLOCK_GATE_DISABLE (1 << 13) # define I965_EM_CLOCK_GATE_DISABLE (1 << 12) # define I965_UC_CLOCK_GATE_DISABLE (1 << 11) # define I965_SI_CLOCK_GATE_DISABLE (1 << 6) # define I965_MT_CLOCK_GATE_DISABLE (1 << 5) # define I965_PL_CLOCK_GATE_DISABLE (1 << 4) # define I965_DG_CLOCK_GATE_DISABLE (1 << 3) # define I965_QC_CLOCK_GATE_DISABLE (1 << 2) # define I965_FT_CLOCK_GATE_DISABLE (1 << 1) # define I965_DM_CLOCK_GATE_DISABLE (1 << 0) #define RENCLK_GATE_D2 0x6208 #define VF_UNIT_CLOCK_GATE_DISABLE (1 << 9) #define GS_UNIT_CLOCK_GATE_DISABLE (1 << 7) #define CL_UNIT_CLOCK_GATE_DISABLE (1 << 6) #define RAMCLK_GATE_D 0x6210 /* CRL only */ #define DEUC 0x6214 /* CRL only */ /* * Palette regs */ #define _PALETTE_A 0x0a000 #define _PALETTE_B 0x0a800 #define PALETTE(pipe) _PIPE(pipe, _PALETTE_A, _PALETTE_B) /* MCH MMIO space */ /* * MCHBAR mirror. * * This mirrors the MCHBAR MMIO space whose location is determined by * device 0 function 0's pci config register 0x44 or 0x48 and matches it in * every way. It is not accessible from the CP register read instructions. * */ #define MCHBAR_MIRROR_BASE 0x10000 #define MCHBAR_MIRROR_BASE_SNB 0x140000 /** 915-945 and GM965 MCH register controlling DRAM channel access */ #define DCC 0x10200 #define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0) #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC (1 << 0) #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0) #define DCC_ADDRESSING_MODE_MASK (3 << 0) #define DCC_CHANNEL_XOR_DISABLE (1 << 10) #define DCC_CHANNEL_XOR_BIT_17 (1 << 9) /** Pineview MCH register contains DDR3 setting */ #define CSHRDDR3CTL 0x101a8 #define CSHRDDR3CTL_DDR3 (1 << 2) /** 965 MCH register controlling DRAM channel configuration */ #define C0DRB3 0x10206 #define C1DRB3 0x10606 /** snb MCH registers for reading the DRAM channel configuration */ #define MAD_DIMM_C0 (MCHBAR_MIRROR_BASE_SNB + 0x5004) #define MAD_DIMM_C1 (MCHBAR_MIRROR_BASE_SNB + 0x5008) #define MAD_DIMM_C2 (MCHBAR_MIRROR_BASE_SNB + 0x500C) #define MAD_DIMM_ECC_MASK (0x3 << 24) #define MAD_DIMM_ECC_OFF (0x0 << 24) #define MAD_DIMM_ECC_IO_ON_LOGIC_OFF (0x1 << 24) #define MAD_DIMM_ECC_IO_OFF_LOGIC_ON (0x2 << 24) #define MAD_DIMM_ECC_ON (0x3 << 24) #define MAD_DIMM_ENH_INTERLEAVE (0x1 << 22) #define MAD_DIMM_RANK_INTERLEAVE (0x1 << 21) #define MAD_DIMM_B_WIDTH_X16 (0x1 << 20) /* X8 chips if unset */ #define MAD_DIMM_A_WIDTH_X16 (0x1 << 19) /* X8 chips if unset */ #define MAD_DIMM_B_DUAL_RANK (0x1 << 18) #define MAD_DIMM_A_DUAL_RANK (0x1 << 17) #define MAD_DIMM_A_SELECT (0x1 << 16) /* DIMM sizes are in multiples of 256mb. */ #define MAD_DIMM_B_SIZE_SHIFT 8 #define MAD_DIMM_B_SIZE_MASK (0xff << MAD_DIMM_B_SIZE_SHIFT) #define MAD_DIMM_A_SIZE_SHIFT 0 #define MAD_DIMM_A_SIZE_MASK (0xff << MAD_DIMM_A_SIZE_SHIFT) /* Clocking configuration register */ #define CLKCFG 0x10c00 #define CLKCFG_FSB_400 (5 << 0) /* hrawclk 100 */ #define CLKCFG_FSB_533 (1 << 0) /* hrawclk 133 */ #define CLKCFG_FSB_667 (3 << 0) /* hrawclk 166 */ #define CLKCFG_FSB_800 (2 << 0) /* hrawclk 200 */ #define CLKCFG_FSB_1067 (6 << 0) /* hrawclk 266 */ #define CLKCFG_FSB_1333 (7 << 0) /* hrawclk 333 */ /* Note, below two are guess */ #define CLKCFG_FSB_1600 (4 << 0) /* hrawclk 400 */ #define CLKCFG_FSB_1600_ALT (0 << 0) /* hrawclk 400 */ #define CLKCFG_FSB_MASK (7 << 0) #define CLKCFG_MEM_533 (1 << 4) #define CLKCFG_MEM_667 (2 << 4) #define CLKCFG_MEM_800 (3 << 4) #define CLKCFG_MEM_MASK (7 << 4) #define TSC1 0x11001 #define TSE (1<<0) #define I915_TR1 0x11006 #define TSFS 0x11020 #define TSFS_SLOPE_MASK 0x0000ff00 #define TSFS_SLOPE_SHIFT 8 #define TSFS_INTR_MASK 0x000000ff #define CRSTANDVID 0x11100 #define PXVFREQ_BASE 0x11110 /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */ #define PXVFREQ_PX_MASK 0x7f000000 #define PXVFREQ_PX_SHIFT 24 #define VIDFREQ_BASE 0x11110 #define VIDFREQ1 0x11110 /* VIDFREQ1-4 (0x1111c) (Cantiga) */ #define VIDFREQ2 0x11114 #define VIDFREQ3 0x11118 #define VIDFREQ4 0x1111c #define VIDFREQ_P0_MASK 0x1f000000 #define VIDFREQ_P0_SHIFT 24 #define VIDFREQ_P0_CSCLK_MASK 0x00f00000 #define VIDFREQ_P0_CSCLK_SHIFT 20 #define VIDFREQ_P0_CRCLK_MASK 0x000f0000 #define VIDFREQ_P0_CRCLK_SHIFT 16 #define VIDFREQ_P1_MASK 0x00001f00 #define VIDFREQ_P1_SHIFT 8 #define VIDFREQ_P1_CSCLK_MASK 0x000000f0 #define VIDFREQ_P1_CSCLK_SHIFT 4 #define VIDFREQ_P1_CRCLK_MASK 0x0000000f #define INTTOEXT_BASE_ILK 0x11300 #define INTTOEXT_BASE 0x11120 /* INTTOEXT1-8 (0x1113c) */ #define INTTOEXT_MAP3_SHIFT 24 #define INTTOEXT_MAP3_MASK (0x1f << INTTOEXT_MAP3_SHIFT) #define INTTOEXT_MAP2_SHIFT 16 #define INTTOEXT_MAP2_MASK (0x1f << INTTOEXT_MAP2_SHIFT) #define INTTOEXT_MAP1_SHIFT 8 #define INTTOEXT_MAP1_MASK (0x1f << INTTOEXT_MAP1_SHIFT) #define INTTOEXT_MAP0_SHIFT 0 #define INTTOEXT_MAP0_MASK (0x1f << INTTOEXT_MAP0_SHIFT) #define MEMSWCTL 0x11170 /* Ironlake only */ #define MEMCTL_CMD_MASK 0xe000 #define MEMCTL_CMD_SHIFT 13 #define MEMCTL_CMD_RCLK_OFF 0 #define MEMCTL_CMD_RCLK_ON 1 #define MEMCTL_CMD_CHFREQ 2 #define MEMCTL_CMD_CHVID 3 #define MEMCTL_CMD_VMMOFF 4 #define MEMCTL_CMD_VMMON 5 #define MEMCTL_CMD_STS (1<<12) /* write 1 triggers command, clears when command complete */ #define MEMCTL_FREQ_MASK 0x0f00 /* jitter, from 0-15 */ #define MEMCTL_FREQ_SHIFT 8 #define MEMCTL_SFCAVM (1<<7) #define MEMCTL_TGT_VID_MASK 0x007f #define MEMIHYST 0x1117c #define MEMINTREN 0x11180 /* 16 bits */ #define MEMINT_RSEXIT_EN (1<<8) #define MEMINT_CX_SUPR_EN (1<<7) #define MEMINT_CONT_BUSY_EN (1<<6) #define MEMINT_AVG_BUSY_EN (1<<5) #define MEMINT_EVAL_CHG_EN (1<<4) #define MEMINT_MON_IDLE_EN (1<<3) #define MEMINT_UP_EVAL_EN (1<<2) #define MEMINT_DOWN_EVAL_EN (1<<1) #define MEMINT_SW_CMD_EN (1<<0) #define MEMINTRSTR 0x11182 /* 16 bits */ #define MEM_RSEXIT_MASK 0xc000 #define MEM_RSEXIT_SHIFT 14 #define MEM_CONT_BUSY_MASK 0x3000 #define MEM_CONT_BUSY_SHIFT 12 #define MEM_AVG_BUSY_MASK 0x0c00 #define MEM_AVG_BUSY_SHIFT 10 #define MEM_EVAL_CHG_MASK 0x0300 #define MEM_EVAL_BUSY_SHIFT 8 #define MEM_MON_IDLE_MASK 0x00c0 #define MEM_MON_IDLE_SHIFT 6 #define MEM_UP_EVAL_MASK 0x0030 #define MEM_UP_EVAL_SHIFT 4 #define MEM_DOWN_EVAL_MASK 0x000c #define MEM_DOWN_EVAL_SHIFT 2 #define MEM_SW_CMD_MASK 0x0003 #define MEM_INT_STEER_GFX 0 #define MEM_INT_STEER_CMR 1 #define MEM_INT_STEER_SMI 2 #define MEM_INT_STEER_SCI 3 #define MEMINTRSTS 0x11184 #define MEMINT_RSEXIT (1<<7) #define MEMINT_CONT_BUSY (1<<6) #define MEMINT_AVG_BUSY (1<<5) #define MEMINT_EVAL_CHG (1<<4) #define MEMINT_MON_IDLE (1<<3) #define MEMINT_UP_EVAL (1<<2) #define MEMINT_DOWN_EVAL (1<<1) #define MEMINT_SW_CMD (1<<0) #define MEMMODECTL 0x11190 #define MEMMODE_BOOST_EN (1<<31) #define MEMMODE_BOOST_FREQ_MASK 0x0f000000 /* jitter for boost, 0-15 */ #define MEMMODE_BOOST_FREQ_SHIFT 24 #define MEMMODE_IDLE_MODE_MASK 0x00030000 #define MEMMODE_IDLE_MODE_SHIFT 16 #define MEMMODE_IDLE_MODE_EVAL 0 #define MEMMODE_IDLE_MODE_CONT 1 #define MEMMODE_HWIDLE_EN (1<<15) #define MEMMODE_SWMODE_EN (1<<14) #define MEMMODE_RCLK_GATE (1<<13) #define MEMMODE_HW_UPDATE (1<<12) #define MEMMODE_FSTART_MASK 0x00000f00 /* starting jitter, 0-15 */ #define MEMMODE_FSTART_SHIFT 8 #define MEMMODE_FMAX_MASK 0x000000f0 /* max jitter, 0-15 */ #define MEMMODE_FMAX_SHIFT 4 #define MEMMODE_FMIN_MASK 0x0000000f /* min jitter, 0-15 */ #define RCBMAXAVG 0x1119c #define MEMSWCTL2 0x1119e /* Cantiga only */ #define SWMEMCMD_RENDER_OFF (0 << 13) #define SWMEMCMD_RENDER_ON (1 << 13) #define SWMEMCMD_SWFREQ (2 << 13) #define SWMEMCMD_TARVID (3 << 13) #define SWMEMCMD_VRM_OFF (4 << 13) #define SWMEMCMD_VRM_ON (5 << 13) #define CMDSTS (1<<12) #define SFCAVM (1<<11) #define SWFREQ_MASK 0x0380 /* P0-7 */ #define SWFREQ_SHIFT 7 #define TARVID_MASK 0x001f #define MEMSTAT_CTG 0x111a0 #define RCBMINAVG 0x111a0 #define RCUPEI 0x111b0 #define RCDNEI 0x111b4 #define RSTDBYCTL 0x111b8 #define RS1EN (1<<31) #define RS2EN (1<<30) #define RS3EN (1<<29) #define D3RS3EN (1<<28) /* Display D3 imlies RS3 */ #define SWPROMORSX (1<<27) /* RSx promotion timers ignored */ #define RCWAKERW (1<<26) /* Resetwarn from PCH causes wakeup */ #define DPRSLPVREN (1<<25) /* Fast voltage ramp enable */ #define GFXTGHYST (1<<24) /* Hysteresis to allow trunk gating */ #define RCX_SW_EXIT (1<<23) /* Leave RSx and prevent re-entry */ #define RSX_STATUS_MASK (7<<20) #define RSX_STATUS_ON (0<<20) #define RSX_STATUS_RC1 (1<<20) #define RSX_STATUS_RC1E (2<<20) #define RSX_STATUS_RS1 (3<<20) #define RSX_STATUS_RS2 (4<<20) /* aka rc6 */ #define RSX_STATUS_RSVD (5<<20) /* deep rc6 unsupported on ilk */ #define RSX_STATUS_RS3 (6<<20) /* rs3 unsupported on ilk */ #define RSX_STATUS_RSVD2 (7<<20) #define UWRCRSXE (1<<19) /* wake counter limit prevents rsx */ #define RSCRP (1<<18) /* rs requests control on rs1/2 reqs */ #define JRSC (1<<17) /* rsx coupled to cpu c-state */ #define RS2INC0 (1<<16) /* allow rs2 in cpu c0 */ #define RS1CONTSAV_MASK (3<<14) #define RS1CONTSAV_NO_RS1 (0<<14) /* rs1 doesn't save/restore context */ #define RS1CONTSAV_RSVD (1<<14) #define RS1CONTSAV_SAVE_RS1 (2<<14) /* rs1 saves context */ #define RS1CONTSAV_FULL_RS1 (3<<14) /* rs1 saves and restores context */ #define NORMSLEXLAT_MASK (3<<12) #define SLOW_RS123 (0<<12) #define SLOW_RS23 (1<<12) #define SLOW_RS3 (2<<12) #define NORMAL_RS123 (3<<12) #define RCMODE_TIMEOUT (1<<11) /* 0 is eval interval method */ #define IMPROMOEN (1<<10) /* promo is immediate or delayed until next idle interval (only for timeout method above) */ #define RCENTSYNC (1<<9) /* rs coupled to cpu c-state (3/6/7) */ #define STATELOCK (1<<7) /* locked to rs_cstate if 0 */ #define RS_CSTATE_MASK (3<<4) #define RS_CSTATE_C367_RS1 (0<<4) #define RS_CSTATE_C36_RS1_C7_RS2 (1<<4) #define RS_CSTATE_RSVD (2<<4) #define RS_CSTATE_C367_RS2 (3<<4) #define REDSAVES (1<<3) /* no context save if was idle during rs0 */ #define REDRESTORES (1<<2) /* no restore if was idle during rs0 */ #define VIDCTL 0x111c0 #define VIDSTS 0x111c8 #define VIDSTART 0x111cc /* 8 bits */ #define MEMSTAT_ILK 0x111f8 #define MEMSTAT_VID_MASK 0x7f00 #define MEMSTAT_VID_SHIFT 8 #define MEMSTAT_PSTATE_MASK 0x00f8 #define MEMSTAT_PSTATE_SHIFT 3 #define MEMSTAT_MON_ACTV (1<<2) #define MEMSTAT_SRC_CTL_MASK 0x0003 #define MEMSTAT_SRC_CTL_CORE 0 #define MEMSTAT_SRC_CTL_TRB 1 #define MEMSTAT_SRC_CTL_THM 2 #define MEMSTAT_SRC_CTL_STDBY 3 #define RCPREVBSYTUPAVG 0x113b8 #define RCPREVBSYTDNAVG 0x113bc #define PMMISC 0x11214 #define MCPPCE_EN (1<<0) /* enable PM_MSG from PCH->MPC */ #define SDEW 0x1124c #define CSIEW0 0x11250 #define CSIEW1 0x11254 #define CSIEW2 0x11258 #define PEW 0x1125c #define DEW 0x11270 #define MCHAFE 0x112c0 #define CSIEC 0x112e0 #define DMIEC 0x112e4 #define DDREC 0x112e8 #define PEG0EC 0x112ec #define PEG1EC 0x112f0 #define GFXEC 0x112f4 #define RPPREVBSYTUPAVG 0x113b8 #define RPPREVBSYTDNAVG 0x113bc #define ECR 0x11600 #define ECR_GPFE (1<<31) #define ECR_IMONE (1<<30) #define ECR_CAP_MASK 0x0000001f /* Event range, 0-31 */ #define OGW0 0x11608 #define OGW1 0x1160c #define EG0 0x11610 #define EG1 0x11614 #define EG2 0x11618 #define EG3 0x1161c #define EG4 0x11620 #define EG5 0x11624 #define EG6 0x11628 #define EG7 0x1162c #define PXW 0x11664 #define PXWL 0x11680 #define LCFUSE02 0x116c0 #define LCFUSE_HIV_MASK 0x000000ff #define CSIPLL0 0x12c10 #define DDRMPLL1 0X12c20 #define PEG_BAND_GAP_DATA 0x14d68 #define GEN6_GT_PERF_STATUS 0x145948 #define GEN6_RP_STATE_LIMITS 0x145994 #define GEN6_RP_STATE_CAP 0x145998 /* * Logical Context regs */ #define CCID 0x2180 #define CCID_EN (1<<0) +#define CXT_SIZE 0x21a0 +#define GEN6_CXT_POWER_SIZE(cxt_reg) ((cxt_reg >> 24) & 0x3f) +#define GEN6_CXT_RING_SIZE(cxt_reg) ((cxt_reg >> 18) & 0x3f) +#define GEN6_CXT_RENDER_SIZE(cxt_reg) ((cxt_reg >> 12) & 0x3f) +#define GEN6_CXT_EXTENDED_SIZE(cxt_reg) ((cxt_reg >> 6) & 0x3f) +#define GEN6_CXT_PIPELINE_SIZE(cxt_reg) ((cxt_reg >> 0) & 0x3f) +#define GEN6_CXT_TOTAL_SIZE(cxt_reg) (GEN6_CXT_POWER_SIZE(cxt_reg) + \ + GEN6_CXT_RING_SIZE(cxt_reg) + \ + GEN6_CXT_RENDER_SIZE(cxt_reg) + \ + GEN6_CXT_EXTENDED_SIZE(cxt_reg) + \ + GEN6_CXT_PIPELINE_SIZE(cxt_reg)) +#define GEN7_CXT_SIZE 0x21a8 +#define GEN7_CXT_POWER_SIZE(ctx_reg) ((ctx_reg >> 25) & 0x7f) +#define GEN7_CXT_RING_SIZE(ctx_reg) ((ctx_reg >> 22) & 0x7) +#define GEN7_CXT_RENDER_SIZE(ctx_reg) ((ctx_reg >> 16) & 0x3f) +#define GEN7_CXT_EXTENDED_SIZE(ctx_reg) ((ctx_reg >> 9) & 0x7f) +#define GEN7_CXT_GT1_SIZE(ctx_reg) ((ctx_reg >> 6) & 0x7) +#define GEN7_CXT_VFSTATE_SIZE(ctx_reg) ((ctx_reg >> 0) & 0x3f) +#define GEN7_CXT_TOTAL_SIZE(ctx_reg) (GEN7_CXT_POWER_SIZE(ctx_reg) + \ + GEN7_CXT_RING_SIZE(ctx_reg) + \ + GEN7_CXT_RENDER_SIZE(ctx_reg) + \ + GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \ + GEN7_CXT_GT1_SIZE(ctx_reg) + \ + GEN7_CXT_VFSTATE_SIZE(ctx_reg)) + /* * Overlay regs */ #define OVADD 0x30000 #define DOVSTA 0x30008 #define OC_BUF (0x3<<20) #define OGAMC5 0x30010 #define OGAMC4 0x30014 #define OGAMC3 0x30018 #define OGAMC2 0x3001c #define OGAMC1 0x30020 #define OGAMC0 0x30024 /* * Display engine regs */ /* Pipe A timing regs */ #define _HTOTAL_A 0x60000 #define _HBLANK_A 0x60004 #define _HSYNC_A 0x60008 #define _VTOTAL_A 0x6000c #define _VBLANK_A 0x60010 #define _VSYNC_A 0x60014 #define _PIPEASRC 0x6001c #define _BCLRPAT_A 0x60020 #define _VSYNCSHIFT_A 0x60028 /* Pipe B timing regs */ #define _HTOTAL_B 0x61000 #define _HBLANK_B 0x61004 #define _HSYNC_B 0x61008 #define _VTOTAL_B 0x6100c #define _VBLANK_B 0x61010 #define _VSYNC_B 0x61014 #define _PIPEBSRC 0x6101c #define _BCLRPAT_B 0x61020 #define _VSYNCSHIFT_B 0x61028 #define HTOTAL(pipe) _PIPE(pipe, _HTOTAL_A, _HTOTAL_B) #define HBLANK(pipe) _PIPE(pipe, _HBLANK_A, _HBLANK_B) #define HSYNC(pipe) _PIPE(pipe, _HSYNC_A, _HSYNC_B) #define VTOTAL(pipe) _PIPE(pipe, _VTOTAL_A, _VTOTAL_B) #define VBLANK(pipe) _PIPE(pipe, _VBLANK_A, _VBLANK_B) #define VSYNC(pipe) _PIPE(pipe, _VSYNC_A, _VSYNC_B) #define BCLRPAT(pipe) _PIPE(pipe, _BCLRPAT_A, _BCLRPAT_B) #define VSYNCSHIFT(pipe) _PIPE(pipe, _VSYNCSHIFT_A, _VSYNCSHIFT_B) /* VGA port control */ #define ADPA 0x61100 #define ADPA_DAC_ENABLE (1<<31) #define ADPA_DAC_DISABLE 0 #define ADPA_PIPE_SELECT_MASK (1<<30) #define ADPA_PIPE_A_SELECT 0 #define ADPA_PIPE_B_SELECT (1<<30) #define ADPA_PIPE_SELECT(pipe) ((pipe) << 30) #define ADPA_USE_VGA_HVPOLARITY (1<<15) #define ADPA_SETS_HVPOLARITY 0 #define ADPA_VSYNC_CNTL_DISABLE (1<<11) #define ADPA_VSYNC_CNTL_ENABLE 0 #define ADPA_HSYNC_CNTL_DISABLE (1<<10) #define ADPA_HSYNC_CNTL_ENABLE 0 #define ADPA_VSYNC_ACTIVE_HIGH (1<<4) #define ADPA_VSYNC_ACTIVE_LOW 0 #define ADPA_HSYNC_ACTIVE_HIGH (1<<3) #define ADPA_HSYNC_ACTIVE_LOW 0 #define ADPA_DPMS_MASK (~(3<<10)) #define ADPA_DPMS_ON (0<<10) #define ADPA_DPMS_SUSPEND (1<<10) #define ADPA_DPMS_STANDBY (2<<10) #define ADPA_DPMS_OFF (3<<10) /* Hotplug control (945+ only) */ #define PORT_HOTPLUG_EN 0x61110 #define HDMIB_HOTPLUG_INT_EN (1 << 29) #define DPB_HOTPLUG_INT_EN (1 << 29) #define HDMIC_HOTPLUG_INT_EN (1 << 28) #define DPC_HOTPLUG_INT_EN (1 << 28) #define HDMID_HOTPLUG_INT_EN (1 << 27) #define DPD_HOTPLUG_INT_EN (1 << 27) #define SDVOB_HOTPLUG_INT_EN (1 << 26) #define SDVOC_HOTPLUG_INT_EN (1 << 25) #define TV_HOTPLUG_INT_EN (1 << 18) #define CRT_HOTPLUG_INT_EN (1 << 9) #define CRT_HOTPLUG_FORCE_DETECT (1 << 3) #define CRT_HOTPLUG_ACTIVATION_PERIOD_32 (0 << 8) /* must use period 64 on GM45 according to docs */ #define CRT_HOTPLUG_ACTIVATION_PERIOD_64 (1 << 8) #define CRT_HOTPLUG_DAC_ON_TIME_2M (0 << 7) #define CRT_HOTPLUG_DAC_ON_TIME_4M (1 << 7) #define CRT_HOTPLUG_VOLTAGE_COMPARE_40 (0 << 5) #define CRT_HOTPLUG_VOLTAGE_COMPARE_50 (1 << 5) #define CRT_HOTPLUG_VOLTAGE_COMPARE_60 (2 << 5) #define CRT_HOTPLUG_VOLTAGE_COMPARE_70 (3 << 5) #define CRT_HOTPLUG_VOLTAGE_COMPARE_MASK (3 << 5) #define CRT_HOTPLUG_DETECT_DELAY_1G (0 << 4) #define CRT_HOTPLUG_DETECT_DELAY_2G (1 << 4) #define CRT_HOTPLUG_DETECT_VOLTAGE_325MV (0 << 2) #define CRT_HOTPLUG_DETECT_VOLTAGE_475MV (1 << 2) #define PORT_HOTPLUG_STAT 0x61114 #define HDMIB_HOTPLUG_INT_STATUS (1 << 29) #define DPB_HOTPLUG_INT_STATUS (1 << 29) #define HDMIC_HOTPLUG_INT_STATUS (1 << 28) #define DPC_HOTPLUG_INT_STATUS (1 << 28) #define HDMID_HOTPLUG_INT_STATUS (1 << 27) #define DPD_HOTPLUG_INT_STATUS (1 << 27) #define CRT_HOTPLUG_INT_STATUS (1 << 11) #define TV_HOTPLUG_INT_STATUS (1 << 10) #define CRT_HOTPLUG_MONITOR_MASK (3 << 8) #define CRT_HOTPLUG_MONITOR_COLOR (3 << 8) #define CRT_HOTPLUG_MONITOR_MONO (2 << 8) #define CRT_HOTPLUG_MONITOR_NONE (0 << 8) #define SDVOC_HOTPLUG_INT_STATUS (1 << 7) #define SDVOB_HOTPLUG_INT_STATUS (1 << 6) /* SDVO port control */ #define SDVOB 0x61140 #define SDVOC 0x61160 #define SDVO_ENABLE (1 << 31) #define SDVO_PIPE_B_SELECT (1 << 30) #define SDVO_STALL_SELECT (1 << 29) #define SDVO_INTERRUPT_ENABLE (1 << 26) /** * 915G/GM SDVO pixel multiplier. * * Programmed value is multiplier - 1, up to 5x. * * \sa DPLL_MD_UDI_MULTIPLIER_MASK */ #define SDVO_PORT_MULTIPLY_MASK (7 << 23) #define SDVO_PORT_MULTIPLY_SHIFT 23 #define SDVO_PHASE_SELECT_MASK (15 << 19) #define SDVO_PHASE_SELECT_DEFAULT (6 << 19) #define SDVO_CLOCK_OUTPUT_INVERT (1 << 18) #define SDVOC_GANG_MODE (1 << 16) #define SDVO_ENCODING_SDVO (0x0 << 10) #define SDVO_ENCODING_HDMI (0x2 << 10) /** Requird for HDMI operation */ #define SDVO_NULL_PACKETS_DURING_VSYNC (1 << 9) #define SDVO_COLOR_RANGE_16_235 (1 << 8) #define SDVO_BORDER_ENABLE (1 << 7) #define SDVO_AUDIO_ENABLE (1 << 6) /** New with 965, default is to be set */ #define SDVO_VSYNC_ACTIVE_HIGH (1 << 4) /** New with 965, default is to be set */ #define SDVO_HSYNC_ACTIVE_HIGH (1 << 3) #define SDVOB_PCIE_CONCURRENCY (1 << 3) #define SDVO_DETECTED (1 << 2) /* Bits to be preserved when writing */ #define SDVOB_PRESERVE_MASK ((1 << 17) | (1 << 16) | (1 << 14) | (1 << 26)) #define SDVOC_PRESERVE_MASK ((1 << 17) | (1 << 26)) /* DVO port control */ #define DVOA 0x61120 #define DVOB 0x61140 #define DVOC 0x61160 #define DVO_ENABLE (1 << 31) #define DVO_PIPE_B_SELECT (1 << 30) #define DVO_PIPE_STALL_UNUSED (0 << 28) #define DVO_PIPE_STALL (1 << 28) #define DVO_PIPE_STALL_TV (2 << 28) #define DVO_PIPE_STALL_MASK (3 << 28) #define DVO_USE_VGA_SYNC (1 << 15) #define DVO_DATA_ORDER_I740 (0 << 14) #define DVO_DATA_ORDER_FP (1 << 14) #define DVO_VSYNC_DISABLE (1 << 11) #define DVO_HSYNC_DISABLE (1 << 10) #define DVO_VSYNC_TRISTATE (1 << 9) #define DVO_HSYNC_TRISTATE (1 << 8) #define DVO_BORDER_ENABLE (1 << 7) #define DVO_DATA_ORDER_GBRG (1 << 6) #define DVO_DATA_ORDER_RGGB (0 << 6) #define DVO_DATA_ORDER_GBRG_ERRATA (0 << 6) #define DVO_DATA_ORDER_RGGB_ERRATA (1 << 6) #define DVO_VSYNC_ACTIVE_HIGH (1 << 4) #define DVO_HSYNC_ACTIVE_HIGH (1 << 3) #define DVO_BLANK_ACTIVE_HIGH (1 << 2) #define DVO_OUTPUT_CSTATE_PIXELS (1 << 1) /* SDG only */ #define DVO_OUTPUT_SOURCE_SIZE_PIXELS (1 << 0) /* SDG only */ #define DVO_PRESERVE_MASK (0x7<<24) #define DVOA_SRCDIM 0x61124 #define DVOB_SRCDIM 0x61144 #define DVOC_SRCDIM 0x61164 #define DVO_SRCDIM_HORIZONTAL_SHIFT 12 #define DVO_SRCDIM_VERTICAL_SHIFT 0 /* LVDS port control */ #define LVDS 0x61180 /* * Enables the LVDS port. This bit must be set before DPLLs are enabled, as * the DPLL semantics change when the LVDS is assigned to that pipe. */ #define LVDS_PORT_EN (1 << 31) /* Selects pipe B for LVDS data. Must be set on pre-965. */ #define LVDS_PIPEB_SELECT (1 << 30) #define LVDS_PIPE_MASK (1 << 30) #define LVDS_PIPE(pipe) ((pipe) << 30) /* LVDS dithering flag on 965/g4x platform */ #define LVDS_ENABLE_DITHER (1 << 25) /* LVDS sync polarity flags. Set to invert (i.e. negative) */ #define LVDS_VSYNC_POLARITY (1 << 21) #define LVDS_HSYNC_POLARITY (1 << 20) /* Enable border for unscaled (or aspect-scaled) display */ #define LVDS_BORDER_ENABLE (1 << 15) /* * Enables the A0-A2 data pairs and CLKA, containing 18 bits of color data per * pixel. */ #define LVDS_A0A2_CLKA_POWER_MASK (3 << 8) #define LVDS_A0A2_CLKA_POWER_DOWN (0 << 8) #define LVDS_A0A2_CLKA_POWER_UP (3 << 8) /* * Controls the A3 data pair, which contains the additional LSBs for 24 bit * mode. Only enabled if LVDS_A0A2_CLKA_POWER_UP also indicates it should be * on. */ #define LVDS_A3_POWER_MASK (3 << 6) #define LVDS_A3_POWER_DOWN (0 << 6) #define LVDS_A3_POWER_UP (3 << 6) /* * Controls the CLKB pair. This should only be set when LVDS_B0B3_POWER_UP * is set. */ #define LVDS_CLKB_POWER_MASK (3 << 4) #define LVDS_CLKB_POWER_DOWN (0 << 4) #define LVDS_CLKB_POWER_UP (3 << 4) /* * Controls the B0-B3 data pairs. This must be set to match the DPLL p2 * setting for whether we are in dual-channel mode. The B3 pair will * additionally only be powered up when LVDS_A3_POWER_UP is set. */ #define LVDS_B0B3_POWER_MASK (3 << 2) #define LVDS_B0B3_POWER_DOWN (0 << 2) #define LVDS_B0B3_POWER_UP (3 << 2) /* Video Data Island Packet control */ #define VIDEO_DIP_DATA 0x61178 #define VIDEO_DIP_CTL 0x61170 #define VIDEO_DIP_ENABLE (1 << 31) #define VIDEO_DIP_PORT_B (1 << 29) #define VIDEO_DIP_PORT_C (2 << 29) #define VIDEO_DIP_ENABLE_AVI (1 << 21) #define VIDEO_DIP_ENABLE_VENDOR (2 << 21) #define VIDEO_DIP_ENABLE_SPD (8 << 21) #define VIDEO_DIP_SELECT_MASK (3 << 19) #define VIDEO_DIP_SELECT_AVI (0 << 19) #define VIDEO_DIP_SELECT_VENDOR (1 << 19) #define VIDEO_DIP_SELECT_SPD (3 << 19) #define VIDEO_DIP_FREQ_ONCE (0 << 16) #define VIDEO_DIP_FREQ_VSYNC (1 << 16) #define VIDEO_DIP_FREQ_2VSYNC (2 << 16) /* Panel power sequencing */ #define PP_STATUS 0x61200 #define PP_ON (1 << 31) /* * Indicates that all dependencies of the panel are on: * * - PLL enabled * - pipe enabled * - LVDS/DVOB/DVOC on */ #define PP_READY (1 << 30) #define PP_SEQUENCE_NONE (0 << 28) #define PP_SEQUENCE_POWER_UP (1 << 28) #define PP_SEQUENCE_POWER_DOWN (2 << 28) #define PP_SEQUENCE_MASK (3 << 28) #define PP_SEQUENCE_SHIFT 28 #define PP_CYCLE_DELAY_ACTIVE (1 << 27) #define PP_SEQUENCE_STATE_MASK 0x0000000f #define PP_SEQUENCE_STATE_OFF_IDLE (0x0 << 0) #define PP_SEQUENCE_STATE_OFF_S0_1 (0x1 << 0) #define PP_SEQUENCE_STATE_OFF_S0_2 (0x2 << 0) #define PP_SEQUENCE_STATE_OFF_S0_3 (0x3 << 0) #define PP_SEQUENCE_STATE_ON_IDLE (0x8 << 0) #define PP_SEQUENCE_STATE_ON_S1_0 (0x9 << 0) #define PP_SEQUENCE_STATE_ON_S1_2 (0xa << 0) #define PP_SEQUENCE_STATE_ON_S1_3 (0xb << 0) #define PP_SEQUENCE_STATE_RESET (0xf << 0) #define PP_CONTROL 0x61204 #define POWER_TARGET_ON (1 << 0) #define PP_ON_DELAYS 0x61208 #define PP_OFF_DELAYS 0x6120c #define PP_DIVISOR 0x61210 /* Panel fitting */ #define PFIT_CONTROL 0x61230 #define PFIT_ENABLE (1 << 31) #define PFIT_PIPE_MASK (3 << 29) #define PFIT_PIPE_SHIFT 29 #define VERT_INTERP_DISABLE (0 << 10) #define VERT_INTERP_BILINEAR (1 << 10) #define VERT_INTERP_MASK (3 << 10) #define VERT_AUTO_SCALE (1 << 9) #define HORIZ_INTERP_DISABLE (0 << 6) #define HORIZ_INTERP_BILINEAR (1 << 6) #define HORIZ_INTERP_MASK (3 << 6) #define HORIZ_AUTO_SCALE (1 << 5) #define PANEL_8TO6_DITHER_ENABLE (1 << 3) #define PFIT_FILTER_FUZZY (0 << 24) #define PFIT_SCALING_AUTO (0 << 26) #define PFIT_SCALING_PROGRAMMED (1 << 26) #define PFIT_SCALING_PILLAR (2 << 26) #define PFIT_SCALING_LETTER (3 << 26) #define PFIT_PGM_RATIOS 0x61234 #define PFIT_VERT_SCALE_MASK 0xfff00000 #define PFIT_HORIZ_SCALE_MASK 0x0000fff0 /* Pre-965 */ #define PFIT_VERT_SCALE_SHIFT 20 #define PFIT_VERT_SCALE_MASK 0xfff00000 #define PFIT_HORIZ_SCALE_SHIFT 4 #define PFIT_HORIZ_SCALE_MASK 0x0000fff0 /* 965+ */ #define PFIT_VERT_SCALE_SHIFT_965 16 #define PFIT_VERT_SCALE_MASK_965 0x1fff0000 #define PFIT_HORIZ_SCALE_SHIFT_965 0 #define PFIT_HORIZ_SCALE_MASK_965 0x00001fff #define PFIT_AUTO_RATIOS 0x61238 /* Backlight control */ #define BLC_PWM_CTL 0x61254 #define BACKLIGHT_MODULATION_FREQ_SHIFT (17) #define BLC_PWM_CTL2 0x61250 /* 965+ only */ #define BLM_COMBINATION_MODE (1 << 30) /* * This is the most significant 15 bits of the number of backlight cycles in a * complete cycle of the modulated backlight control. * * The actual value is this field multiplied by two. */ #define BACKLIGHT_MODULATION_FREQ_MASK (0x7fff << 17) #define BLM_LEGACY_MODE (1 << 16) /* * This is the number of cycles out of the backlight modulation cycle for which * the backlight is on. * * This field must be no greater than the number of cycles in the complete * backlight modulation cycle. */ #define BACKLIGHT_DUTY_CYCLE_SHIFT (0) #define BACKLIGHT_DUTY_CYCLE_MASK (0xffff) #define BLC_HIST_CTL 0x61260 /* TV port control */ #define TV_CTL 0x68000 /** Enables the TV encoder */ # define TV_ENC_ENABLE (1 << 31) /** Sources the TV encoder input from pipe B instead of A. */ # define TV_ENC_PIPEB_SELECT (1 << 30) /** Outputs composite video (DAC A only) */ # define TV_ENC_OUTPUT_COMPOSITE (0 << 28) /** Outputs SVideo video (DAC B/C) */ # define TV_ENC_OUTPUT_SVIDEO (1 << 28) /** Outputs Component video (DAC A/B/C) */ # define TV_ENC_OUTPUT_COMPONENT (2 << 28) /** Outputs Composite and SVideo (DAC A/B/C) */ # define TV_ENC_OUTPUT_SVIDEO_COMPOSITE (3 << 28) # define TV_TRILEVEL_SYNC (1 << 21) /** Enables slow sync generation (945GM only) */ # define TV_SLOW_SYNC (1 << 20) /** Selects 4x oversampling for 480i and 576p */ # define TV_OVERSAMPLE_4X (0 << 18) /** Selects 2x oversampling for 720p and 1080i */ # define TV_OVERSAMPLE_2X (1 << 18) /** Selects no oversampling for 1080p */ # define TV_OVERSAMPLE_NONE (2 << 18) /** Selects 8x oversampling */ # define TV_OVERSAMPLE_8X (3 << 18) /** Selects progressive mode rather than interlaced */ # define TV_PROGRESSIVE (1 << 17) /** Sets the colorburst to PAL mode. Required for non-M PAL modes. */ # define TV_PAL_BURST (1 << 16) /** Field for setting delay of Y compared to C */ # define TV_YC_SKEW_MASK (7 << 12) /** Enables a fix for 480p/576p standard definition modes on the 915GM only */ # define TV_ENC_SDP_FIX (1 << 11) /** * Enables a fix for the 915GM only. * * Not sure what it does. */ # define TV_ENC_C0_FIX (1 << 10) /** Bits that must be preserved by software */ # define TV_CTL_SAVE ((1 << 11) | (3 << 9) | (7 << 6) | 0xf) # define TV_FUSE_STATE_MASK (3 << 4) /** Read-only state that reports all features enabled */ # define TV_FUSE_STATE_ENABLED (0 << 4) /** Read-only state that reports that Macrovision is disabled in hardware*/ # define TV_FUSE_STATE_NO_MACROVISION (1 << 4) /** Read-only state that reports that TV-out is disabled in hardware. */ # define TV_FUSE_STATE_DISABLED (2 << 4) /** Normal operation */ # define TV_TEST_MODE_NORMAL (0 << 0) /** Encoder test pattern 1 - combo pattern */ # define TV_TEST_MODE_PATTERN_1 (1 << 0) /** Encoder test pattern 2 - full screen vertical 75% color bars */ # define TV_TEST_MODE_PATTERN_2 (2 << 0) /** Encoder test pattern 3 - full screen horizontal 75% color bars */ # define TV_TEST_MODE_PATTERN_3 (3 << 0) /** Encoder test pattern 4 - random noise */ # define TV_TEST_MODE_PATTERN_4 (4 << 0) /** Encoder test pattern 5 - linear color ramps */ # define TV_TEST_MODE_PATTERN_5 (5 << 0) /** * This test mode forces the DACs to 50% of full output. * * This is used for load detection in combination with TVDAC_SENSE_MASK */ # define TV_TEST_MODE_MONITOR_DETECT (7 << 0) # define TV_TEST_MODE_MASK (7 << 0) #define TV_DAC 0x68004 # define TV_DAC_SAVE 0x00ffff00 /** * Reports that DAC state change logic has reported change (RO). * * This gets cleared when TV_DAC_STATE_EN is cleared */ # define TVDAC_STATE_CHG (1 << 31) # define TVDAC_SENSE_MASK (7 << 28) /** Reports that DAC A voltage is above the detect threshold */ # define TVDAC_A_SENSE (1 << 30) /** Reports that DAC B voltage is above the detect threshold */ # define TVDAC_B_SENSE (1 << 29) /** Reports that DAC C voltage is above the detect threshold */ # define TVDAC_C_SENSE (1 << 28) /** * Enables DAC state detection logic, for load-based TV detection. * * The PLL of the chosen pipe (in TV_CTL) must be running, and the encoder set * to off, for load detection to work. */ # define TVDAC_STATE_CHG_EN (1 << 27) /** Sets the DAC A sense value to high */ # define TVDAC_A_SENSE_CTL (1 << 26) /** Sets the DAC B sense value to high */ # define TVDAC_B_SENSE_CTL (1 << 25) /** Sets the DAC C sense value to high */ # define TVDAC_C_SENSE_CTL (1 << 24) /** Overrides the ENC_ENABLE and DAC voltage levels */ # define DAC_CTL_OVERRIDE (1 << 7) /** Sets the slew rate. Must be preserved in software */ # define ENC_TVDAC_SLEW_FAST (1 << 6) # define DAC_A_1_3_V (0 << 4) # define DAC_A_1_1_V (1 << 4) # define DAC_A_0_7_V (2 << 4) # define DAC_A_MASK (3 << 4) # define DAC_B_1_3_V (0 << 2) # define DAC_B_1_1_V (1 << 2) # define DAC_B_0_7_V (2 << 2) # define DAC_B_MASK (3 << 2) # define DAC_C_1_3_V (0 << 0) # define DAC_C_1_1_V (1 << 0) # define DAC_C_0_7_V (2 << 0) # define DAC_C_MASK (3 << 0) /** * CSC coefficients are stored in a floating point format with 9 bits of * mantissa and 2 or 3 bits of exponent. The exponent is represented as 2**-n, * where 2-bit exponents are unsigned n, and 3-bit exponents are signed n with * -1 (0x3) being the only legal negative value. */ #define TV_CSC_Y 0x68010 # define TV_RY_MASK 0x07ff0000 # define TV_RY_SHIFT 16 # define TV_GY_MASK 0x00000fff # define TV_GY_SHIFT 0 #define TV_CSC_Y2 0x68014 # define TV_BY_MASK 0x07ff0000 # define TV_BY_SHIFT 16 /** * Y attenuation for component video. * * Stored in 1.9 fixed point. */ # define TV_AY_MASK 0x000003ff # define TV_AY_SHIFT 0 #define TV_CSC_U 0x68018 # define TV_RU_MASK 0x07ff0000 # define TV_RU_SHIFT 16 # define TV_GU_MASK 0x000007ff # define TV_GU_SHIFT 0 #define TV_CSC_U2 0x6801c # define TV_BU_MASK 0x07ff0000 # define TV_BU_SHIFT 16 /** * U attenuation for component video. * * Stored in 1.9 fixed point. */ # define TV_AU_MASK 0x000003ff # define TV_AU_SHIFT 0 #define TV_CSC_V 0x68020 # define TV_RV_MASK 0x0fff0000 # define TV_RV_SHIFT 16 # define TV_GV_MASK 0x000007ff # define TV_GV_SHIFT 0 #define TV_CSC_V2 0x68024 # define TV_BV_MASK 0x07ff0000 # define TV_BV_SHIFT 16 /** * V attenuation for component video. * * Stored in 1.9 fixed point. */ # define TV_AV_MASK 0x000007ff # define TV_AV_SHIFT 0 #define TV_CLR_KNOBS 0x68028 /** 2s-complement brightness adjustment */ # define TV_BRIGHTNESS_MASK 0xff000000 # define TV_BRIGHTNESS_SHIFT 24 /** Contrast adjustment, as a 2.6 unsigned floating point number */ # define TV_CONTRAST_MASK 0x00ff0000 # define TV_CONTRAST_SHIFT 16 /** Saturation adjustment, as a 2.6 unsigned floating point number */ # define TV_SATURATION_MASK 0x0000ff00 # define TV_SATURATION_SHIFT 8 /** Hue adjustment, as an integer phase angle in degrees */ # define TV_HUE_MASK 0x000000ff # define TV_HUE_SHIFT 0 #define TV_CLR_LEVEL 0x6802c /** Controls the DAC level for black */ # define TV_BLACK_LEVEL_MASK 0x01ff0000 # define TV_BLACK_LEVEL_SHIFT 16 /** Controls the DAC level for blanking */ # define TV_BLANK_LEVEL_MASK 0x000001ff # define TV_BLANK_LEVEL_SHIFT 0 #define TV_H_CTL_1 0x68030 /** Number of pixels in the hsync. */ # define TV_HSYNC_END_MASK 0x1fff0000 # define TV_HSYNC_END_SHIFT 16 /** Total number of pixels minus one in the line (display and blanking). */ # define TV_HTOTAL_MASK 0x00001fff # define TV_HTOTAL_SHIFT 0 #define TV_H_CTL_2 0x68034 /** Enables the colorburst (needed for non-component color) */ # define TV_BURST_ENA (1 << 31) /** Offset of the colorburst from the start of hsync, in pixels minus one. */ # define TV_HBURST_START_SHIFT 16 # define TV_HBURST_START_MASK 0x1fff0000 /** Length of the colorburst */ # define TV_HBURST_LEN_SHIFT 0 # define TV_HBURST_LEN_MASK 0x0001fff #define TV_H_CTL_3 0x68038 /** End of hblank, measured in pixels minus one from start of hsync */ # define TV_HBLANK_END_SHIFT 16 # define TV_HBLANK_END_MASK 0x1fff0000 /** Start of hblank, measured in pixels minus one from start of hsync */ # define TV_HBLANK_START_SHIFT 0 # define TV_HBLANK_START_MASK 0x0001fff #define TV_V_CTL_1 0x6803c /** XXX */ # define TV_NBR_END_SHIFT 16 # define TV_NBR_END_MASK 0x07ff0000 /** XXX */ # define TV_VI_END_F1_SHIFT 8 # define TV_VI_END_F1_MASK 0x00003f00 /** XXX */ # define TV_VI_END_F2_SHIFT 0 # define TV_VI_END_F2_MASK 0x0000003f #define TV_V_CTL_2 0x68040 /** Length of vsync, in half lines */ # define TV_VSYNC_LEN_MASK 0x07ff0000 # define TV_VSYNC_LEN_SHIFT 16 /** Offset of the start of vsync in field 1, measured in one less than the * number of half lines. */ # define TV_VSYNC_START_F1_MASK 0x00007f00 # define TV_VSYNC_START_F1_SHIFT 8 /** * Offset of the start of vsync in field 2, measured in one less than the * number of half lines. */ # define TV_VSYNC_START_F2_MASK 0x0000007f # define TV_VSYNC_START_F2_SHIFT 0 #define TV_V_CTL_3 0x68044 /** Enables generation of the equalization signal */ # define TV_EQUAL_ENA (1 << 31) /** Length of vsync, in half lines */ # define TV_VEQ_LEN_MASK 0x007f0000 # define TV_VEQ_LEN_SHIFT 16 /** Offset of the start of equalization in field 1, measured in one less than * the number of half lines. */ # define TV_VEQ_START_F1_MASK 0x0007f00 # define TV_VEQ_START_F1_SHIFT 8 /** * Offset of the start of equalization in field 2, measured in one less than * the number of half lines. */ # define TV_VEQ_START_F2_MASK 0x000007f # define TV_VEQ_START_F2_SHIFT 0 #define TV_V_CTL_4 0x68048 /** * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. */ # define TV_VBURST_START_F1_MASK 0x003f0000 # define TV_VBURST_START_F1_SHIFT 16 /** * Offset to the end of vertical colorburst, measured in one less than the * number of lines from the start of NBR. */ # define TV_VBURST_END_F1_MASK 0x000000ff # define TV_VBURST_END_F1_SHIFT 0 #define TV_V_CTL_5 0x6804c /** * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. */ # define TV_VBURST_START_F2_MASK 0x003f0000 # define TV_VBURST_START_F2_SHIFT 16 /** * Offset to the end of vertical colorburst, measured in one less than the * number of lines from the start of NBR. */ # define TV_VBURST_END_F2_MASK 0x000000ff # define TV_VBURST_END_F2_SHIFT 0 #define TV_V_CTL_6 0x68050 /** * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. */ # define TV_VBURST_START_F3_MASK 0x003f0000 # define TV_VBURST_START_F3_SHIFT 16 /** * Offset to the end of vertical colorburst, measured in one less than the * number of lines from the start of NBR. */ # define TV_VBURST_END_F3_MASK 0x000000ff # define TV_VBURST_END_F3_SHIFT 0 #define TV_V_CTL_7 0x68054 /** * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. */ # define TV_VBURST_START_F4_MASK 0x003f0000 # define TV_VBURST_START_F4_SHIFT 16 /** * Offset to the end of vertical colorburst, measured in one less than the * number of lines from the start of NBR. */ # define TV_VBURST_END_F4_MASK 0x000000ff # define TV_VBURST_END_F4_SHIFT 0 #define TV_SC_CTL_1 0x68060 /** Turns on the first subcarrier phase generation DDA */ # define TV_SC_DDA1_EN (1 << 31) /** Turns on the first subcarrier phase generation DDA */ # define TV_SC_DDA2_EN (1 << 30) /** Turns on the first subcarrier phase generation DDA */ # define TV_SC_DDA3_EN (1 << 29) /** Sets the subcarrier DDA to reset frequency every other field */ # define TV_SC_RESET_EVERY_2 (0 << 24) /** Sets the subcarrier DDA to reset frequency every fourth field */ # define TV_SC_RESET_EVERY_4 (1 << 24) /** Sets the subcarrier DDA to reset frequency every eighth field */ # define TV_SC_RESET_EVERY_8 (2 << 24) /** Sets the subcarrier DDA to never reset the frequency */ # define TV_SC_RESET_NEVER (3 << 24) /** Sets the peak amplitude of the colorburst.*/ # define TV_BURST_LEVEL_MASK 0x00ff0000 # define TV_BURST_LEVEL_SHIFT 16 /** Sets the increment of the first subcarrier phase generation DDA */ # define TV_SCDDA1_INC_MASK 0x00000fff # define TV_SCDDA1_INC_SHIFT 0 #define TV_SC_CTL_2 0x68064 /** Sets the rollover for the second subcarrier phase generation DDA */ # define TV_SCDDA2_SIZE_MASK 0x7fff0000 # define TV_SCDDA2_SIZE_SHIFT 16 /** Sets the increent of the second subcarrier phase generation DDA */ # define TV_SCDDA2_INC_MASK 0x00007fff # define TV_SCDDA2_INC_SHIFT 0 #define TV_SC_CTL_3 0x68068 /** Sets the rollover for the third subcarrier phase generation DDA */ # define TV_SCDDA3_SIZE_MASK 0x7fff0000 # define TV_SCDDA3_SIZE_SHIFT 16 /** Sets the increent of the third subcarrier phase generation DDA */ # define TV_SCDDA3_INC_MASK 0x00007fff # define TV_SCDDA3_INC_SHIFT 0 #define TV_WIN_POS 0x68070 /** X coordinate of the display from the start of horizontal active */ # define TV_XPOS_MASK 0x1fff0000 # define TV_XPOS_SHIFT 16 /** Y coordinate of the display from the start of vertical active (NBR) */ # define TV_YPOS_MASK 0x00000fff # define TV_YPOS_SHIFT 0 #define TV_WIN_SIZE 0x68074 /** Horizontal size of the display window, measured in pixels*/ # define TV_XSIZE_MASK 0x1fff0000 # define TV_XSIZE_SHIFT 16 /** * Vertical size of the display window, measured in pixels. * * Must be even for interlaced modes. */ # define TV_YSIZE_MASK 0x00000fff # define TV_YSIZE_SHIFT 0 #define TV_FILTER_CTL_1 0x68080 /** * Enables automatic scaling calculation. * * If set, the rest of the registers are ignored, and the calculated values can * be read back from the register. */ # define TV_AUTO_SCALE (1 << 31) /** * Disables the vertical filter. * * This is required on modes more than 1024 pixels wide */ # define TV_V_FILTER_BYPASS (1 << 29) /** Enables adaptive vertical filtering */ # define TV_VADAPT (1 << 28) # define TV_VADAPT_MODE_MASK (3 << 26) /** Selects the least adaptive vertical filtering mode */ # define TV_VADAPT_MODE_LEAST (0 << 26) /** Selects the moderately adaptive vertical filtering mode */ # define TV_VADAPT_MODE_MODERATE (1 << 26) /** Selects the most adaptive vertical filtering mode */ # define TV_VADAPT_MODE_MOST (3 << 26) /** * Sets the horizontal scaling factor. * * This should be the fractional part of the horizontal scaling factor divided * by the oversampling rate. TV_HSCALE should be less than 1, and set to: * * (src width - 1) / ((oversample * dest width) - 1) */ # define TV_HSCALE_FRAC_MASK 0x00003fff # define TV_HSCALE_FRAC_SHIFT 0 #define TV_FILTER_CTL_2 0x68084 /** * Sets the integer part of the 3.15 fixed-point vertical scaling factor. * * TV_VSCALE should be (src height - 1) / ((interlace * dest height) - 1) */ # define TV_VSCALE_INT_MASK 0x00038000 # define TV_VSCALE_INT_SHIFT 15 /** * Sets the fractional part of the 3.15 fixed-point vertical scaling factor. * * \sa TV_VSCALE_INT_MASK */ # define TV_VSCALE_FRAC_MASK 0x00007fff # define TV_VSCALE_FRAC_SHIFT 0 #define TV_FILTER_CTL_3 0x68088 /** * Sets the integer part of the 3.15 fixed-point vertical scaling factor. * * TV_VSCALE should be (src height - 1) / (1/4 * (dest height - 1)) * * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes. */ # define TV_VSCALE_IP_INT_MASK 0x00038000 # define TV_VSCALE_IP_INT_SHIFT 15 /** * Sets the fractional part of the 3.15 fixed-point vertical scaling factor. * * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes. * * \sa TV_VSCALE_IP_INT_MASK */ # define TV_VSCALE_IP_FRAC_MASK 0x00007fff # define TV_VSCALE_IP_FRAC_SHIFT 0 #define TV_CC_CONTROL 0x68090 # define TV_CC_ENABLE (1 << 31) /** * Specifies which field to send the CC data in. * * CC data is usually sent in field 0. */ # define TV_CC_FID_MASK (1 << 27) # define TV_CC_FID_SHIFT 27 /** Sets the horizontal position of the CC data. Usually 135. */ # define TV_CC_HOFF_MASK 0x03ff0000 # define TV_CC_HOFF_SHIFT 16 /** Sets the vertical position of the CC data. Usually 21 */ # define TV_CC_LINE_MASK 0x0000003f # define TV_CC_LINE_SHIFT 0 #define TV_CC_DATA 0x68094 # define TV_CC_RDY (1 << 31) /** Second word of CC data to be transmitted. */ # define TV_CC_DATA_2_MASK 0x007f0000 # define TV_CC_DATA_2_SHIFT 16 /** First word of CC data to be transmitted. */ # define TV_CC_DATA_1_MASK 0x0000007f # define TV_CC_DATA_1_SHIFT 0 #define TV_H_LUMA_0 0x68100 #define TV_H_LUMA_59 0x681ec #define TV_H_CHROMA_0 0x68200 #define TV_H_CHROMA_59 0x682ec #define TV_V_LUMA_0 0x68300 #define TV_V_LUMA_42 0x683a8 #define TV_V_CHROMA_0 0x68400 #define TV_V_CHROMA_42 0x684a8 /* Display Port */ #define DP_A 0x64000 /* eDP */ #define DP_B 0x64100 #define DP_C 0x64200 #define DP_D 0x64300 #define DP_PORT_EN (1 << 31) #define DP_PIPEB_SELECT (1 << 30) #define DP_PIPE_MASK (1 << 30) /* Link training mode - select a suitable mode for each stage */ #define DP_LINK_TRAIN_PAT_1 (0 << 28) #define DP_LINK_TRAIN_PAT_2 (1 << 28) #define DP_LINK_TRAIN_PAT_IDLE (2 << 28) #define DP_LINK_TRAIN_OFF (3 << 28) #define DP_LINK_TRAIN_MASK (3 << 28) #define DP_LINK_TRAIN_SHIFT 28 /* CPT Link training mode */ #define DP_LINK_TRAIN_PAT_1_CPT (0 << 8) #define DP_LINK_TRAIN_PAT_2_CPT (1 << 8) #define DP_LINK_TRAIN_PAT_IDLE_CPT (2 << 8) #define DP_LINK_TRAIN_OFF_CPT (3 << 8) #define DP_LINK_TRAIN_MASK_CPT (7 << 8) #define DP_LINK_TRAIN_SHIFT_CPT 8 /* Signal voltages. These are mostly controlled by the other end */ #define DP_VOLTAGE_0_4 (0 << 25) #define DP_VOLTAGE_0_6 (1 << 25) #define DP_VOLTAGE_0_8 (2 << 25) #define DP_VOLTAGE_1_2 (3 << 25) #define DP_VOLTAGE_MASK (7 << 25) #define DP_VOLTAGE_SHIFT 25 /* Signal pre-emphasis levels, like voltages, the other end tells us what * they want */ #define DP_PRE_EMPHASIS_0 (0 << 22) #define DP_PRE_EMPHASIS_3_5 (1 << 22) #define DP_PRE_EMPHASIS_6 (2 << 22) #define DP_PRE_EMPHASIS_9_5 (3 << 22) #define DP_PRE_EMPHASIS_MASK (7 << 22) #define DP_PRE_EMPHASIS_SHIFT 22 /* How many wires to use. I guess 3 was too hard */ #define DP_PORT_WIDTH_1 (0 << 19) #define DP_PORT_WIDTH_2 (1 << 19) #define DP_PORT_WIDTH_4 (3 << 19) #define DP_PORT_WIDTH_MASK (7 << 19) /* Mystic DPCD version 1.1 special mode */ #define DP_ENHANCED_FRAMING (1 << 18) /* eDP */ #define DP_PLL_FREQ_270MHZ (0 << 16) #define DP_PLL_FREQ_160MHZ (1 << 16) #define DP_PLL_FREQ_MASK (3 << 16) /** locked once port is enabled */ #define DP_PORT_REVERSAL (1 << 15) /* eDP */ #define DP_PLL_ENABLE (1 << 14) /** sends the clock on lane 15 of the PEG for debug */ #define DP_CLOCK_OUTPUT_ENABLE (1 << 13) #define DP_SCRAMBLING_DISABLE (1 << 12) #define DP_SCRAMBLING_DISABLE_IRONLAKE (1 << 7) /** limit RGB values to avoid confusing TVs */ #define DP_COLOR_RANGE_16_235 (1 << 8) /** Turn on the audio link */ #define DP_AUDIO_OUTPUT_ENABLE (1 << 6) /** vs and hs sync polarity */ #define DP_SYNC_VS_HIGH (1 << 4) #define DP_SYNC_HS_HIGH (1 << 3) /** A fantasy */ #define DP_DETECTED (1 << 2) /** The aux channel provides a way to talk to the * signal sink for DDC etc. Max packet size supported * is 20 bytes in each direction, hence the 5 fixed * data registers */ #define DPA_AUX_CH_CTL 0x64010 #define DPA_AUX_CH_DATA1 0x64014 #define DPA_AUX_CH_DATA2 0x64018 #define DPA_AUX_CH_DATA3 0x6401c #define DPA_AUX_CH_DATA4 0x64020 #define DPA_AUX_CH_DATA5 0x64024 #define DPB_AUX_CH_CTL 0x64110 #define DPB_AUX_CH_DATA1 0x64114 #define DPB_AUX_CH_DATA2 0x64118 #define DPB_AUX_CH_DATA3 0x6411c #define DPB_AUX_CH_DATA4 0x64120 #define DPB_AUX_CH_DATA5 0x64124 #define DPC_AUX_CH_CTL 0x64210 #define DPC_AUX_CH_DATA1 0x64214 #define DPC_AUX_CH_DATA2 0x64218 #define DPC_AUX_CH_DATA3 0x6421c #define DPC_AUX_CH_DATA4 0x64220 #define DPC_AUX_CH_DATA5 0x64224 #define DPD_AUX_CH_CTL 0x64310 #define DPD_AUX_CH_DATA1 0x64314 #define DPD_AUX_CH_DATA2 0x64318 #define DPD_AUX_CH_DATA3 0x6431c #define DPD_AUX_CH_DATA4 0x64320 #define DPD_AUX_CH_DATA5 0x64324 #define DP_AUX_CH_CTL_SEND_BUSY (1 << 31) #define DP_AUX_CH_CTL_DONE (1 << 30) #define DP_AUX_CH_CTL_INTERRUPT (1 << 29) #define DP_AUX_CH_CTL_TIME_OUT_ERROR (1 << 28) #define DP_AUX_CH_CTL_TIME_OUT_400us (0 << 26) #define DP_AUX_CH_CTL_TIME_OUT_600us (1 << 26) #define DP_AUX_CH_CTL_TIME_OUT_800us (2 << 26) #define DP_AUX_CH_CTL_TIME_OUT_1600us (3 << 26) #define DP_AUX_CH_CTL_TIME_OUT_MASK (3 << 26) #define DP_AUX_CH_CTL_RECEIVE_ERROR (1 << 25) #define DP_AUX_CH_CTL_MESSAGE_SIZE_MASK (0x1f << 20) #define DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT 20 #define DP_AUX_CH_CTL_PRECHARGE_2US_MASK (0xf << 16) #define DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT 16 #define DP_AUX_CH_CTL_AUX_AKSV_SELECT (1 << 15) #define DP_AUX_CH_CTL_MANCHESTER_TEST (1 << 14) #define DP_AUX_CH_CTL_SYNC_TEST (1 << 13) #define DP_AUX_CH_CTL_DEGLITCH_TEST (1 << 12) #define DP_AUX_CH_CTL_PRECHARGE_TEST (1 << 11) #define DP_AUX_CH_CTL_BIT_CLOCK_2X_MASK (0x7ff) #define DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT 0 /* * Computing GMCH M and N values for the Display Port link * * GMCH M/N = dot clock * bytes per pixel / ls_clk * # of lanes * * ls_clk (we assume) is the DP link clock (1.62 or 2.7 GHz) * * The GMCH value is used internally * * bytes_per_pixel is the number of bytes coming out of the plane, * which is after the LUTs, so we want the bytes for our color format. * For our current usage, this is always 3, one byte for R, G and B. */ #define _PIPEA_GMCH_DATA_M 0x70050 #define _PIPEB_GMCH_DATA_M 0x71050 /* Transfer unit size for display port - 1, default is 0x3f (for TU size 64) */ #define PIPE_GMCH_DATA_M_TU_SIZE_MASK (0x3f << 25) #define PIPE_GMCH_DATA_M_TU_SIZE_SHIFT 25 #define PIPE_GMCH_DATA_M_MASK (0xffffff) #define _PIPEA_GMCH_DATA_N 0x70054 #define _PIPEB_GMCH_DATA_N 0x71054 #define PIPE_GMCH_DATA_N_MASK (0xffffff) /* * Computing Link M and N values for the Display Port link * * Link M / N = pixel_clock / ls_clk * * (the DP spec calls pixel_clock the 'strm_clk') * * The Link value is transmitted in the Main Stream * Attributes and VB-ID. */ #define _PIPEA_DP_LINK_M 0x70060 #define _PIPEB_DP_LINK_M 0x71060 #define PIPEA_DP_LINK_M_MASK (0xffffff) #define _PIPEA_DP_LINK_N 0x70064 #define _PIPEB_DP_LINK_N 0x71064 #define PIPEA_DP_LINK_N_MASK (0xffffff) #define PIPE_GMCH_DATA_M(pipe) _PIPE(pipe, _PIPEA_GMCH_DATA_M, _PIPEB_GMCH_DATA_M) #define PIPE_GMCH_DATA_N(pipe) _PIPE(pipe, _PIPEA_GMCH_DATA_N, _PIPEB_GMCH_DATA_N) #define PIPE_DP_LINK_M(pipe) _PIPE(pipe, _PIPEA_DP_LINK_M, _PIPEB_DP_LINK_M) #define PIPE_DP_LINK_N(pipe) _PIPE(pipe, _PIPEA_DP_LINK_N, _PIPEB_DP_LINK_N) /* Display & cursor control */ /* Pipe A */ #define _PIPEADSL 0x70000 #define DSL_LINEMASK 0x00000fff #define _PIPEACONF 0x70008 #define PIPECONF_ENABLE (1<<31) #define PIPECONF_DISABLE 0 #define PIPECONF_DOUBLE_WIDE (1<<30) #define I965_PIPECONF_ACTIVE (1<<30) #define PIPECONF_FRAME_START_DELAY_MASK (3<<27) #define PIPECONF_SINGLE_WIDE 0 #define PIPECONF_PIPE_UNLOCKED 0 #define PIPECONF_PIPE_LOCKED (1<<25) #define PIPECONF_PALETTE 0 #define PIPECONF_GAMMA (1<<24) #define PIPECONF_FORCE_BORDER (1<<25) #define PIPECONF_INTERLACE_MASK (7 << 21) /* Note that pre-gen3 does not support interlaced display directly. Panel * fitting must be disabled on pre-ilk for interlaced. */ #define PIPECONF_PROGRESSIVE (0 << 21) #define PIPECONF_INTERLACE_W_SYNC_SHIFT_PANEL (4 << 21) /* gen4 only */ #define PIPECONF_INTERLACE_W_SYNC_SHIFT (5 << 21) /* gen4 only */ #define PIPECONF_INTERLACE_W_FIELD_INDICATION (6 << 21) #define PIPECONF_INTERLACE_FIELD_0_ONLY (7 << 21) /* gen3 only */ /* Ironlake and later have a complete new set of values for interlaced. PFIT * means panel fitter required, PF means progressive fetch, DBL means power * saving pixel doubling. */ #define PIPECONF_PFIT_PF_INTERLACED_ILK (1 << 21) #define PIPECONF_INTERLACED_ILK (3 << 21) #define PIPECONF_INTERLACED_DBL_ILK (4 << 21) /* ilk/snb only */ #define PIPECONF_PFIT_PF_INTERLACED_DBL_ILK (5 << 21) /* ilk/snb only */ #define PIPECONF_CXSR_DOWNCLOCK (1<<16) #define PIPECONF_BPP_MASK (0x000000e0) #define PIPECONF_BPP_8 (0<<5) #define PIPECONF_BPP_10 (1<<5) #define PIPECONF_BPP_6 (2<<5) #define PIPECONF_BPP_12 (3<<5) #define PIPECONF_DITHER_EN (1<<4) #define PIPECONF_DITHER_TYPE_MASK (0x0000000c) #define PIPECONF_DITHER_TYPE_SP (0<<2) #define PIPECONF_DITHER_TYPE_ST1 (1<<2) #define PIPECONF_DITHER_TYPE_ST2 (2<<2) #define PIPECONF_DITHER_TYPE_TEMP (3<<2) #define _PIPEASTAT 0x70024 #define PIPE_FIFO_UNDERRUN_STATUS (1UL<<31) #define PIPE_CRC_ERROR_ENABLE (1UL<<29) #define PIPE_CRC_DONE_ENABLE (1UL<<28) #define PIPE_GMBUS_EVENT_ENABLE (1UL<<27) #define PIPE_HOTPLUG_INTERRUPT_ENABLE (1UL<<26) #define PIPE_VSYNC_INTERRUPT_ENABLE (1UL<<25) #define PIPE_DISPLAY_LINE_COMPARE_ENABLE (1UL<<24) #define PIPE_DPST_EVENT_ENABLE (1UL<<23) #define PIPE_LEGACY_BLC_EVENT_ENABLE (1UL<<22) #define PIPE_ODD_FIELD_INTERRUPT_ENABLE (1UL<<21) #define PIPE_EVEN_FIELD_INTERRUPT_ENABLE (1UL<<20) #define PIPE_HOTPLUG_TV_INTERRUPT_ENABLE (1UL<<18) /* pre-965 */ #define PIPE_START_VBLANK_INTERRUPT_ENABLE (1UL<<18) /* 965 or later */ #define PIPE_VBLANK_INTERRUPT_ENABLE (1UL<<17) #define PIPE_OVERLAY_UPDATED_ENABLE (1UL<<16) #define PIPE_CRC_ERROR_INTERRUPT_STATUS (1UL<<13) #define PIPE_CRC_DONE_INTERRUPT_STATUS (1UL<<12) #define PIPE_GMBUS_INTERRUPT_STATUS (1UL<<11) #define PIPE_HOTPLUG_INTERRUPT_STATUS (1UL<<10) #define PIPE_VSYNC_INTERRUPT_STATUS (1UL<<9) #define PIPE_DISPLAY_LINE_COMPARE_STATUS (1UL<<8) #define PIPE_DPST_EVENT_STATUS (1UL<<7) #define PIPE_LEGACY_BLC_EVENT_STATUS (1UL<<6) #define PIPE_ODD_FIELD_INTERRUPT_STATUS (1UL<<5) #define PIPE_EVEN_FIELD_INTERRUPT_STATUS (1UL<<4) #define PIPE_HOTPLUG_TV_INTERRUPT_STATUS (1UL<<2) /* pre-965 */ #define PIPE_START_VBLANK_INTERRUPT_STATUS (1UL<<2) /* 965 or later */ #define PIPE_VBLANK_INTERRUPT_STATUS (1UL<<1) #define PIPE_OVERLAY_UPDATED_STATUS (1UL<<0) #define PIPE_BPC_MASK (7 << 5) /* Ironlake */ #define PIPE_8BPC (0 << 5) #define PIPE_10BPC (1 << 5) #define PIPE_6BPC (2 << 5) #define PIPE_12BPC (3 << 5) #define PIPESRC(pipe) _PIPE(pipe, _PIPEASRC, _PIPEBSRC) #define PIPECONF(pipe) _PIPE(pipe, _PIPEACONF, _PIPEBCONF) #define PIPEDSL(pipe) _PIPE(pipe, _PIPEADSL, _PIPEBDSL) #define PIPEFRAME(pipe) _PIPE(pipe, _PIPEAFRAMEHIGH, _PIPEBFRAMEHIGH) #define PIPEFRAMEPIXEL(pipe) _PIPE(pipe, _PIPEAFRAMEPIXEL, _PIPEBFRAMEPIXEL) #define PIPESTAT(pipe) _PIPE(pipe, _PIPEASTAT, _PIPEBSTAT) #define DSPARB 0x70030 #define DSPARB_CSTART_MASK (0x7f << 7) #define DSPARB_CSTART_SHIFT 7 #define DSPARB_BSTART_MASK (0x7f) #define DSPARB_BSTART_SHIFT 0 #define DSPARB_BEND_SHIFT 9 /* on 855 */ #define DSPARB_AEND_SHIFT 0 #define DSPFW1 0x70034 #define DSPFW_SR_SHIFT 23 #define DSPFW_SR_MASK (0x1ff<<23) #define DSPFW_CURSORB_SHIFT 16 #define DSPFW_CURSORB_MASK (0x3f<<16) #define DSPFW_PLANEB_SHIFT 8 #define DSPFW_PLANEB_MASK (0x7f<<8) #define DSPFW_PLANEA_MASK (0x7f) #define DSPFW2 0x70038 #define DSPFW_CURSORA_MASK 0x00003f00 #define DSPFW_CURSORA_SHIFT 8 #define DSPFW_PLANEC_MASK (0x7f) #define DSPFW3 0x7003c #define DSPFW_HPLL_SR_EN (1<<31) #define DSPFW_CURSOR_SR_SHIFT 24 #define PINEVIEW_SELF_REFRESH_EN (1<<30) #define DSPFW_CURSOR_SR_MASK (0x3f<<24) #define DSPFW_HPLL_CURSOR_SHIFT 16 #define DSPFW_HPLL_CURSOR_MASK (0x3f<<16) #define DSPFW_HPLL_SR_MASK (0x1ff) /* FIFO watermark sizes etc */ #define G4X_FIFO_LINE_SIZE 64 #define I915_FIFO_LINE_SIZE 64 #define I830_FIFO_LINE_SIZE 32 #define G4X_FIFO_SIZE 127 #define I965_FIFO_SIZE 512 #define I945_FIFO_SIZE 127 #define I915_FIFO_SIZE 95 #define I855GM_FIFO_SIZE 127 /* In cachelines */ #define I830_FIFO_SIZE 95 #define G4X_MAX_WM 0x3f #define I915_MAX_WM 0x3f #define PINEVIEW_DISPLAY_FIFO 512 /* in 64byte unit */ #define PINEVIEW_FIFO_LINE_SIZE 64 #define PINEVIEW_MAX_WM 0x1ff #define PINEVIEW_DFT_WM 0x3f #define PINEVIEW_DFT_HPLLOFF_WM 0 #define PINEVIEW_GUARD_WM 10 #define PINEVIEW_CURSOR_FIFO 64 #define PINEVIEW_CURSOR_MAX_WM 0x3f #define PINEVIEW_CURSOR_DFT_WM 0 #define PINEVIEW_CURSOR_GUARD_WM 5 #define I965_CURSOR_FIFO 64 #define I965_CURSOR_MAX_WM 32 #define I965_CURSOR_DFT_WM 8 /* define the Watermark register on Ironlake */ #define WM0_PIPEA_ILK 0x45100 #define WM0_PIPE_PLANE_MASK (0x7f<<16) #define WM0_PIPE_PLANE_SHIFT 16 #define WM0_PIPE_SPRITE_MASK (0x3f<<8) #define WM0_PIPE_SPRITE_SHIFT 8 #define WM0_PIPE_CURSOR_MASK (0x1f) #define WM0_PIPEB_ILK 0x45104 #define WM0_PIPEC_IVB 0x45200 #define WM1_LP_ILK 0x45108 #define WM1_LP_SR_EN (1<<31) #define WM1_LP_LATENCY_SHIFT 24 #define WM1_LP_LATENCY_MASK (0x7f<<24) #define WM1_LP_FBC_MASK (0xf<<20) #define WM1_LP_FBC_SHIFT 20 #define WM1_LP_SR_MASK (0x1ff<<8) #define WM1_LP_SR_SHIFT 8 #define WM1_LP_CURSOR_MASK (0x3f) #define WM2_LP_ILK 0x4510c #define WM2_LP_EN (1<<31) #define WM3_LP_ILK 0x45110 #define WM3_LP_EN (1<<31) #define WM1S_LP_ILK 0x45120 #define WM2S_LP_IVB 0x45124 #define WM3S_LP_IVB 0x45128 #define WM1S_LP_EN (1<<31) /* Memory latency timer register */ #define MLTR_ILK 0x11222 #define MLTR_WM1_SHIFT 0 #define MLTR_WM2_SHIFT 8 /* the unit of memory self-refresh latency time is 0.5us */ #define ILK_SRLT_MASK 0x3f #define ILK_LATENCY(shift) (I915_READ(MLTR_ILK) >> (shift) & ILK_SRLT_MASK) #define ILK_READ_WM1_LATENCY() ILK_LATENCY(MLTR_WM1_SHIFT) #define ILK_READ_WM2_LATENCY() ILK_LATENCY(MLTR_WM2_SHIFT) /* define the fifo size on Ironlake */ #define ILK_DISPLAY_FIFO 128 #define ILK_DISPLAY_MAXWM 64 #define ILK_DISPLAY_DFTWM 8 #define ILK_CURSOR_FIFO 32 #define ILK_CURSOR_MAXWM 16 #define ILK_CURSOR_DFTWM 8 #define ILK_DISPLAY_SR_FIFO 512 #define ILK_DISPLAY_MAX_SRWM 0x1ff #define ILK_DISPLAY_DFT_SRWM 0x3f #define ILK_CURSOR_SR_FIFO 64 #define ILK_CURSOR_MAX_SRWM 0x3f #define ILK_CURSOR_DFT_SRWM 8 #define ILK_FIFO_LINE_SIZE 64 /* define the WM info on Sandybridge */ #define SNB_DISPLAY_FIFO 128 #define SNB_DISPLAY_MAXWM 0x7f /* bit 16:22 */ #define SNB_DISPLAY_DFTWM 8 #define SNB_CURSOR_FIFO 32 #define SNB_CURSOR_MAXWM 0x1f /* bit 4:0 */ #define SNB_CURSOR_DFTWM 8 #define SNB_DISPLAY_SR_FIFO 512 #define SNB_DISPLAY_MAX_SRWM 0x1ff /* bit 16:8 */ #define SNB_DISPLAY_DFT_SRWM 0x3f #define SNB_CURSOR_SR_FIFO 64 #define SNB_CURSOR_MAX_SRWM 0x3f /* bit 5:0 */ #define SNB_CURSOR_DFT_SRWM 8 #define SNB_FBC_MAX_SRWM 0xf /* bit 23:20 */ #define SNB_FIFO_LINE_SIZE 64 /* the address where we get all kinds of latency value */ #define SSKPD 0x5d10 #define SSKPD_WM_MASK 0x3f #define SSKPD_WM0_SHIFT 0 #define SSKPD_WM1_SHIFT 8 #define SSKPD_WM2_SHIFT 16 #define SSKPD_WM3_SHIFT 24 #define SNB_LATENCY(shift) (I915_READ(MCHBAR_MIRROR_BASE_SNB + SSKPD) >> (shift) & SSKPD_WM_MASK) #define SNB_READ_WM0_LATENCY() SNB_LATENCY(SSKPD_WM0_SHIFT) #define SNB_READ_WM1_LATENCY() SNB_LATENCY(SSKPD_WM1_SHIFT) #define SNB_READ_WM2_LATENCY() SNB_LATENCY(SSKPD_WM2_SHIFT) #define SNB_READ_WM3_LATENCY() SNB_LATENCY(SSKPD_WM3_SHIFT) /* * The two pipe frame counter registers are not synchronized, so * reading a stable value is somewhat tricky. The following code * should work: * * do { * high1 = ((INREG(PIPEAFRAMEHIGH) & PIPE_FRAME_HIGH_MASK) >> * PIPE_FRAME_HIGH_SHIFT; * low1 = ((INREG(PIPEAFRAMEPIXEL) & PIPE_FRAME_LOW_MASK) >> * PIPE_FRAME_LOW_SHIFT); * high2 = ((INREG(PIPEAFRAMEHIGH) & PIPE_FRAME_HIGH_MASK) >> * PIPE_FRAME_HIGH_SHIFT); * } while (high1 != high2); * frame = (high1 << 8) | low1; */ #define _PIPEAFRAMEHIGH 0x70040 #define PIPE_FRAME_HIGH_MASK 0x0000ffff #define PIPE_FRAME_HIGH_SHIFT 0 #define _PIPEAFRAMEPIXEL 0x70044 #define PIPE_FRAME_LOW_MASK 0xff000000 #define PIPE_FRAME_LOW_SHIFT 24 #define PIPE_PIXEL_MASK 0x00ffffff #define PIPE_PIXEL_SHIFT 0 /* GM45+ just has to be different */ #define _PIPEA_FRMCOUNT_GM45 0x70040 #define _PIPEA_FLIPCOUNT_GM45 0x70044 #define PIPE_FRMCOUNT_GM45(pipe) _PIPE(pipe, _PIPEA_FRMCOUNT_GM45, _PIPEB_FRMCOUNT_GM45) /* Cursor A & B regs */ #define _CURACNTR 0x70080 /* Old style CUR*CNTR flags (desktop 8xx) */ #define CURSOR_ENABLE 0x80000000 #define CURSOR_GAMMA_ENABLE 0x40000000 #define CURSOR_STRIDE_MASK 0x30000000 #define CURSOR_FORMAT_SHIFT 24 #define CURSOR_FORMAT_MASK (0x07 << CURSOR_FORMAT_SHIFT) #define CURSOR_FORMAT_2C (0x00 << CURSOR_FORMAT_SHIFT) #define CURSOR_FORMAT_3C (0x01 << CURSOR_FORMAT_SHIFT) #define CURSOR_FORMAT_4C (0x02 << CURSOR_FORMAT_SHIFT) #define CURSOR_FORMAT_ARGB (0x04 << CURSOR_FORMAT_SHIFT) #define CURSOR_FORMAT_XRGB (0x05 << CURSOR_FORMAT_SHIFT) /* New style CUR*CNTR flags */ #define CURSOR_MODE 0x27 #define CURSOR_MODE_DISABLE 0x00 #define CURSOR_MODE_64_32B_AX 0x07 #define CURSOR_MODE_64_ARGB_AX ((1 << 5) | CURSOR_MODE_64_32B_AX) #define MCURSOR_PIPE_SELECT (1 << 28) #define MCURSOR_PIPE_A 0x00 #define MCURSOR_PIPE_B (1 << 28) #define MCURSOR_GAMMA_ENABLE (1 << 26) #define _CURABASE 0x70084 #define _CURAPOS 0x70088 #define CURSOR_POS_MASK 0x007FF #define CURSOR_POS_SIGN 0x8000 #define CURSOR_X_SHIFT 0 #define CURSOR_Y_SHIFT 16 #define CURSIZE 0x700a0 #define _CURBCNTR 0x700c0 #define _CURBBASE 0x700c4 #define _CURBPOS 0x700c8 #define _CURBCNTR_IVB 0x71080 #define _CURBBASE_IVB 0x71084 #define _CURBPOS_IVB 0x71088 #define CURCNTR(pipe) _PIPE(pipe, _CURACNTR, _CURBCNTR) #define CURBASE(pipe) _PIPE(pipe, _CURABASE, _CURBBASE) #define CURPOS(pipe) _PIPE(pipe, _CURAPOS, _CURBPOS) #define CURCNTR_IVB(pipe) _PIPE(pipe, _CURACNTR, _CURBCNTR_IVB) #define CURBASE_IVB(pipe) _PIPE(pipe, _CURABASE, _CURBBASE_IVB) #define CURPOS_IVB(pipe) _PIPE(pipe, _CURAPOS, _CURBPOS_IVB) /* Display A control */ #define _DSPACNTR 0x70180 #define DISPLAY_PLANE_ENABLE (1<<31) #define DISPLAY_PLANE_DISABLE 0 #define DISPPLANE_GAMMA_ENABLE (1<<30) #define DISPPLANE_GAMMA_DISABLE 0 #define DISPPLANE_PIXFORMAT_MASK (0xf<<26) #define DISPPLANE_8BPP (0x2<<26) #define DISPPLANE_15_16BPP (0x4<<26) #define DISPPLANE_16BPP (0x5<<26) #define DISPPLANE_32BPP_NO_ALPHA (0x6<<26) #define DISPPLANE_32BPP (0x7<<26) #define DISPPLANE_32BPP_30BIT_NO_ALPHA (0xa<<26) #define DISPPLANE_STEREO_ENABLE (1<<25) #define DISPPLANE_STEREO_DISABLE 0 #define DISPPLANE_SEL_PIPE_SHIFT 24 #define DISPPLANE_SEL_PIPE_MASK (3< * Zou Nan hai * Xiang Hai hao * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include /* * 965+ support PIPE_CONTROL commands, which provide finer grained control * over cache flushing. */ struct pipe_control { struct drm_i915_gem_object *obj; volatile u32 *cpu_page; u32 gtt_offset; }; void i915_trace_irq_get(struct intel_ring_buffer *ring, uint32_t seqno) { if (ring->trace_irq_seqno == 0) { mtx_lock(&ring->irq_lock); if (ring->irq_get(ring)) ring->trace_irq_seqno = seqno; mtx_unlock(&ring->irq_lock); } } static inline int ring_space(struct intel_ring_buffer *ring) { int space = (ring->head & HEAD_ADDR) - (ring->tail + 8); if (space < 0) space += ring->size; return space; } static int render_ring_flush(struct intel_ring_buffer *ring, uint32_t invalidate_domains, uint32_t flush_domains) { struct drm_device *dev = ring->dev; uint32_t cmd; int ret; /* * read/write caches: * * I915_GEM_DOMAIN_RENDER is always invalidated, but is * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is * also flushed at 2d versus 3d pipeline switches. * * read-only caches: * * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if * MI_READ_FLUSH is set, and is always flushed on 965. * * I915_GEM_DOMAIN_COMMAND may not exist? * * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is * invalidated when MI_EXE_FLUSH is set. * * I915_GEM_DOMAIN_VERTEX, which exists on 965, is * invalidated with every MI_FLUSH. * * TLBs: * * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER * are flushed at any MI_FLUSH. */ cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) cmd &= ~MI_NO_WRITE_FLUSH; if (INTEL_INFO(dev)->gen < 4) { /* * On the 965, the sampler cache always gets flushed * and this bit is reserved. */ if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER) cmd |= MI_READ_FLUSH; } if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) cmd |= MI_EXE_FLUSH; if (invalidate_domains & I915_GEM_DOMAIN_COMMAND && (IS_G4X(dev) || IS_GEN5(dev))) cmd |= MI_INVALIDATE_ISP; ret = intel_ring_begin(ring, 2); if (ret) return ret; intel_ring_emit(ring, cmd); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } /** * Emits a PIPE_CONTROL with a non-zero post-sync operation, for * implementing two workarounds on gen6. From section 1.4.7.1 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: * * [DevSNB-C+{W/A}] Before any depth stall flush (including those * produced by non-pipelined state commands), software needs to first * send a PIPE_CONTROL with no bits set except Post-Sync Operation != * 0. * * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. * * And the workaround for these two requires this workaround first: * * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent * BEFORE the pipe-control with a post-sync op and no write-cache * flushes. * * And this last workaround is tricky because of the requirements on * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM * volume 2 part 1: * * "1 of the following must also be set: * - Render Target Cache Flush Enable ([12] of DW1) * - Depth Cache Flush Enable ([0] of DW1) * - Stall at Pixel Scoreboard ([1] of DW1) * - Depth Stall ([13] of DW1) * - Post-Sync Operation ([13] of DW1) * - Notify Enable ([8] of DW1)" * * The cache flushes require the workaround flush that triggered this * one, so we can't use it. Depth stall would trigger the same. * Post-sync nonzero is what triggered this second workaround, so we * can't use that one either. Notify enable is IRQs, which aren't * really our business. That leaves only stall at scoreboard. */ static int intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring) { struct pipe_control *pc = ring->private; u32 scratch_addr = pc->gtt_offset + 128; int ret; ret = intel_ring_begin(ring, 6); if (ret) return ret; intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD); intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ intel_ring_emit(ring, 0); /* low dword */ intel_ring_emit(ring, 0); /* high dword */ intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); ret = intel_ring_begin(ring, 6); if (ret) return ret; intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE); intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ intel_ring_emit(ring, 0); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static int gen6_render_ring_flush(struct intel_ring_buffer *ring, u32 invalidate_domains, u32 flush_domains) { u32 flags = 0; struct pipe_control *pc = ring->private; u32 scratch_addr = pc->gtt_offset + 128; int ret; /* Force SNB workarounds for PIPE_CONTROL flushes */ intel_emit_post_sync_nonzero_flush(ring); /* Just flush everything. Experiments have shown that reducing the * number of bits based on the write domains has little performance * impact. */ flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; ret = intel_ring_begin(ring, 6); if (ret) return ret; intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); intel_ring_emit(ring, flags); intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); intel_ring_emit(ring, 0); /* lower dword */ intel_ring_emit(ring, 0); /* uppwer dword */ intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static void ring_write_tail(struct intel_ring_buffer *ring, uint32_t value) { drm_i915_private_t *dev_priv = ring->dev->dev_private; I915_WRITE_TAIL(ring, value); } u32 intel_ring_get_active_head(struct intel_ring_buffer *ring) { drm_i915_private_t *dev_priv = ring->dev->dev_private; uint32_t acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ? RING_ACTHD(ring->mmio_base) : ACTHD; return I915_READ(acthd_reg); } static int init_ring_common(struct intel_ring_buffer *ring) { drm_i915_private_t *dev_priv = ring->dev->dev_private; struct drm_i915_gem_object *obj = ring->obj; uint32_t head; /* Stop the ring if it's running. */ I915_WRITE_CTL(ring, 0); I915_WRITE_HEAD(ring, 0); ring->write_tail(ring, 0); /* Initialize the ring. */ I915_WRITE_START(ring, obj->gtt_offset); head = I915_READ_HEAD(ring) & HEAD_ADDR; /* G45 ring initialization fails to reset head to zero */ if (head != 0) { DRM_DEBUG("%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", ring->name, I915_READ_CTL(ring), I915_READ_HEAD(ring), I915_READ_TAIL(ring), I915_READ_START(ring)); I915_WRITE_HEAD(ring, 0); if (I915_READ_HEAD(ring) & HEAD_ADDR) { DRM_ERROR("failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ring->name, I915_READ_CTL(ring), I915_READ_HEAD(ring), I915_READ_TAIL(ring), I915_READ_START(ring)); } } I915_WRITE_CTL(ring, ((ring->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID); /* If the head is still not zero, the ring is dead */ if (_intel_wait_for(ring->dev, (I915_READ_CTL(ring) & RING_VALID) != 0 && I915_READ_START(ring) == obj->gtt_offset && (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50, 1, "915rii")) { DRM_ERROR("%s initialization failed " "ctl %08x head %08x tail %08x start %08x\n", ring->name, I915_READ_CTL(ring), I915_READ_HEAD(ring), I915_READ_TAIL(ring), I915_READ_START(ring)); return -EIO; } if (!drm_core_check_feature(ring->dev, DRIVER_MODESET)) i915_kernel_lost_context(ring->dev); else { ring->head = I915_READ_HEAD(ring); ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; ring->space = ring_space(ring); } return 0; } static int init_pipe_control(struct intel_ring_buffer *ring) { struct pipe_control *pc; struct drm_i915_gem_object *obj; int ret; if (ring->private) return 0; pc = malloc(sizeof(*pc), DRM_I915_GEM, M_WAITOK); if (!pc) return -ENOMEM; obj = i915_gem_alloc_object(ring->dev, 4096); if (obj == NULL) { DRM_ERROR("Failed to allocate seqno page\n"); ret = -ENOMEM; goto err; } i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); ret = i915_gem_object_pin(obj, 4096, true); if (ret) goto err_unref; pc->gtt_offset = obj->gtt_offset; pc->cpu_page = (uint32_t *)kva_alloc(PAGE_SIZE); if (pc->cpu_page == NULL) goto err_unpin; pmap_qenter((uintptr_t)pc->cpu_page, &obj->pages[0], 1); pmap_invalidate_cache_range((vm_offset_t)pc->cpu_page, (vm_offset_t)pc->cpu_page + PAGE_SIZE); pc->obj = obj; ring->private = pc; return 0; err_unpin: i915_gem_object_unpin(obj); err_unref: drm_gem_object_unreference(&obj->base); err: free(pc, DRM_I915_GEM); return ret; } static void cleanup_pipe_control(struct intel_ring_buffer *ring) { struct pipe_control *pc = ring->private; struct drm_i915_gem_object *obj; if (!ring->private) return; obj = pc->obj; pmap_qremove((vm_offset_t)pc->cpu_page, 1); kva_free((uintptr_t)pc->cpu_page, PAGE_SIZE); i915_gem_object_unpin(obj); drm_gem_object_unreference(&obj->base); free(pc, DRM_I915_GEM); ring->private = NULL; } static int init_render_ring(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; int ret = init_ring_common(ring); if (INTEL_INFO(dev)->gen > 3) { int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH; I915_WRITE(MI_MODE, mode); if (IS_GEN7(dev)) I915_WRITE(GFX_MODE_GEN7, GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) | GFX_MODE_ENABLE(GFX_REPLAY_MODE)); } if (INTEL_INFO(dev)->gen >= 5) { ret = init_pipe_control(ring); if (ret) return ret; } if (IS_GEN6(dev)) { /* From the Sandybridge PRM, volume 1 part 3, page 24: * "If this bit is set, STCunit will have LRA as replacement * policy. [...] This bit must be reset. LRA replacement * policy is not supported." */ I915_WRITE(CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB << CM0_MASK_SHIFT); + + /* This is not explicitly set for GEN6, so read the register. + * see intel_ring_mi_set_context() for why we care. + * TODO: consider explicitly setting the bit for GEN5 + */ + ring->itlb_before_ctx_switch = + !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS); } if (INTEL_INFO(dev)->gen >= 6) { I915_WRITE(INSTPM, INSTPM_FORCE_ORDERING << 16 | INSTPM_FORCE_ORDERING); } return ret; } static void render_ring_cleanup(struct intel_ring_buffer *ring) { if (!ring->private) return; cleanup_pipe_control(ring); } static void update_mboxes(struct intel_ring_buffer *ring, u32 seqno, u32 mmio_offset) { intel_ring_emit(ring, MI_SEMAPHORE_MBOX | MI_SEMAPHORE_GLOBAL_GTT | MI_SEMAPHORE_REGISTER | MI_SEMAPHORE_UPDATE); intel_ring_emit(ring, seqno); intel_ring_emit(ring, mmio_offset); } /** * gen6_add_request - Update the semaphore mailbox registers * * @ring - ring that is adding a request * @seqno - return seqno stuck into the ring * * Update the mailbox registers in the *other* rings with the current seqno. * This acts like a signal in the canonical semaphore. */ static int gen6_add_request(struct intel_ring_buffer *ring, u32 *seqno) { u32 mbox1_reg; u32 mbox2_reg; int ret; ret = intel_ring_begin(ring, 10); if (ret) return ret; mbox1_reg = ring->signal_mbox[0]; mbox2_reg = ring->signal_mbox[1]; *seqno = i915_gem_next_request_seqno(ring); update_mboxes(ring, *seqno, mbox1_reg); update_mboxes(ring, *seqno, mbox2_reg); intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); intel_ring_emit(ring, *seqno); intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_advance(ring); return 0; } /** * intel_ring_sync - sync the waiter to the signaller on seqno * * @waiter - ring that is waiting * @signaller - ring which has, or will signal * @seqno - seqno which the waiter will block on */ static int intel_ring_sync(struct intel_ring_buffer *waiter, struct intel_ring_buffer *signaller, int ring, u32 seqno) { int ret; u32 dw1 = MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER; ret = intel_ring_begin(waiter, 4); if (ret) return ret; intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]); intel_ring_emit(waiter, seqno); intel_ring_emit(waiter, 0); intel_ring_emit(waiter, MI_NOOP); intel_ring_advance(waiter); return 0; } int render_ring_sync_to(struct intel_ring_buffer *waiter, struct intel_ring_buffer *signaller, u32 seqno); int gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter, struct intel_ring_buffer *signaller, u32 seqno); int gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter, struct intel_ring_buffer *signaller, u32 seqno); /* VCS->RCS (RVSYNC) or BCS->RCS (RBSYNC) */ int render_ring_sync_to(struct intel_ring_buffer *waiter, struct intel_ring_buffer *signaller, u32 seqno) { KASSERT(signaller->semaphore_register[RCS] != MI_SEMAPHORE_SYNC_INVALID, ("valid RCS semaphore")); return intel_ring_sync(waiter, signaller, RCS, seqno); } /* RCS->VCS (VRSYNC) or BCS->VCS (VBSYNC) */ int gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter, struct intel_ring_buffer *signaller, u32 seqno) { KASSERT(signaller->semaphore_register[VCS] != MI_SEMAPHORE_SYNC_INVALID, ("Valid VCS semaphore")); return intel_ring_sync(waiter, signaller, VCS, seqno); } /* RCS->BCS (BRSYNC) or VCS->BCS (BVSYNC) */ int gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter, struct intel_ring_buffer *signaller, u32 seqno) { KASSERT(signaller->semaphore_register[BCS] != MI_SEMAPHORE_SYNC_INVALID, ("Valid BCS semaphore")); return intel_ring_sync(waiter, signaller, BCS, seqno); } #define PIPE_CONTROL_FLUSH(ring__, addr__) \ do { \ intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \ PIPE_CONTROL_DEPTH_STALL); \ intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \ intel_ring_emit(ring__, 0); \ intel_ring_emit(ring__, 0); \ } while (0) static int pc_render_add_request(struct intel_ring_buffer *ring, uint32_t *result) { u32 seqno = i915_gem_next_request_seqno(ring); struct pipe_control *pc = ring->private; u32 scratch_addr = pc->gtt_offset + 128; int ret; /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently * incoherent with writes to memory, i.e. completely fubar, * so we need to use PIPE_NOTIFY instead. * * However, we also need to workaround the qword write * incoherence by flushing the 6 PIPE_NOTIFY buffers out to * memory before requesting an interrupt. */ ret = intel_ring_begin(ring, 32); if (ret) return ret; intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_WRITE_FLUSH | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT); intel_ring_emit(ring, seqno); intel_ring_emit(ring, 0); PIPE_CONTROL_FLUSH(ring, scratch_addr); scratch_addr += 128; /* write to separate cachelines */ PIPE_CONTROL_FLUSH(ring, scratch_addr); scratch_addr += 128; PIPE_CONTROL_FLUSH(ring, scratch_addr); scratch_addr += 128; PIPE_CONTROL_FLUSH(ring, scratch_addr); scratch_addr += 128; PIPE_CONTROL_FLUSH(ring, scratch_addr); scratch_addr += 128; PIPE_CONTROL_FLUSH(ring, scratch_addr); intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_WRITE_FLUSH | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | PIPE_CONTROL_NOTIFY); intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT); intel_ring_emit(ring, seqno); intel_ring_emit(ring, 0); intel_ring_advance(ring); *result = seqno; return 0; } static int render_ring_add_request(struct intel_ring_buffer *ring, uint32_t *result) { u32 seqno = i915_gem_next_request_seqno(ring); int ret; ret = intel_ring_begin(ring, 4); if (ret) return ret; intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); intel_ring_emit(ring, seqno); intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_advance(ring); *result = seqno; return 0; } static u32 gen6_ring_get_seqno(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; /* Workaround to force correct ordering between irq and seqno writes on * ivb (and maybe also on snb) by reading from a CS register (like * ACTHD) before reading the status page. */ if (/* IS_GEN6(dev) || */IS_GEN7(dev)) intel_ring_get_active_head(ring); return intel_read_status_page(ring, I915_GEM_HWS_INDEX); } static uint32_t ring_get_seqno(struct intel_ring_buffer *ring) { if (ring->status_page.page_addr == NULL) return (-1); return intel_read_status_page(ring, I915_GEM_HWS_INDEX); } static uint32_t pc_render_get_seqno(struct intel_ring_buffer *ring) { struct pipe_control *pc = ring->private; if (pc != NULL) return pc->cpu_page[0]; else return (-1); } static void ironlake_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask) { dev_priv->gt_irq_mask &= ~mask; I915_WRITE(GTIMR, dev_priv->gt_irq_mask); POSTING_READ(GTIMR); } static void ironlake_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask) { dev_priv->gt_irq_mask |= mask; I915_WRITE(GTIMR, dev_priv->gt_irq_mask); POSTING_READ(GTIMR); } static void i915_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask) { dev_priv->irq_mask &= ~mask; I915_WRITE(IMR, dev_priv->irq_mask); POSTING_READ(IMR); } static void i915_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask) { dev_priv->irq_mask |= mask; I915_WRITE(IMR, dev_priv->irq_mask); POSTING_READ(IMR); } static bool render_ring_get_irq(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; if (!dev->irq_enabled) return false; mtx_assert(&ring->irq_lock, MA_OWNED); if (ring->irq_refcount++ == 0) { if (HAS_PCH_SPLIT(dev)) ironlake_enable_irq(dev_priv, GT_PIPE_NOTIFY | GT_USER_INTERRUPT); else i915_enable_irq(dev_priv, I915_USER_INTERRUPT); } return true; } static void render_ring_put_irq(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; mtx_assert(&ring->irq_lock, MA_OWNED); if (--ring->irq_refcount == 0) { if (HAS_PCH_SPLIT(dev)) ironlake_disable_irq(dev_priv, GT_USER_INTERRUPT | GT_PIPE_NOTIFY); else i915_disable_irq(dev_priv, I915_USER_INTERRUPT); } } void intel_ring_setup_status_page(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; uint32_t mmio = 0; /* The ring status page addresses are no longer next to the rest of * the ring registers as of gen7. */ if (IS_GEN7(dev)) { switch (ring->id) { case RCS: mmio = RENDER_HWS_PGA_GEN7; break; case BCS: mmio = BLT_HWS_PGA_GEN7; break; case VCS: mmio = BSD_HWS_PGA_GEN7; break; } } else if (IS_GEN6(dev)) { mmio = RING_HWS_PGA_GEN6(ring->mmio_base); } else { mmio = RING_HWS_PGA(ring->mmio_base); } I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); POSTING_READ(mmio); } static int bsd_ring_flush(struct intel_ring_buffer *ring, uint32_t invalidate_domains, uint32_t flush_domains) { int ret; ret = intel_ring_begin(ring, 2); if (ret) return ret; intel_ring_emit(ring, MI_FLUSH); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static int ring_add_request(struct intel_ring_buffer *ring, uint32_t *result) { uint32_t seqno; int ret; ret = intel_ring_begin(ring, 4); if (ret) return ret; seqno = i915_gem_next_request_seqno(ring); intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); intel_ring_emit(ring, seqno); intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_advance(ring); *result = seqno; return 0; } static bool gen6_ring_get_irq(struct intel_ring_buffer *ring, uint32_t gflag, uint32_t rflag) { struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; if (!dev->irq_enabled) return false; gen6_gt_force_wake_get(dev_priv); mtx_assert(&ring->irq_lock, MA_OWNED); if (ring->irq_refcount++ == 0) { ring->irq_mask &= ~rflag; I915_WRITE_IMR(ring, ring->irq_mask); ironlake_enable_irq(dev_priv, gflag); } return true; } static void gen6_ring_put_irq(struct intel_ring_buffer *ring, uint32_t gflag, uint32_t rflag) { struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; mtx_assert(&ring->irq_lock, MA_OWNED); if (--ring->irq_refcount == 0) { ring->irq_mask |= rflag; I915_WRITE_IMR(ring, ring->irq_mask); ironlake_disable_irq(dev_priv, gflag); } gen6_gt_force_wake_put(dev_priv); } static bool bsd_ring_get_irq(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; if (!dev->irq_enabled) return false; mtx_assert(&ring->irq_lock, MA_OWNED); if (ring->irq_refcount++ == 0) { if (IS_G4X(dev)) i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT); else ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT); } return true; } static void bsd_ring_put_irq(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; mtx_assert(&ring->irq_lock, MA_OWNED); if (--ring->irq_refcount == 0) { if (IS_G4X(dev)) i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT); else ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT); } } static int ring_dispatch_execbuffer(struct intel_ring_buffer *ring, uint32_t offset, uint32_t length) { int ret; ret = intel_ring_begin(ring, 2); if (ret) return ret; intel_ring_emit(ring, MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965); intel_ring_emit(ring, offset); intel_ring_advance(ring); return 0; } static int render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, uint32_t offset, uint32_t len) { struct drm_device *dev = ring->dev; int ret; if (IS_I830(dev) || IS_845G(dev)) { ret = intel_ring_begin(ring, 4); if (ret) return ret; intel_ring_emit(ring, MI_BATCH_BUFFER); intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE); intel_ring_emit(ring, offset + len - 8); intel_ring_emit(ring, 0); } else { ret = intel_ring_begin(ring, 2); if (ret) return ret; if (INTEL_INFO(dev)->gen >= 4) { intel_ring_emit(ring, MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965); intel_ring_emit(ring, offset); } else { intel_ring_emit(ring, MI_BATCH_BUFFER_START | (2 << 6)); intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE); } } intel_ring_advance(ring); return 0; } static void cleanup_status_page(struct intel_ring_buffer *ring) { drm_i915_private_t *dev_priv = ring->dev->dev_private; struct drm_i915_gem_object *obj; obj = ring->status_page.obj; if (obj == NULL) return; pmap_qremove((vm_offset_t)ring->status_page.page_addr, 1); kva_free((vm_offset_t)ring->status_page.page_addr, PAGE_SIZE); i915_gem_object_unpin(obj); drm_gem_object_unreference(&obj->base); ring->status_page.obj = NULL; memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); } static int init_status_page(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; int ret; obj = i915_gem_alloc_object(dev, 4096); if (obj == NULL) { DRM_ERROR("Failed to allocate status page\n"); ret = -ENOMEM; goto err; } i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); ret = i915_gem_object_pin(obj, 4096, true); if (ret != 0) { goto err_unref; } ring->status_page.gfx_addr = obj->gtt_offset; ring->status_page.page_addr = (void *)kva_alloc(PAGE_SIZE); if (ring->status_page.page_addr == NULL) { memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); goto err_unpin; } pmap_qenter((vm_offset_t)ring->status_page.page_addr, &obj->pages[0], 1); pmap_invalidate_cache_range((vm_offset_t)ring->status_page.page_addr, (vm_offset_t)ring->status_page.page_addr + PAGE_SIZE); ring->status_page.obj = obj; memset(ring->status_page.page_addr, 0, PAGE_SIZE); intel_ring_setup_status_page(ring); DRM_DEBUG("i915: init_status_page %s hws offset: 0x%08x\n", ring->name, ring->status_page.gfx_addr); return 0; err_unpin: i915_gem_object_unpin(obj); err_unref: drm_gem_object_unreference(&obj->base); err: return ret; } static int intel_init_ring_buffer(struct drm_device *dev, struct intel_ring_buffer *ring) { struct drm_i915_gem_object *obj; int ret; ring->dev = dev; INIT_LIST_HEAD(&ring->active_list); INIT_LIST_HEAD(&ring->request_list); INIT_LIST_HEAD(&ring->gpu_write_list); mtx_init(&ring->irq_lock, "ringb", NULL, MTX_DEF); ring->irq_mask = ~0; if (I915_NEED_GFX_HWS(dev)) { ret = init_status_page(ring); if (ret) return ret; } obj = i915_gem_alloc_object(dev, ring->size); if (obj == NULL) { DRM_ERROR("Failed to allocate ringbuffer\n"); ret = -ENOMEM; goto err_hws; } ring->obj = obj; ret = i915_gem_object_pin(obj, PAGE_SIZE, true); if (ret) goto err_unref; ring->map.size = ring->size; ring->map.offset = dev->agp->base + obj->gtt_offset; ring->map.type = 0; ring->map.flags = 0; ring->map.mtrr = 0; drm_core_ioremap_wc(&ring->map, dev); if (ring->map.virtual == NULL) { DRM_ERROR("Failed to map ringbuffer.\n"); ret = -EINVAL; goto err_unpin; } ring->virtual_start = ring->map.virtual; ret = ring->init(ring); if (ret) goto err_unmap; /* Workaround an erratum on the i830 which causes a hang if * the TAIL pointer points to within the last 2 cachelines * of the buffer. */ ring->effective_size = ring->size; if (IS_I830(ring->dev) || IS_845G(ring->dev)) ring->effective_size -= 128; return 0; err_unmap: drm_core_ioremapfree(&ring->map, dev); err_unpin: i915_gem_object_unpin(obj); err_unref: drm_gem_object_unreference(&obj->base); ring->obj = NULL; err_hws: cleanup_status_page(ring); return ret; } void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring) { struct drm_i915_private *dev_priv; int ret; if (ring->obj == NULL) return; /* Disable the ring buffer. The ring must be idle at this point */ dev_priv = ring->dev->dev_private; ret = intel_wait_ring_idle(ring); I915_WRITE_CTL(ring, 0); drm_core_ioremapfree(&ring->map, ring->dev); i915_gem_object_unpin(ring->obj); drm_gem_object_unreference(&ring->obj->base); ring->obj = NULL; if (ring->cleanup) ring->cleanup(ring); cleanup_status_page(ring); } static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring) { unsigned int *virt; int rem = ring->size - ring->tail; if (ring->space < rem) { int ret = intel_wait_ring_buffer(ring, rem); if (ret) return ret; } virt = (unsigned int *)((char *)ring->virtual_start + ring->tail); rem /= 8; while (rem--) { *virt++ = MI_NOOP; *virt++ = MI_NOOP; } ring->tail = 0; ring->space = ring_space(ring); return 0; } static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno) { struct drm_i915_private *dev_priv = ring->dev->dev_private; bool was_interruptible; int ret; /* XXX As we have not yet audited all the paths to check that * they are ready for ERESTARTSYS from intel_ring_begin, do not * allow us to be interruptible by a signal. */ was_interruptible = dev_priv->mm.interruptible; dev_priv->mm.interruptible = false; ret = i915_wait_request(ring, seqno, true); dev_priv->mm.interruptible = was_interruptible; return ret; } static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n) { struct drm_i915_gem_request *request; u32 seqno = 0; int ret; i915_gem_retire_requests_ring(ring); if (ring->last_retired_head != -1) { ring->head = ring->last_retired_head; ring->last_retired_head = -1; ring->space = ring_space(ring); if (ring->space >= n) return 0; } list_for_each_entry(request, &ring->request_list, list) { int space; if (request->tail == -1) continue; space = request->tail - (ring->tail + 8); if (space < 0) space += ring->size; if (space >= n) { seqno = request->seqno; break; } /* Consume this request in case we need more space than * is available and so need to prevent a race between * updating last_retired_head and direct reads of * I915_RING_HEAD. It also provides a nice sanity check. */ request->tail = -1; } if (seqno == 0) return -ENOSPC; ret = intel_ring_wait_seqno(ring, seqno); if (ret) return ret; if (ring->last_retired_head == -1) return -ENOSPC; ring->head = ring->last_retired_head; ring->last_retired_head = -1; ring->space = ring_space(ring); if (ring->space < n) return -ENOSPC; return 0; } int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; int end; int ret; ret = intel_ring_wait_request(ring, n); if (ret != -ENOSPC) return ret; CTR1(KTR_DRM, "ring_wait_begin %s", ring->name); if (drm_core_check_feature(dev, DRIVER_GEM)) /* With GEM the hangcheck timer should kick us out of the loop, * leaving it early runs the risk of corrupting GEM state (due * to running on almost untested codepaths). But on resume * timers don't work yet, so prevent a complete hang in that * case by choosing an insanely large timeout. */ end = ticks + hz * 60; else end = ticks + hz * 3; do { ring->head = I915_READ_HEAD(ring); ring->space = ring_space(ring); if (ring->space >= n) { CTR1(KTR_DRM, "ring_wait_end %s", ring->name); return 0; } #if 0 if (dev->primary->master) { struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; if (master_priv->sarea_priv) master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; } #else if (dev_priv->sarea_priv) dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; #endif pause("915rng", 1); if (atomic_load_acq_32(&dev_priv->mm.wedged) != 0) { CTR1(KTR_DRM, "ring_wait_end %s wedged", ring->name); return -EAGAIN; } } while (!time_after(ticks, end)); CTR1(KTR_DRM, "ring_wait_end %s busy", ring->name); return -EBUSY; } int intel_ring_begin(struct intel_ring_buffer *ring, int num_dwords) { struct drm_i915_private *dev_priv = ring->dev->dev_private; int n = 4*num_dwords; int ret; if (atomic_load_acq_int(&dev_priv->mm.wedged)) return -EIO; if (ring->tail + n > ring->effective_size) { ret = intel_wrap_ring_buffer(ring); if (ret != 0) return ret; } if (ring->space < n) { ret = intel_wait_ring_buffer(ring, n); if (ret != 0) return ret; } ring->space -= n; return 0; } void intel_ring_advance(struct intel_ring_buffer *ring) { ring->tail &= ring->size - 1; ring->write_tail(ring, ring->tail); } static const struct intel_ring_buffer render_ring = { .name = "render ring", .id = RCS, .mmio_base = RENDER_RING_BASE, .size = 32 * PAGE_SIZE, .init = init_render_ring, .write_tail = ring_write_tail, .flush = render_ring_flush, .add_request = render_ring_add_request, .get_seqno = ring_get_seqno, .irq_get = render_ring_get_irq, .irq_put = render_ring_put_irq, .dispatch_execbuffer = render_ring_dispatch_execbuffer, .cleanup = render_ring_cleanup, .sync_to = render_ring_sync_to, .semaphore_register = {MI_SEMAPHORE_SYNC_INVALID, MI_SEMAPHORE_SYNC_RV, MI_SEMAPHORE_SYNC_RB}, .signal_mbox = {GEN6_VRSYNC, GEN6_BRSYNC}, }; /* ring buffer for bit-stream decoder */ static const struct intel_ring_buffer bsd_ring = { .name = "bsd ring", .id = VCS, .mmio_base = BSD_RING_BASE, .size = 32 * PAGE_SIZE, .init = init_ring_common, .write_tail = ring_write_tail, .flush = bsd_ring_flush, .add_request = ring_add_request, .get_seqno = ring_get_seqno, .irq_get = bsd_ring_get_irq, .irq_put = bsd_ring_put_irq, .dispatch_execbuffer = ring_dispatch_execbuffer, }; static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, uint32_t value) { drm_i915_private_t *dev_priv = ring->dev->dev_private; /* Every tail move must follow the sequence below */ I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK | GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE); I915_WRITE(GEN6_BSD_RNCID, 0x0); if (_intel_wait_for(ring->dev, (I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) & GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0, 50, true, "915g6i") != 0) DRM_ERROR("timed out waiting for IDLE Indicator\n"); I915_WRITE_TAIL(ring, value); I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK | GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE); } static int gen6_ring_flush(struct intel_ring_buffer *ring, uint32_t invalidate, uint32_t flush) { uint32_t cmd; int ret; ret = intel_ring_begin(ring, 4); if (ret) return ret; cmd = MI_FLUSH_DW; if (invalidate & I915_GEM_GPU_DOMAINS) cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD; intel_ring_emit(ring, cmd); intel_ring_emit(ring, 0); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static int gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, uint32_t offset, uint32_t len) { int ret; ret = intel_ring_begin(ring, 2); if (ret) return ret; intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965); /* bit0-7 is the length on GEN6+ */ intel_ring_emit(ring, offset); intel_ring_advance(ring); return 0; } static bool gen6_render_ring_get_irq(struct intel_ring_buffer *ring) { return gen6_ring_get_irq(ring, GT_USER_INTERRUPT, GEN6_RENDER_USER_INTERRUPT); } static void gen6_render_ring_put_irq(struct intel_ring_buffer *ring) { return gen6_ring_put_irq(ring, GT_USER_INTERRUPT, GEN6_RENDER_USER_INTERRUPT); } static bool gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring) { return gen6_ring_get_irq(ring, GT_GEN6_BSD_USER_INTERRUPT, GEN6_BSD_USER_INTERRUPT); } static void gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring) { return gen6_ring_put_irq(ring, GT_GEN6_BSD_USER_INTERRUPT, GEN6_BSD_USER_INTERRUPT); } /* ring buffer for Video Codec for Gen6+ */ static const struct intel_ring_buffer gen6_bsd_ring = { .name = "gen6 bsd ring", .id = VCS, .mmio_base = GEN6_BSD_RING_BASE, .size = 32 * PAGE_SIZE, .init = init_ring_common, .write_tail = gen6_bsd_ring_write_tail, .flush = gen6_ring_flush, .add_request = gen6_add_request, .get_seqno = gen6_ring_get_seqno, .irq_get = gen6_bsd_ring_get_irq, .irq_put = gen6_bsd_ring_put_irq, .dispatch_execbuffer = gen6_ring_dispatch_execbuffer, .sync_to = gen6_bsd_ring_sync_to, .semaphore_register = {MI_SEMAPHORE_SYNC_VR, MI_SEMAPHORE_SYNC_INVALID, MI_SEMAPHORE_SYNC_VB}, .signal_mbox = {GEN6_RVSYNC, GEN6_BVSYNC}, }; /* Blitter support (SandyBridge+) */ static bool blt_ring_get_irq(struct intel_ring_buffer *ring) { return gen6_ring_get_irq(ring, GT_BLT_USER_INTERRUPT, GEN6_BLITTER_USER_INTERRUPT); } static void blt_ring_put_irq(struct intel_ring_buffer *ring) { gen6_ring_put_irq(ring, GT_BLT_USER_INTERRUPT, GEN6_BLITTER_USER_INTERRUPT); } static int blt_ring_flush(struct intel_ring_buffer *ring, uint32_t invalidate, uint32_t flush) { uint32_t cmd; int ret; ret = intel_ring_begin(ring, 4); if (ret) return ret; cmd = MI_FLUSH_DW; if (invalidate & I915_GEM_DOMAIN_RENDER) cmd |= MI_INVALIDATE_TLB; intel_ring_emit(ring, cmd); intel_ring_emit(ring, 0); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static const struct intel_ring_buffer gen6_blt_ring = { .name = "blt ring", .id = BCS, .mmio_base = BLT_RING_BASE, .size = 32 * PAGE_SIZE, .init = init_ring_common, .write_tail = ring_write_tail, .flush = blt_ring_flush, .add_request = gen6_add_request, .get_seqno = gen6_ring_get_seqno, .irq_get = blt_ring_get_irq, .irq_put = blt_ring_put_irq, .dispatch_execbuffer = gen6_ring_dispatch_execbuffer, .sync_to = gen6_blt_ring_sync_to, .semaphore_register = {MI_SEMAPHORE_SYNC_BR, MI_SEMAPHORE_SYNC_BV, MI_SEMAPHORE_SYNC_INVALID}, .signal_mbox = {GEN6_RBSYNC, GEN6_VBSYNC}, }; int intel_init_render_ring_buffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = &dev_priv->rings[RCS]; *ring = render_ring; if (INTEL_INFO(dev)->gen >= 6) { ring->add_request = gen6_add_request; ring->flush = gen6_render_ring_flush; ring->irq_get = gen6_render_ring_get_irq; ring->irq_put = gen6_render_ring_put_irq; ring->get_seqno = gen6_ring_get_seqno; } else if (IS_GEN5(dev)) { ring->add_request = pc_render_add_request; ring->get_seqno = pc_render_get_seqno; } if (!I915_NEED_GFX_HWS(dev)) { ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; memset(ring->status_page.page_addr, 0, PAGE_SIZE); } return intel_init_ring_buffer(dev, ring); } int intel_render_ring_init_dri(struct drm_device *dev, uint64_t start, uint32_t size) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = &dev_priv->rings[RCS]; *ring = render_ring; if (INTEL_INFO(dev)->gen >= 6) { ring->add_request = gen6_add_request; ring->irq_get = gen6_render_ring_get_irq; ring->irq_put = gen6_render_ring_put_irq; } else if (IS_GEN5(dev)) { ring->add_request = pc_render_add_request; ring->get_seqno = pc_render_get_seqno; } ring->dev = dev; INIT_LIST_HEAD(&ring->active_list); INIT_LIST_HEAD(&ring->request_list); INIT_LIST_HEAD(&ring->gpu_write_list); ring->size = size; ring->effective_size = ring->size; if (IS_I830(ring->dev)) ring->effective_size -= 128; ring->map.offset = start; ring->map.size = size; ring->map.type = 0; ring->map.flags = 0; ring->map.mtrr = 0; drm_core_ioremap_wc(&ring->map, dev); if (ring->map.virtual == NULL) { DRM_ERROR("can not ioremap virtual address for" " ring buffer\n"); return -ENOMEM; } ring->virtual_start = (void *)ring->map.virtual; return 0; } int intel_init_bsd_ring_buffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = &dev_priv->rings[VCS]; if (IS_GEN6(dev) || IS_GEN7(dev)) *ring = gen6_bsd_ring; else *ring = bsd_ring; return intel_init_ring_buffer(dev, ring); } int intel_init_blt_ring_buffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = &dev_priv->rings[BCS]; *ring = gen6_blt_ring; return intel_init_ring_buffer(dev, ring); } Index: head/sys/dev/drm2/i915/intel_ringbuffer.h =================================================================== --- head/sys/dev/drm2/i915/intel_ringbuffer.h (revision 271704) +++ head/sys/dev/drm2/i915/intel_ringbuffer.h (revision 271705) @@ -1,203 +1,216 @@ /* * $FreeBSD$ */ #ifndef _INTEL_RINGBUFFER_H_ #define _INTEL_RINGBUFFER_H_ struct intel_hw_status_page { uint32_t *page_addr; unsigned int gfx_addr; struct drm_i915_gem_object *obj; }; #define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base)) #define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val) #define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base)) #define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val) #define I915_READ_HEAD(ring) I915_READ(RING_HEAD((ring)->mmio_base)) #define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val) #define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base)) #define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val) #define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base)) #define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val) #define I915_READ_NOPID(ring) I915_READ(RING_NOPID((ring)->mmio_base)) #define I915_READ_SYNC_0(ring) I915_READ(RING_SYNC_0((ring)->mmio_base)) #define I915_READ_SYNC_1(ring) I915_READ(RING_SYNC_1((ring)->mmio_base)) struct intel_ring_buffer { const char *name; enum intel_ring_id { RCS = 0x0, VCS, BCS, } id; #define I915_NUM_RINGS 3 uint32_t mmio_base; void *virtual_start; struct drm_device *dev; struct drm_i915_gem_object *obj; uint32_t head; uint32_t tail; int space; int size; int effective_size; struct intel_hw_status_page status_page; /** We track the position of the requests in the ring buffer, and * when each is retired we increment last_retired_head as the GPU * must have finished processing the request and so we know we * can advance the ringbuffer up to that position. * * last_retired_head is set to -1 after the value is consumed so * we can detect new retirements. */ u32 last_retired_head; struct mtx irq_lock; uint32_t irq_refcount; uint32_t irq_mask; uint32_t irq_seqno; /* last seq seem at irq time */ uint32_t trace_irq_seqno; uint32_t waiting_seqno; uint32_t sync_seqno[I915_NUM_RINGS-1]; bool (*irq_get)(struct intel_ring_buffer *ring); void (*irq_put)(struct intel_ring_buffer *ring); int (*init)(struct intel_ring_buffer *ring); void (*write_tail)(struct intel_ring_buffer *ring, uint32_t value); int (*flush)(struct intel_ring_buffer *ring, uint32_t invalidate_domains, uint32_t flush_domains); int (*add_request)(struct intel_ring_buffer *ring, uint32_t *seqno); uint32_t (*get_seqno)(struct intel_ring_buffer *ring); int (*dispatch_execbuffer)(struct intel_ring_buffer *ring, uint32_t offset, uint32_t length); void (*cleanup)(struct intel_ring_buffer *ring); int (*sync_to)(struct intel_ring_buffer *ring, struct intel_ring_buffer *to, u32 seqno); u32 semaphore_register[3]; /*our mbox written by others */ u32 signal_mbox[2]; /* mboxes this ring signals to */ /** * List of objects currently involved in rendering from the * ringbuffer. * * Includes buffers having the contents of their GPU caches * flushed, not necessarily primitives. last_rendering_seqno * represents when the rendering involved will be completed. * * A reference is held on the buffer while on this list. */ struct list_head active_list; /** * List of breadcrumbs associated with GPU requests currently * outstanding. */ struct list_head request_list; /** * List of objects currently pending a GPU write flush. * * All elements on this list will belong to either the * active_list or flushing_list, last_rendering_seqno can * be used to differentiate between the two elements. */ struct list_head gpu_write_list; /** * Do we have some not yet emitted requests outstanding? */ uint32_t outstanding_lazy_request; + /** + * Do an explicit TLB flush before MI_SET_CONTEXT + */ + bool itlb_before_ctx_switch; + struct i915_hw_context *default_context; + struct drm_i915_gem_object *last_context_obj; + drm_local_map_t map; void *private; }; + +static inline bool +intel_ring_initialized(struct intel_ring_buffer *ring) +{ + return ring->obj != NULL; +} static inline unsigned intel_ring_flag(struct intel_ring_buffer *ring) { return 1 << ring->id; } static inline uint32_t intel_ring_sync_index(struct intel_ring_buffer *ring, struct intel_ring_buffer *other) { int idx; /* * cs -> 0 = vcs, 1 = bcs * vcs -> 0 = bcs, 1 = cs, * bcs -> 0 = cs, 1 = vcs. */ idx = (other - ring) - 1; if (idx < 0) idx += I915_NUM_RINGS; return idx; } static inline uint32_t intel_read_status_page(struct intel_ring_buffer *ring, int reg) { return (atomic_load_acq_32(ring->status_page.page_addr + reg)); } void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring); int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n); static inline int intel_wait_ring_idle(struct intel_ring_buffer *ring) { return (intel_wait_ring_buffer(ring, ring->size - 8)); } int intel_ring_begin(struct intel_ring_buffer *ring, int n); static inline void intel_ring_emit(struct intel_ring_buffer *ring, uint32_t data) { *(volatile uint32_t *)((char *)ring->virtual_start + ring->tail) = data; ring->tail += 4; } void intel_ring_advance(struct intel_ring_buffer *ring); uint32_t intel_ring_get_seqno(struct intel_ring_buffer *ring); int intel_init_render_ring_buffer(struct drm_device *dev); int intel_init_bsd_ring_buffer(struct drm_device *dev); int intel_init_blt_ring_buffer(struct drm_device *dev); u32 intel_ring_get_active_head(struct intel_ring_buffer *ring); void intel_ring_setup_status_page(struct intel_ring_buffer *ring); static inline u32 intel_ring_get_tail(struct intel_ring_buffer *ring) { return ring->tail; } void i915_trace_irq_get(struct intel_ring_buffer *ring, uint32_t seqno); /* DRI warts */ int intel_render_ring_init_dri(struct drm_device *dev, uint64_t start, uint32_t size); #endif /* _INTEL_RINGBUFFER_H_ */ Index: head/sys/modules/drm2/i915kms/Makefile =================================================================== --- head/sys/modules/drm2/i915kms/Makefile (revision 271704) +++ head/sys/modules/drm2/i915kms/Makefile (revision 271705) @@ -1,55 +1,56 @@ # $FreeBSD$ .PATH: ${.CURDIR}/../../../dev/drm2/i915 KMOD = i915kms SRCS = \ i915_debug.c \ i915_dma.c \ i915_drv.c \ i915_gem.c \ + i915_gem_context.c \ i915_gem_execbuffer.c \ i915_gem_evict.c \ i915_gem_gtt.c \ i915_gem_tiling.c \ i915_irq.c \ i915_suspend.c \ intel_bios.c \ intel_crt.c \ intel_display.c \ intel_dp.c \ intel_fb.c \ intel_hdmi.c \ intel_iic.c \ intel_lvds.c \ intel_modes.c \ intel_opregion.c \ intel_overlay.c \ intel_panel.c \ intel_ringbuffer.c \ intel_sdvo.c \ intel_sprite.c \ intel_tv.c .if ${MACHINE_CPUARCH} == "amd64" SRCS += i915_ioc32.c .endif SRCS += \ opt_acpi.h \ opt_compat.h \ opt_drm.h \ opt_syscons.h \ acpi_if.h \ bus_if.h \ fb_if.h \ device_if.h \ iicbb_if.h \ iicbus_if.h \ pci_if.h .include CWARNFLAGS.i915_debug.c= -Wno-unused-function CWARNFLAGS.intel_lvds.c= -Wno-unused CWARNFLAGS.intel_tv.c= -Wno-unused CWARNFLAGS+= ${CWARNFLAGS.${.IMPSRC:T}} Index: head/sys/sys/param.h =================================================================== --- head/sys/sys/param.h (revision 271704) +++ head/sys/sys/param.h (revision 271705) @@ -1,347 +1,347 @@ /*- * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)param.h 8.3 (Berkeley) 4/4/95 * $FreeBSD$ */ #ifndef _SYS_PARAM_H_ #define _SYS_PARAM_H_ #include #define BSD 199506 /* System version (year & month). */ #define BSD4_3 1 #define BSD4_4 1 /* * __FreeBSD_version numbers are documented in the Porter's Handbook. * If you bump the version for any reason, you should update the documentation * there. * Currently this lives here in the doc/ repository: * * head/en_US.ISO8859-1/books/porters-handbook/book.xml * * scheme is: Rxx * 'R' is in the range 0 to 4 if this is a release branch or * x.0-CURRENT before RELENG_*_0 is created, otherwise 'R' is * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 1100034 /* Master, propagated to newvers */ +#define __FreeBSD_version 1100035 /* Master, propagated to newvers */ /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, * which by definition is always true on FreeBSD. This macro is also defined * on other systems that use the kernel of FreeBSD, such as GNU/kFreeBSD. * * It is tempting to use this macro in userland code when we want to enable * kernel-specific routines, and in fact it's fine to do this in code that * is part of FreeBSD itself. However, be aware that as presence of this * macro is still not widespread (e.g. older FreeBSD versions, 3rd party * compilers, etc), it is STRONGLY DISCOURAGED to check for this macro in * external applications without also checking for __FreeBSD__ as an * alternative. */ #undef __FreeBSD_kernel__ #define __FreeBSD_kernel__ #ifdef _KERNEL #define P_OSREL_SIGWAIT 700000 #define P_OSREL_SIGSEGV 700004 #define P_OSREL_MAP_ANON 800104 #define P_OSREL_MAJOR(x) ((x) / 100000) #endif #ifndef LOCORE #include #endif /* * Machine-independent constants (some used in following include files). * Redefined constants are from POSIX 1003.1 limits file. * * MAXCOMLEN should be >= sizeof(ac_comm) (see ) */ #include #define MAXCOMLEN 19 /* max command name remembered */ #define MAXINTERP PATH_MAX /* max interpreter file name length */ #define MAXLOGNAME 33 /* max login name length (incl. NUL) */ #define MAXUPRC CHILD_MAX /* max simultaneous processes */ #define NCARGS ARG_MAX /* max bytes for an exec function */ #define NGROUPS (NGROUPS_MAX+1) /* max number groups */ #define NOFILE OPEN_MAX /* max open files per process */ #define NOGROUP 65535 /* marker for empty group set member */ #define MAXHOSTNAMELEN 256 /* max hostname size */ #define SPECNAMELEN 63 /* max length of devicename */ /* More types and definitions used throughout the kernel. */ #ifdef _KERNEL #include #include #ifndef LOCORE #include #include #endif #ifndef FALSE #define FALSE 0 #endif #ifndef TRUE #define TRUE 1 #endif #endif #ifndef _KERNEL /* Signals. */ #include #endif /* Machine type dependent parameters. */ #include #ifndef _KERNEL #include #endif #ifndef DEV_BSHIFT #define DEV_BSHIFT 9 /* log2(DEV_BSIZE) */ #endif #define DEV_BSIZE (1<>PAGE_SHIFT) #endif /* * btodb() is messy and perhaps slow because `bytes' may be an off_t. We * want to shift an unsigned type to avoid sign extension and we don't * want to widen `bytes' unnecessarily. Assume that the result fits in * a daddr_t. */ #ifndef btodb #define btodb(bytes) /* calculates (bytes / DEV_BSIZE) */ \ (sizeof (bytes) > sizeof(long) \ ? (daddr_t)((unsigned long long)(bytes) >> DEV_BSHIFT) \ : (daddr_t)((unsigned long)(bytes) >> DEV_BSHIFT)) #endif #ifndef dbtob #define dbtob(db) /* calculates (db * DEV_BSIZE) */ \ ((off_t)(db) << DEV_BSHIFT) #endif #define PRIMASK 0x0ff #define PCATCH 0x100 /* OR'd with pri for tsleep to check signals */ #define PDROP 0x200 /* OR'd with pri to stop re-entry of interlock mutex */ #define NZERO 0 /* default "nice" */ #define NBBY 8 /* number of bits in a byte */ #define NBPW sizeof(int) /* number of bytes per word (integer) */ #define CMASK 022 /* default file mask: S_IWGRP|S_IWOTH */ #define NODEV (dev_t)(-1) /* non-existent device */ /* * File system parameters and macros. * * MAXBSIZE - Filesystems are made out of blocks of at most MAXBSIZE bytes * per block. MAXBSIZE may be made larger without effecting * any existing filesystems as long as it does not exceed MAXPHYS, * and may be made smaller at the risk of not being able to use * filesystems which require a block size exceeding MAXBSIZE. * * BKVASIZE - Nominal buffer space per buffer, in bytes. BKVASIZE is the * minimum KVM memory reservation the kernel is willing to make. * Filesystems can of course request smaller chunks. Actual * backing memory uses a chunk size of a page (PAGE_SIZE). * * If you make BKVASIZE too small you risk seriously fragmenting * the buffer KVM map which may slow things down a bit. If you * make it too big the kernel will not be able to optimally use * the KVM memory reserved for the buffer cache and will wind * up with too-few buffers. * * The default is 16384, roughly 2x the block size used by a * normal UFS filesystem. */ #define MAXBSIZE 65536 /* must be power of 2 */ #define BKVASIZE 16384 /* must be power of 2 */ #define BKVAMASK (BKVASIZE-1) /* * MAXPATHLEN defines the longest permissible path length after expanding * symbolic links. It is used to allocate a temporary buffer from the buffer * pool in which to do the name expansion, hence should be a power of two, * and must be less than or equal to MAXBSIZE. MAXSYMLINKS defines the * maximum number of symbolic links that may be expanded in a path name. * It should be set high enough to allow all legitimate uses, but halt * infinite loops reasonably quickly. */ #define MAXPATHLEN PATH_MAX #define MAXSYMLINKS 32 /* Bit map related macros. */ #define setbit(a,i) (((unsigned char *)(a))[(i)/NBBY] |= 1<<((i)%NBBY)) #define clrbit(a,i) (((unsigned char *)(a))[(i)/NBBY] &= ~(1<<((i)%NBBY))) #define isset(a,i) \ (((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) #define isclr(a,i) \ ((((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) == 0) /* Macros for counting and rounding. */ #ifndef howmany #define howmany(x, y) (((x)+((y)-1))/(y)) #endif #define nitems(x) (sizeof((x)) / sizeof((x)[0])) #define rounddown(x, y) (((x)/(y))*(y)) #define rounddown2(x, y) ((x)&(~((y)-1))) /* if y is power of two */ #define roundup(x, y) ((((x)+((y)-1))/(y))*(y)) /* to any y */ #define roundup2(x, y) (((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */ #define powerof2(x) ((((x)-1)&(x))==0) /* Macros for min/max. */ #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) #ifdef _KERNEL /* * Basic byte order function prototypes for non-inline functions. */ #ifndef LOCORE #ifndef _BYTEORDER_PROTOTYPED #define _BYTEORDER_PROTOTYPED __BEGIN_DECLS __uint32_t htonl(__uint32_t); __uint16_t htons(__uint16_t); __uint32_t ntohl(__uint32_t); __uint16_t ntohs(__uint16_t); __END_DECLS #endif #endif #ifndef lint #ifndef _BYTEORDER_FUNC_DEFINED #define _BYTEORDER_FUNC_DEFINED #define htonl(x) __htonl(x) #define htons(x) __htons(x) #define ntohl(x) __ntohl(x) #define ntohs(x) __ntohs(x) #endif /* !_BYTEORDER_FUNC_DEFINED */ #endif /* lint */ #endif /* _KERNEL */ /* * Scale factor for scaled integers used to count %cpu time and load avgs. * * The number of CPU `tick's that map to a unique `%age' can be expressed * by the formula (1 / (2 ^ (FSHIFT - 11))). The maximum load average that * can be calculated (assuming 32 bits) can be closely approximated using * the formula (2 ^ (2 * (16 - FSHIFT))) for (FSHIFT < 15). * * For the scheduler to maintain a 1:1 mapping of CPU `tick' to `%age', * FSHIFT must be at least 11; this gives us a maximum load avg of ~1024. */ #define FSHIFT 11 /* bits to right of fixed binary point */ #define FSCALE (1<> (PAGE_SHIFT - DEV_BSHIFT)) #define ctodb(db) /* calculates pages to devblks */ \ ((db) << (PAGE_SHIFT - DEV_BSHIFT)) /* * Old spelling of __containerof(). */ #define member2struct(s, m, x) \ ((struct s *)(void *)((char *)(x) - offsetof(struct s, m))) /* * Access a variable length array that has been declared as a fixed * length array. */ #define __PAST_END(array, offset) (((__typeof__(*(array)) *)(array))[offset]) #endif /* _SYS_PARAM_H_ */