40 #include <linux/slab.h>
44 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
77 rdev->
mman.mem_global_referenced =
false;
78 global_ref = &rdev->
mman.mem_global_ref;
81 global_ref->
init = &radeon_ttm_mem_global_init;
82 global_ref->
release = &radeon_ttm_mem_global_release;
85 DRM_ERROR(
"Failed setting up TTM memory accounting "
90 rdev->
mman.bo_global_ref.mem_glob =
91 rdev->
mman.mem_global_ref.object;
92 global_ref = &rdev->
mman.bo_global_ref.ref;
99 DRM_ERROR(
"Failed setting up TTM BO subsystem.\n");
104 rdev->
mman.mem_global_referenced =
true;
108 static void radeon_ttm_global_fini(
struct radeon_device *rdev)
110 if (rdev->
mman.mem_global_referenced) {
113 rdev->
mman.mem_global_referenced =
false;
127 rdev = radeon_get_rdev(bdev);
144 if (!(drm_core_has_AGP(rdev->
ddev) && rdev->
ddev->agp)) {
145 DRM_ERROR(
"AGP is not enabled for memory type %u\n",
149 if (!rdev->
ddev->agp->cant_use_aperture)
167 DRM_ERROR(
"Unsupported memory type %u\n", (
unsigned)type);
189 switch (bo->
mem.mem_type) {
219 bool evict,
int no_wait_reserve,
bool no_wait_gpu,
228 rdev = radeon_get_rdev(bo->
bdev);
235 old_start += rdev->
mc.vram_start;
238 old_start += rdev->
mc.gtt_start;
241 DRM_ERROR(
"Unknown placement %d\n", old_mem->
mem_type);
246 new_start += rdev->
mc.vram_start;
249 new_start += rdev->
mc.gtt_start;
252 DRM_ERROR(
"Unknown placement %d\n", old_mem->
mem_type);
255 if (!rdev->
ring[ridx].ready) {
256 DRM_ERROR(
"Trying to move memory with ring turned off.\n");
269 evict, no_wait_reserve, no_wait_gpu, new_mem);
275 bool evict,
bool interruptible,
276 bool no_wait_reserve,
bool no_wait_gpu,
286 rdev = radeon_get_rdev(bo->
bdev);
297 interruptible, no_wait_reserve, no_wait_gpu);
311 r = radeon_move_blit(bo,
true, no_wait_reserve, no_wait_gpu, &tmp_mem, old_mem);
322 bool evict,
bool interruptible,
323 bool no_wait_reserve,
bool no_wait_gpu,
333 rdev = radeon_get_rdev(bo->
bdev);
343 r =
ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_reserve, no_wait_gpu);
351 r = radeon_move_blit(bo,
true, no_wait_reserve, no_wait_gpu, new_mem, old_mem);
361 bool evict,
bool interruptible,
362 bool no_wait_reserve,
bool no_wait_gpu,
369 rdev = radeon_get_rdev(bo->
bdev);
371 radeon_move_null(bo, new_mem);
379 radeon_move_null(bo, new_mem);
390 r = radeon_move_vram_ram(bo, evict, interruptible,
391 no_wait_reserve, no_wait_gpu, new_mem);
394 r = radeon_move_ram_vram(bo, evict, interruptible,
395 no_wait_reserve, no_wait_gpu, new_mem);
397 r = radeon_move_blit(bo, evict, no_wait_reserve, no_wait_gpu, new_mem, old_mem);
416 mem->
bus.is_iomem =
false;
428 mem->
bus.base = rdev->
mc.agp_base;
429 mem->
bus.is_iomem = !rdev->
ddev->agp->cant_use_aperture;
436 if ((mem->
bus.offset + mem->
bus.size) > rdev->
mc.visible_vram_size)
438 mem->
bus.base = rdev->
mc.aper_base;
439 mem->
bus.is_iomem =
true;
460 mem->
bus.base = (mem->
bus.base & 0x0ffffffff
UL) +
461 rdev->
ddev->hose->dense_mem_base;
474 static int radeon_sync_obj_wait(
void *sync_obj,
void *sync_arg,
475 bool lazy,
bool interruptible)
480 static int radeon_sync_obj_flush(
void *sync_obj,
void *sync_arg)
485 static void radeon_sync_obj_unref(
void **sync_obj)
490 static void *radeon_sync_obj_ref(
void *sync_obj)
495 static bool radeon_sync_obj_signaled(
void *sync_obj,
void *sync_arg)
509 static int radeon_ttm_backend_bind(
struct ttm_tt *ttm,
517 WARN(1,
"nothing to bind %lu pages for mreg %p back %p!\n",
523 DRM_ERROR(
"failed to bind %lu pages at 0x%08X\n",
530 static int radeon_ttm_backend_unbind(
struct ttm_tt *ttm)
538 static void radeon_ttm_backend_destroy(
struct ttm_tt *ttm)
547 .bind = &radeon_ttm_backend_bind,
548 .unbind = &radeon_ttm_backend_unbind,
549 .destroy = &radeon_ttm_backend_destroy,
559 rdev = radeon_get_rdev(bdev);
562 return ttm_agp_tt_create(bdev, rdev->
ddev->agp->bridge,
563 size, page_flags, dummy_read_page);
571 gtt->
ttm.ttm.func = &radeon_backend_func;
577 return >t->
ttm.ttm;
580 static int radeon_ttm_tt_populate(
struct ttm_tt *ttm)
588 if (ttm->
state != tt_unpopulated)
591 if (slave && ttm->
sg) {
594 ttm->
state = tt_unbound;
598 rdev = radeon_get_rdev(ttm->
bdev);
601 return ttm_agp_tt_populate(ttm);
605 #ifdef CONFIG_SWIOTLB
617 gtt->
ttm.dma_address[
i] = pci_map_page(rdev->
pdev, ttm->
pages[i],
620 if (pci_dma_mapping_error(rdev->
pdev, gtt->
ttm.dma_address[i])) {
622 pci_unmap_page(rdev->
pdev, gtt->
ttm.dma_address[i],
624 gtt->
ttm.dma_address[
i] = 0;
633 static void radeon_ttm_tt_unpopulate(
struct ttm_tt *ttm)
643 rdev = radeon_get_rdev(ttm->
bdev);
646 ttm_agp_tt_unpopulate(ttm);
651 #ifdef CONFIG_SWIOTLB
659 if (gtt->
ttm.dma_address[i]) {
660 pci_unmap_page(rdev->
pdev, gtt->
ttm.dma_address[i],
669 .ttm_tt_create = &radeon_ttm_tt_create,
670 .ttm_tt_populate = &radeon_ttm_tt_populate,
671 .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate,
672 .invalidate_caches = &radeon_invalidate_caches,
673 .init_mem_type = &radeon_init_mem_type,
674 .evict_flags = &radeon_evict_flags,
675 .move = &radeon_bo_move,
676 .verify_access = &radeon_verify_access,
677 .sync_obj_signaled = &radeon_sync_obj_signaled,
678 .sync_obj_wait = &radeon_sync_obj_wait,
679 .sync_obj_flush = &radeon_sync_obj_flush,
680 .sync_obj_unref = &radeon_sync_obj_unref,
681 .sync_obj_ref = &radeon_sync_obj_ref,
684 .io_mem_reserve = &radeon_ttm_io_mem_reserve,
685 .io_mem_free = &radeon_ttm_io_mem_free,
692 r = radeon_ttm_global_init(rdev);
698 rdev->
mman.bo_global_ref.ref.object,
702 DRM_ERROR(
"failed initializing buffer object driver(%d).\n", r);
705 rdev->
mman.initialized =
true;
709 DRM_ERROR(
"Failed initializing VRAM heap.\n");
727 DRM_INFO(
"radeon: %uM of VRAM memory ready\n",
728 (
unsigned)rdev->
mc.real_vram_size / (1024 * 1024));
732 DRM_ERROR(
"Failed initializing GTT heap.\n");
735 DRM_INFO(
"radeon: %uM of GTT memory ready.\n",
736 (
unsigned)(rdev->
mc.gtt_size / (1024 * 1024)));
737 rdev->
mman.bdev.dev_mapping = rdev->
ddev->dev_mapping;
739 r = radeon_ttm_debugfs_init(rdev);
741 DRM_ERROR(
"Failed to init debugfs\n");
751 if (!rdev->
mman.initialized)
765 radeon_ttm_global_fini(rdev);
766 rdev->
mman.initialized =
false;
767 DRM_INFO(
"radeon: ttm finalized\n");
776 if (!rdev->
mman.initialized)
784 static struct vm_operations_struct radeon_ttm_vm_ops;
785 static const struct vm_operations_struct *ttm_vm_ops =
NULL;
787 static int radeon_ttm_fault(
struct vm_area_struct *vma,
struct vm_fault *vmf)
795 return VM_FAULT_NOPAGE;
797 rdev = radeon_get_rdev(bo->
bdev);
799 r = ttm_vm_ops->fault(vma, vmf);
806 struct drm_file *file_priv;
815 rdev = file_priv->minor->
dev->dev_private;
825 radeon_ttm_vm_ops = *ttm_vm_ops;
826 radeon_ttm_vm_ops.fault = &radeon_ttm_fault;
828 vma->
vm_ops = &radeon_ttm_vm_ops;
833 #define RADEON_DEBUGFS_MEM_TYPES 2
835 #if defined(CONFIG_DEBUG_FS)
836 static int radeon_mm_dump_table(
struct seq_file *
m,
void *
data)
838 struct drm_info_node *
node = (
struct drm_info_node *)m->
private;
839 struct drm_mm *mm = (
struct drm_mm *)node->info_ent->data;
846 ret = drm_mm_dump_table(m, mm);
852 static int radeon_ttm_debugfs_init(
struct radeon_device *rdev)
854 #if defined(CONFIG_DEBUG_FS)
861 sprintf(radeon_mem_types_names[i],
"radeon_vram_mm");
863 sprintf(radeon_mem_types_names[i],
"radeon_gtt_mm");
864 radeon_mem_types_list[
i].name = radeon_mem_types_names[
i];
865 radeon_mem_types_list[
i].show = &radeon_mm_dump_table;
866 radeon_mem_types_list[
i].driver_features = 0;
870 radeon_mem_types_list[
i].data = rdev->
mman.bdev.man[
TTM_PL_TT].priv;
874 sprintf(radeon_mem_types_names[i],
"ttm_page_pool");
875 radeon_mem_types_list[
i].name = radeon_mem_types_names[
i];
877 radeon_mem_types_list[
i].driver_features = 0;
878 radeon_mem_types_list[i++].data =
NULL;
879 #ifdef CONFIG_SWIOTLB
881 sprintf(radeon_mem_types_names[i],
"ttm_dma_page_pool");
882 radeon_mem_types_list[
i].name = radeon_mem_types_names[
i];
884 radeon_mem_types_list[
i].driver_features = 0;
885 radeon_mem_types_list[i++].data =
NULL;