diff --git a/drivers/gpu/ion/ion_carveout_heap.c b/drivers/gpu/ion/ion_carveout_heap.c index 1a7c31d16ff..a591eb41a92 100644 --- a/drivers/gpu/ion/ion_carveout_heap.c +++ b/drivers/gpu/ion/ion_carveout_heap.c @@ -30,6 +30,7 @@ #include #include +#include struct ion_carveout_heap { struct ion_heap heap; @@ -41,6 +42,7 @@ struct ion_carveout_heap { int (*release_region)(void *); atomic_t map_count; void *bus_id; + unsigned int has_outer_cache; }; ion_phys_addr_t ion_carveout_allocate(struct ion_heap *heap, @@ -238,25 +240,31 @@ int ion_carveout_cache_ops(struct ion_heap *heap, struct ion_buffer *buffer, void *vaddr, unsigned int offset, unsigned int length, unsigned int cmd) { - unsigned long vstart, pstart; - - pstart = buffer->priv_phys + offset; - vstart = (unsigned long)vaddr; + void (*outer_cache_op)(phys_addr_t, phys_addr_t); + struct ion_carveout_heap *carveout_heap = + container_of(heap, struct ion_carveout_heap, heap); switch (cmd) { case ION_IOC_CLEAN_CACHES: - clean_caches(vstart, length, pstart); + dmac_clean_range(vaddr, vaddr + length); + outer_cache_op = outer_clean_range; break; case ION_IOC_INV_CACHES: - invalidate_caches(vstart, length, pstart); + dmac_inv_range(vaddr, vaddr + length); + outer_cache_op = outer_inv_range; break; case ION_IOC_CLEAN_INV_CACHES: - clean_and_invalidate_caches(vstart, length, pstart); + dmac_flush_range(vaddr, vaddr + length); + outer_cache_op = outer_flush_range; break; default: return -EINVAL; } + if (carveout_heap->has_outer_cache) { + unsigned long pstart = buffer->priv_phys + offset; + outer_cache_op(pstart, pstart + length); + } return 0; } @@ -456,6 +464,7 @@ struct ion_heap *ion_carveout_heap_create(struct ion_platform_heap *heap_data) carveout_heap->heap.type = ION_HEAP_TYPE_CARVEOUT; carveout_heap->allocated_bytes = 0; carveout_heap->total_size = heap_data->size; + carveout_heap->has_outer_cache = heap_data->has_outer_cache; if (heap_data->extra_data) { struct ion_co_heap_pdata *extra_data = diff --git a/drivers/gpu/ion/ion_cp_heap.c b/drivers/gpu/ion/ion_cp_heap.c index a7c0a5102f3..525836929fc 100644 --- a/drivers/gpu/ion/ion_cp_heap.c +++ b/drivers/gpu/ion/ion_cp_heap.c @@ -38,6 +38,7 @@ #include "ion_priv.h" #include +#include /** * struct ion_cp_heap - container for the heap and shared heap data @@ -70,7 +71,8 @@ * @reserved_vrange: reserved virtual address range for use with fmem * @iommu_map_all: Indicates whether we should map whole heap into IOMMU. * @iommu_2x_map_domain: Indicates the domain to use for overmapping. - */ + * @has_outer_cache: set to 1 if outer cache is used, 0 otherwise. +*/ struct ion_cp_heap { struct ion_heap heap; struct gen_pool *pool; @@ -94,7 +96,7 @@ struct ion_cp_heap { void *reserved_vrange; int iommu_map_all; int iommu_2x_map_domain; - + unsigned int has_outer_cache; }; enum { @@ -550,25 +552,31 @@ int ion_cp_cache_ops(struct ion_heap *heap, struct ion_buffer *buffer, void *vaddr, unsigned int offset, unsigned int length, unsigned int cmd) { - unsigned long vstart, pstart; - - pstart = buffer->priv_phys + offset; - vstart = (unsigned long)vaddr; + void (*outer_cache_op)(phys_addr_t, phys_addr_t); + struct ion_cp_heap *cp_heap = + container_of(heap, struct ion_cp_heap, heap); switch (cmd) { case ION_IOC_CLEAN_CACHES: - clean_caches(vstart, length, pstart); + dmac_clean_range(vaddr, vaddr + length); + outer_cache_op = outer_clean_range; break; case ION_IOC_INV_CACHES: - invalidate_caches(vstart, length, pstart); + dmac_inv_range(vaddr, vaddr + length); + outer_cache_op = outer_inv_range; break; case ION_IOC_CLEAN_INV_CACHES: - clean_and_invalidate_caches(vstart, length, pstart); + dmac_flush_range(vaddr, vaddr + length); + outer_cache_op = outer_flush_range; break; default: return -EINVAL; } + if (cp_heap->has_outer_cache) { + unsigned long pstart = buffer->priv_phys + offset; + outer_cache_op(pstart, pstart + length); + } return 0; } @@ -914,6 +922,7 @@ struct ion_heap *ion_cp_heap_create(struct ion_platform_heap *heap_data) cp_heap->heap_protected = HEAP_NOT_PROTECTED; cp_heap->secure_base = cp_heap->base; cp_heap->secure_size = heap_data->size; + cp_heap->has_outer_cache = heap_data->has_outer_cache; if (heap_data->extra_data) { struct ion_cp_heap_pdata *extra_data = heap_data->extra_data; diff --git a/drivers/gpu/ion/ion_iommu_heap.c b/drivers/gpu/ion/ion_iommu_heap.c index 8c014b54a38..01d432d8c73 100644 --- a/drivers/gpu/ion/ion_iommu_heap.c +++ b/drivers/gpu/ion/ion_iommu_heap.c @@ -23,10 +23,12 @@ #include #include +#include #include struct ion_iommu_heap { struct ion_heap heap; + unsigned int has_outer_cache; }; struct ion_iommu_priv_data { @@ -263,34 +265,39 @@ static int ion_iommu_cache_ops(struct ion_heap *heap, struct ion_buffer *buffer, void *vaddr, unsigned int offset, unsigned int length, unsigned int cmd) { - unsigned long vstart, pstart; - void (*op)(unsigned long, unsigned long, unsigned long); - unsigned int i; - struct ion_iommu_priv_data *data = buffer->priv_virt; - - if (!data) - return -ENOMEM; + void (*outer_cache_op)(phys_addr_t, phys_addr_t); + struct ion_iommu_heap *iommu_heap = + container_of(heap, struct ion_iommu_heap, heap); switch (cmd) { case ION_IOC_CLEAN_CACHES: - op = clean_caches; + dmac_clean_range(vaddr, vaddr + length); + outer_cache_op = outer_clean_range; break; case ION_IOC_INV_CACHES: - op = invalidate_caches; + dmac_inv_range(vaddr, vaddr + length); + outer_cache_op = outer_inv_range; break; case ION_IOC_CLEAN_INV_CACHES: - op = clean_and_invalidate_caches; + dmac_flush_range(vaddr, vaddr + length); + outer_cache_op = outer_flush_range; break; default: return -EINVAL; } - vstart = (unsigned long) vaddr; - for (i = 0; i < data->nrpages; ++i, vstart += PAGE_SIZE) { - pstart = page_to_phys(data->pages[i]); - op(vstart, PAGE_SIZE, pstart); - } + if (iommu_heap->has_outer_cache) { + unsigned long pstart; + unsigned int i; + struct ion_iommu_priv_data *data = buffer->priv_virt; + if (!data) + return -ENOMEM; + for (i = 0; i < data->nrpages; ++i) { + pstart = page_to_phys(data->pages[i]); + outer_cache_op(pstart, pstart + PAGE_SIZE); + } + } return 0; } @@ -332,6 +339,7 @@ struct ion_heap *ion_iommu_heap_create(struct ion_platform_heap *heap_data) iommu_heap->heap.ops = &iommu_heap_ops; iommu_heap->heap.type = ION_HEAP_TYPE_IOMMU; + iommu_heap->has_outer_cache = heap_data->has_outer_cache; return &iommu_heap->heap; } diff --git a/drivers/gpu/ion/ion_system_heap.c b/drivers/gpu/ion/ion_system_heap.c index 43064556776..4bcee4caeb6 100644 --- a/drivers/gpu/ion/ion_system_heap.c +++ b/drivers/gpu/ion/ion_system_heap.c @@ -26,9 +26,12 @@ #include #include "ion_priv.h" #include +#include static atomic_t system_heap_allocated; static atomic_t system_contig_heap_allocated; +static unsigned int system_heap_has_outer_cache; +static unsigned int system_heap_contig_has_outer_cache; static int ion_system_heap_allocate(struct ion_heap *heap, struct ion_buffer *buffer, @@ -176,41 +179,44 @@ int ion_system_heap_cache_ops(struct ion_heap *heap, struct ion_buffer *buffer, void *vaddr, unsigned int offset, unsigned int length, unsigned int cmd) { - unsigned long vstart, pstart; - void (*op)(unsigned long, unsigned long, unsigned long); - struct sg_table *table = buffer->priv_virt; - struct scatterlist *sg; - int i; + void (*outer_cache_op)(phys_addr_t, phys_addr_t); switch (cmd) { case ION_IOC_CLEAN_CACHES: - op = clean_caches; + dmac_clean_range(vaddr, vaddr + length); + outer_cache_op = outer_clean_range; break; case ION_IOC_INV_CACHES: - op = invalidate_caches; + dmac_inv_range(vaddr, vaddr + length); + outer_cache_op = outer_inv_range; break; case ION_IOC_CLEAN_INV_CACHES: - op = clean_and_invalidate_caches; + dmac_flush_range(vaddr, vaddr + length); + outer_cache_op = outer_flush_range; break; default: return -EINVAL; } - for_each_sg(table->sgl, sg, table->nents, i) { - struct page *page = sg_page(sg); - pstart = page_to_phys(page); - /* - * If page -> phys is returning NULL, something - * has really gone wrong... - */ - if (!pstart) { - WARN(1, "Could not translate virtual address to physical address\n"); - return -EINVAL; + if (system_heap_has_outer_cache) { + unsigned long pstart; + struct sg_table *table = buffer->priv_virt; + struct scatterlist *sg; + int i; + for_each_sg(table->sgl, sg, table->nents, i) { + struct page *page = sg_page(sg); + pstart = page_to_phys(page); + /* + * If page -> phys is returning NULL, something + * has really gone wrong... + */ + if (!pstart) { + WARN(1, "Could not translate virtual address to physical address\n"); + return -EINVAL; + } + outer_cache_op(pstart, pstart + PAGE_SIZE); } - vstart = page_to_virt(page); - op(vstart, PAGE_SIZE, pstart); } - return 0; } @@ -303,7 +309,7 @@ static struct ion_heap_ops vmalloc_ops = { .unmap_iommu = ion_system_heap_unmap_iommu, }; -struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused) +struct ion_heap *ion_system_heap_create(struct ion_platform_heap *pheap) { struct ion_heap *heap; @@ -312,6 +318,7 @@ struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused) return ERR_PTR(-ENOMEM); heap->ops = &vmalloc_ops; heap->type = ION_HEAP_TYPE_SYSTEM; + system_heap_has_outer_cache = pheap->has_outer_cache; return heap; } @@ -388,31 +395,38 @@ int ion_system_contig_heap_cache_ops(struct ion_heap *heap, unsigned int offset, unsigned int length, unsigned int cmd) { - unsigned long vstart, pstart; - - pstart = virt_to_phys(buffer->priv_virt) + offset; - if (!pstart) { - WARN(1, "Could not do virt to phys translation on %p\n", - buffer->priv_virt); - return -EINVAL; - } - - vstart = (unsigned long) vaddr; + void (*outer_cache_op)(phys_addr_t, phys_addr_t); switch (cmd) { case ION_IOC_CLEAN_CACHES: - clean_caches(vstart, length, pstart); + dmac_clean_range(vaddr, vaddr + length); + outer_cache_op = outer_clean_range; break; case ION_IOC_INV_CACHES: - invalidate_caches(vstart, length, pstart); + dmac_inv_range(vaddr, vaddr + length); + outer_cache_op = outer_inv_range; break; case ION_IOC_CLEAN_INV_CACHES: - clean_and_invalidate_caches(vstart, length, pstart); + dmac_flush_range(vaddr, vaddr + length); + outer_cache_op = outer_flush_range; break; default: return -EINVAL; } + if (system_heap_contig_has_outer_cache) { + unsigned long pstart; + + pstart = virt_to_phys(buffer->priv_virt) + offset; + if (!pstart) { + WARN(1, "Could not do virt to phys translation on %p\n", + buffer->priv_virt); + return -EINVAL; + } + + outer_cache_op(pstart, pstart + PAGE_SIZE); + } + return 0; } @@ -518,7 +532,7 @@ static struct ion_heap_ops kmalloc_ops = { .unmap_iommu = ion_system_heap_unmap_iommu, }; -struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *unused) +struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *pheap) { struct ion_heap *heap; @@ -527,6 +541,7 @@ struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *unused) return ERR_PTR(-ENOMEM); heap->ops = &kmalloc_ops; heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG; + system_heap_contig_has_outer_cache = pheap->has_outer_cache; return heap; } diff --git a/drivers/gpu/ion/msm/msm_ion.c b/drivers/gpu/ion/msm/msm_ion.c index 9256444e747..822dd162f60 100644 --- a/drivers/gpu/ion/msm/msm_ion.c +++ b/drivers/gpu/ion/msm/msm_ion.c @@ -282,6 +282,7 @@ static int msm_ion_probe(struct platform_device *pdev) struct ion_platform_heap *heap_data = &pdata->heaps[i]; msm_ion_allocate(heap_data); + heap_data->has_outer_cache = pdata->has_outer_cache; heaps[i] = ion_heap_create(heap_data); if (IS_ERR_OR_NULL(heaps[i])) { heaps[i] = 0; diff --git a/include/linux/ion.h b/include/linux/ion.h index a0a421aab26..29eed22b211 100644 --- a/include/linux/ion.h +++ b/include/linux/ion.h @@ -147,6 +147,7 @@ struct ion_buffer; * @base: base address of heap in physical memory if applicable * @size: size of the heap in bytes if applicable * @memory_type:Memory type used for the heap + * @has_outer_cache: set to 1 if outer cache is used, 0 otherwise. * @extra_data: Extra data specific to each heap type */ struct ion_platform_heap { @@ -156,6 +157,7 @@ struct ion_platform_heap { ion_phys_addr_t base; size_t size; enum ion_memory_types memory_type; + unsigned int has_outer_cache; void *extra_data; }; @@ -227,6 +229,7 @@ struct ion_co_heap_pdata { /** * struct ion_platform_data - array of platform heaps passed from board file + * @has_outer_cache: set to 1 if outer cache is used, 0 otherwise. * @nr: number of structures in the array * @request_region: function to be called when the number of allocations goes * from 0 -> 1 @@ -238,6 +241,7 @@ struct ion_co_heap_pdata { * Provided by the board file in the form of platform data to a platform device. */ struct ion_platform_data { + unsigned int has_outer_cache; int nr; int (*request_region)(void *); int (*release_region)(void *);