@@ -2,6 +2,21 @@
#ifndef __NVKM_MMU_H__
#define __NVKM_MMU_H__
#include <core/subdev.h>
+#include <linux/hmm.h>
+
+/* Need to change HMM to be more driver friendly */
+#if IS_ENABLED(CONFIG_HMM)
+#else
+typedef unsigned long hmm_pfn_t;
+#define HMM_PFN_VALID (1 << 0)
+#define HMM_PFN_READ (1 << 1)
+#define HMM_PFN_WRITE (1 << 2)
+#define HMM_PFN_ERROR (1 << 3)
+#define HMM_PFN_EMPTY (1 << 4)
+#define HMM_PFN_SPECIAL (1 << 5)
+#define HMM_PFN_DEVICE_UNADDRESSABLE (1 << 6)
+#define HMM_PFN_SHIFT 7
+#endif
struct nvkm_vma {
struct list_head head;
@@ -56,6 +71,7 @@ void nvkm_vmm_part(struct nvkm_vmm *, struct nvkm_memory *inst);
int nvkm_vmm_get(struct nvkm_vmm *, u8 page, u64 size, struct nvkm_vma **);
void nvkm_vmm_put(struct nvkm_vmm *, struct nvkm_vma **);
+
struct nvkm_vmm_map {
struct nvkm_memory *memory;
u64 offset;
@@ -63,6 +79,11 @@ struct nvkm_vmm_map {
struct nvkm_mm_node *mem;
struct scatterlist *sgl;
dma_addr_t *dma;
+#define NV_HMM_PAGE_FLAG_V HMM_PFN_VALID
+#define NV_HMM_PAGE_FLAG_W HMM_PFN_WRITE
+#define NV_HMM_PAGE_FLAG_E HMM_PFN_ERROR
+#define NV_HMM_PAGE_PFN_SHIFT HMM_PFN_SHIFT
+ u64 *pages;
u64 off;
const struct nvkm_vmm_page *page;
@@ -75,7 +75,7 @@ struct nvkm_vmm_iter {
struct nvkm_vmm *vmm;
u64 cnt;
u16 max, lvl;
- u64 start, addr;
+ u64 start, addr, *pages;
u32 pte[NVKM_VMM_LEVELS_MAX];
struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
int flush;
@@ -281,6 +281,59 @@ nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
return true;
}
+static bool
+nvkm_vmm_unref_hmm_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+{
+ const struct nvkm_vmm_desc *desc = it->desc;
+ const int type = desc->type == SPT;
+ struct nvkm_vmm_pt *pgt = it->pt[0];
+ struct nvkm_mmu_pt *pt;
+ int mapped;
+
+ pt = pgt->pt[type];
+ mapped = desc->func->hmm_unmap(it->vmm, pt, ptei, ptes, NULL);
+ if (mapped <= 0)
+ return false;
+ ptes = mapped;
+
+ /* Dual-PTs need special handling, unless PDE becoming invalid. */
+ if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
+ nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
+
+ /* GPU may have cached the PTs, flush before freeing. */
+ nvkm_vmm_flush_mark(it);
+ nvkm_vmm_flush(it);
+
+ nvkm_kmap(pt->memory);
+ while (mapped--) {
+ u64 data = nvkm_ro64(pt->memory, pt->base + ptei * 8);
+ dma_addr_t dma = (data >> 8) << 12;
+
+ if (!data) {
+ ptei++;
+ continue;
+ }
+ dma_unmap_page(it->vmm->mmu->subdev.device->dev, dma,
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ VMM_WO064(pt, it->vmm, ptei++ * 8, 0UL);
+ }
+ nvkm_done(pt->memory);
+
+ /* Drop PTE references. */
+ pgt->refs[type] -= ptes;
+
+ /* PT no longer neeed? Destroy it. */
+ if (!pgt->refs[type]) {
+ it->lvl++;
+ TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
+ it->lvl--;
+ nvkm_vmm_unref_pdes(it);
+ return false; /* PTE writes for unmap() not necessary. */
+ }
+
+ return true;
+}
+
static void
nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
@@ -349,6 +402,32 @@ nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
}
}
+static bool
+nvkm_vmm_ref_hmm_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+{
+ const struct nvkm_vmm_desc *desc = it->desc;
+ const int type = desc->type == SPT;
+ struct nvkm_vmm_pt *pgt = it->pt[0];
+ struct nvkm_mmu_pt *pt;
+ int mapped;
+
+ pt = pgt->pt[type];
+ mapped = desc->func->hmm_map(it->vmm, pt, ptei, ptes,
+ &it->pages[(it->addr - it->start) >> PAGE_SHIFT]);
+ if (mapped <= 0)
+ return false;
+ ptes = mapped;
+
+ /* Take PTE references. */
+ pgt->refs[type] += ptes;
+
+ /* Dual-PTs need special handling. */
+ if (desc->type == SPT)
+ nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
+
+ return true;
+}
+
static bool
nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
{
@@ -520,6 +599,7 @@ nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
it.cnt = size >> page->shift;
it.flush = NVKM_VMM_LEVELS_MAX;
it.start = it.addr = addr;
+ it.pages = map ? map->pages : NULL;
/* Deconstruct address into PTE indices for each mapping level. */
for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
@@ -1184,6 +1264,29 @@ nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
return ret;
}
+void
+nvkm_vmm_hmm_map(struct nvkm_vmm *vmm, u64 addr, u64 npages, u64 *pages)
+{
+ struct nvkm_vmm_map map = {0};
+
+ for (map.page = vmm->func->page; map.page->shift != 12; map.page++);
+ map.pages = pages;
+
+ nvkm_vmm_iter(vmm, map.page, addr, npages << PAGE_SHIFT, "ref + map",
+ true, nvkm_vmm_ref_hmm_ptes, NULL, &map, NULL);
+}
+
+void
+nvkm_vmm_hmm_unmap(struct nvkm_vmm *vmm, u64 addr, u64 npages)
+{
+ struct nvkm_vmm_map map = {0};
+
+ for (map.page = vmm->func->page; map.page->shift != 12; map.page++);
+
+ nvkm_vmm_iter(vmm, map.page, addr, npages << PAGE_SHIFT, "unmap + unref",
+ false, nvkm_vmm_unref_hmm_ptes, NULL, NULL, NULL);
+}
+
static void
nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
{
@@ -56,6 +56,8 @@ typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *,
struct nvkm_vmm_pt *, u32 pdei);
typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *,
u32 ptei, u32 ptes, struct nvkm_vmm_map *);
+typedef int (*nvkm_vmm_hmm_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *,
+ u32 ptei, u32 ptes, u64 *pages);
struct nvkm_vmm_desc_func {
nvkm_vmm_pxe_func invalid;
@@ -67,6 +69,8 @@ struct nvkm_vmm_desc_func {
nvkm_vmm_pte_func mem;
nvkm_vmm_pte_func dma;
nvkm_vmm_pte_func sgl;
+ nvkm_vmm_hmm_func hmm_map;
+ nvkm_vmm_hmm_func hmm_unmap;
};
extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;
@@ -163,6 +167,8 @@ int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *);
void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma);
+void nvkm_vmm_hmm_map(struct nvkm_vmm *vmm, u64 addr, u64 npages, u64 *pages);
+void nvkm_vmm_hmm_unmap(struct nvkm_vmm *vmm, u64 addr, u64 npages);
struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *);
@@ -67,6 +67,77 @@ gp100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte);
}
+static int
+gp100_vmm_pgt_hmm_map(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, u64 *pages)
+{
+ int mapped = 0;
+
+ nvkm_kmap(pt->memory);
+ while (ptes--) {
+ u64 data = nvkm_ro64(pt->memory, pt->base + ptei * 8);
+ u64 page = *pages;
+ struct page *tmp;
+ dma_addr_t dma;
+
+ if (!(page & NV_HMM_PAGE_FLAG_V)) {
+ pages++; ptei++;
+ continue;
+ }
+
+ if ((data & 1)) {
+ *pages |= NV_HMM_PAGE_FLAG_V;
+ pages++; ptei++;
+ continue;
+ }
+
+ tmp = pfn_to_page(page >> NV_HMM_PAGE_PFN_SHIFT);
+ dma = dma_map_page(vmm->mmu->subdev.device->dev, tmp,
+ 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(vmm->mmu->subdev.device->dev, dma)) {
+ *pages |= NV_HMM_PAGE_FLAG_E;
+ pages++; ptei++;
+ continue;
+ }
+
+ data = (2 << 1);
+ data |= ((dma >> PAGE_SHIFT) << 8);
+ data |= page & NV_HMM_PAGE_FLAG_V ? (1 << 0) : 0;
+ data |= page & NV_HMM_PAGE_FLAG_W ? 0 : (1 << 6);
+
+ VMM_WO064(pt, vmm, ptei++ * 8, data);
+ mapped++;
+ pages++;
+ }
+ nvkm_done(pt->memory);
+
+ return mapped;
+}
+
+static int
+gp100_vmm_pgt_hmm_unmap(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, u64 *pages)
+{
+ int unmapped = 0;
+
+ nvkm_kmap(pt->memory);
+ while (ptes--) {
+ u64 data = nvkm_ro64(pt->memory, pt->base + ptei * 8);
+
+ if (!(data & 1)) {
+ VMM_WO064(pt, vmm, ptei++ * 8, 0UL);
+ continue;
+ }
+
+ /* Clear valid but keep pte value so we can dma_unmap() */
+ VMM_WO064(pt, vmm, ptei++ * 8, data ^ 1);
+ unmapped++;
+ }
+ nvkm_done(pt->memory);
+
+ return unmapped;
+}
+
static void
gp100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
@@ -89,6 +160,8 @@ gp100_vmm_desc_spt = {
.mem = gp100_vmm_pgt_mem,
.dma = gp100_vmm_pgt_dma,
.sgl = gp100_vmm_pgt_sgl,
+ .hmm_map = gp100_vmm_pgt_hmm_map,
+ .hmm_unmap = gp100_vmm_pgt_hmm_unmap,
};
static void