projects / openwrt / staging / jow.git / blob
? search:


e0d12eb8bc13778d29d17dc00fc708cd4f018236
[openwrt/staging/jow.git] /
1 From 3e471b58fb766ed5b45ff8b560231da4704c946d Mon Sep 17 00:00:00 2001
2 From: Sugizaki Yukimasa <i.can.speak.c.and.basic@gmail.com>
3 Date: Wed, 10 Jan 2018 06:25:51 +0900
4 Subject: [PATCH 156/454] vcsm: Revert to do page-table-walk-based cache
5 manipulating on some ioctl calls
6
7 On FLUSH, INVALID, CLEAN_INVALID ioctl calls, cache operations based on
8 page table walk were used in case that the buffer of the cache is not
9 pinned. So reverted to do page-table-based cache manipulating.
10
11 Signed-off-by: Sugizaki Yukimasa <i.can.speak.c.and.basic@gmail.com>
12 ---
13 drivers/char/broadcom/vc_sm/vmcs_sm.c | 141 ++++++++++++++++++++------
14 1 file changed, 110 insertions(+), 31 deletions(-)
15
16 --- a/drivers/char/broadcom/vc_sm/vmcs_sm.c
17 +++ b/drivers/char/broadcom/vc_sm/vmcs_sm.c
18 @@ -1256,7 +1256,33 @@ static const struct vm_operations_struct
19 .fault = vcsm_vma_fault,
20 };
21
22 -static int clean_invalid_mem_2d(const void __user *addr,
23 +/* Converts VCSM_CACHE_OP_* to an operating function. */
24 +static void (*cache_op_to_func(const unsigned cache_op))
25 + (const void*, const void*)
26 +{
27 + switch (cache_op) {
28 + case VCSM_CACHE_OP_NOP:
29 + return NULL;
30 +
31 + case VCSM_CACHE_OP_INV:
32 + return dmac_inv_range;
33 +
34 + case VCSM_CACHE_OP_CLEAN:
35 + return dmac_clean_range;
36 +
37 + case VCSM_CACHE_OP_FLUSH:
38 + return dmac_flush_range;
39 +
40 + default:
41 + pr_err("[%s]: Invalid cache_op: 0x%08x\n", __func__, cache_op);
42 + return NULL;
43 + }
44 +}
45 +
46 +/*
47 + * Clean/invalid/flush cache of which buffer is already pinned (i.e. accessed).
48 + */
49 +static int clean_invalid_contiguous_mem_2d(const void __user *addr,
50 const size_t block_count, const size_t block_size, const size_t stride,
51 const unsigned cache_op)
52 {
53 @@ -1268,22 +1294,9 @@ static int clean_invalid_mem_2d(const vo
54 return -EINVAL;
55 }
56
57 - switch (cache_op) {
58 - case VCSM_CACHE_OP_NOP:
59 - return 0;
60 - case VCSM_CACHE_OP_INV:
61 - op_fn = dmac_inv_range;
62 - break;
63 - case VCSM_CACHE_OP_CLEAN:
64 - op_fn = dmac_clean_range;
65 - break;
66 - case VCSM_CACHE_OP_FLUSH:
67 - op_fn = dmac_flush_range;
68 - break;
69 - default:
70 - pr_err("[%s]: Invalid cache_op: 0x%08x\n", __func__, cache_op);
71 + op_fn = cache_op_to_func(cache_op);
72 + if (op_fn == NULL)
73 return -EINVAL;
74 - }
75
76 for (i = 0; i < block_count; i ++, addr += stride)
77 op_fn(addr, addr + block_size);
78 @@ -1291,14 +1304,73 @@ static int clean_invalid_mem_2d(const vo
79 return 0;
80 }
81
82 -static int clean_invalid_mem(const void __user *addr, const size_t size,
83 +/* Clean/invalid/flush cache of which buffer may be non-pinned. */
84 +/* The caller must lock current->mm->mmap_sem for read. */
85 +static int clean_invalid_mem_walk(unsigned long addr, const size_t size,
86 const unsigned cache_op)
87 {
88 - return clean_invalid_mem_2d(addr, 1, size, 0, cache_op);
89 + pgd_t *pgd;
90 + pud_t *pud;
91 + pmd_t *pmd;
92 + pte_t *pte;
93 + unsigned long pgd_next, pud_next, pmd_next;
94 + const unsigned long end = ALIGN(addr + size, PAGE_SIZE);
95 + void (*op_fn)(const void*, const void*);
96 +
97 + addr &= PAGE_MASK;
98 +
99 + if (addr >= end)
100 + return 0;
101 +
102 + op_fn = cache_op_to_func(cache_op);
103 + if (op_fn == NULL)
104 + return -EINVAL;
105 +
106 + /* Walk PGD */
107 + pgd = pgd_offset(current->mm, addr);
108 + do {
109 + pgd_next = pgd_addr_end(addr, end);
110 +
111 + if (pgd_none(*pgd) || pgd_bad(*pgd))
112 + continue;
113 +
114 + /* Walk PUD */
115 + pud = pud_offset(pgd, addr);
116 + do {
117 + pud_next = pud_addr_end(addr, pgd_next);
118 + if (pud_none(*pud) || pud_bad(*pud))
119 + continue;
120 +
121 + /* Walk PMD */
122 + pmd = pmd_offset(pud, addr);
123 + do {
124 + pmd_next = pmd_addr_end(addr, pud_next);
125 + if (pmd_none(*pmd) || pmd_bad(*pmd))
126 + continue;
127 +
128 + /* Walk PTE */
129 + pte = pte_offset_map(pmd, addr);
130 + do {
131 + if (pte_none(*pte) || !pte_present(*pte))
132 + continue;
133 +
134 + op_fn((const void __user*) addr,
135 + (const void __user*) (addr + PAGE_SIZE));
136 + } while (pte++, addr += PAGE_SIZE, addr != pmd_next);
137 + pte_unmap(pte);
138 +
139 + } while (pmd++, addr = pmd_next, addr != pud_next);
140 +
141 + } while (pud++, addr = pud_next, addr != pgd_next);
142 +
143 + } while (pgd++, addr = pgd_next, addr != end);
144 +
145 + return 0;
146 }
147
148 -static int clean_invalid_resource(const void __user *addr, const size_t size,
149 - const unsigned cache_op, const int usr_hdl,
150 +/* Clean/invalid/flush cache of buffer in resource */
151 +static int clean_invalid_resource_walk(const void __user *addr,
152 + const size_t size, const unsigned cache_op, const int usr_hdl,
153 struct sm_resource_t *resource)
154 {
155 int err;
156 @@ -1355,7 +1427,10 @@ static int clean_invalid_resource(const
157 return -EFAULT;
158 }
159
160 - err = clean_invalid_mem(addr, size, cache_op);
161 + down_read(&current->mm->mmap_sem);
162 + err = clean_invalid_mem_walk((unsigned long) addr, size, cache_op);
163 + up_read(&current->mm->mmap_sem);
164 +
165 if (err)
166 resource->res_stats[stat_failure]++;
167
168 @@ -2004,7 +2079,7 @@ static int vc_sm_ioctl_unlock(struct sm_
169 const unsigned long start = map->vma->vm_start;
170 const unsigned long end = map->vma->vm_end;
171
172 - ret = clean_invalid_mem((void __user*) start, end - start,
173 + ret = clean_invalid_mem_walk(start, end - start,
174 VCSM_CACHE_OP_FLUSH);
175 if (ret)
176 goto error;
177 @@ -2886,7 +2961,7 @@ static long vc_sm_ioctl(struct file *fil
178 goto out;
179 }
180
181 - ret = clean_invalid_resource((void __user*) ioparam.addr,
182 + ret = clean_invalid_resource_walk((void __user*) ioparam.addr,
183 ioparam.size, VCSM_CACHE_OP_FLUSH, ioparam.handle,
184 resource);
185 vmcs_sm_release_resource(resource, 0);
186 @@ -2917,7 +2992,7 @@ static long vc_sm_ioctl(struct file *fil
187 goto out;
188 }
189
190 - ret = clean_invalid_resource((void __user*) ioparam.addr,
191 + ret = clean_invalid_resource_walk((void __user*) ioparam.addr,
192 ioparam.size, VCSM_CACHE_OP_INV, ioparam.handle, resource);
193 vmcs_sm_release_resource(resource, 0);
194 if (ret)
195 @@ -2951,16 +3026,19 @@ static long vc_sm_ioctl(struct file *fil
196 goto out;
197 }
198
199 - ret = clean_invalid_resource((void __user*) ioparam.s[i].addr,
200 - ioparam.s[i].size, ioparam.s[i].cmd,
201 - ioparam.s[i].handle, resource);
202 + ret = clean_invalid_resource_walk(
203 + (void __user*) ioparam.s[i].addr, ioparam.s[i].size,
204 + ioparam.s[i].cmd, ioparam.s[i].handle, resource);
205 vmcs_sm_release_resource(resource, 0);
206 if (ret)
207 goto out;
208 }
209 }
210 break;
211 - /* Flush/Invalidate the cache for a given mapping. */
212 + /*
213 + * Flush/Invalidate the cache for a given mapping.
214 + * Blocks must be pinned (i.e. accessed) before this call.
215 + */
216 case VMCS_SM_CMD_CLEAN_INVALID2:
217 {
218 int i;
219 @@ -2993,12 +3071,13 @@ static long vc_sm_ioctl(struct file *fil
220 for (i = 0; i < ioparam.op_count; i++) {
221 const struct vmcs_sm_ioctl_clean_invalid_block * const op = block + i;
222
223 - ret = clean_invalid_mem_2d((void __user*) op->start_address,
224 - op->block_count, op->block_size,
225 - op->inter_block_stride, op->invalidate_mode);
226 if (op->invalidate_mode == VCSM_CACHE_OP_NOP)
227 continue;
228
229 + ret = clean_invalid_contiguous_mem_2d(
230 + (void __user*) op->start_address, op->block_count,
231 + op->block_size, op->inter_block_stride,
232 + op->invalidate_mode);
233 if (ret)
234 break;
235 }
Staging tree of Jo-Philipp Wich