84 pd->
pg[0].shift = shift;
86 err = percpu_counter_init(&pd->
pg[0].events, 0);
90 err = percpu_counter_init(&pd->
pg[1].events, 0);
117 index = pd->
index ^ 1;
118 offset = pd->
pg[pd->
index].shift - shift;
125 events = percpu_counter_sum(&pd->
pg[pd->
index].events);
170 prop_adjust_shift(
int *pl_shift,
unsigned long *pl_period,
int new_shift)
172 int offset = *pl_shift - new_shift;
182 *pl_shift = new_shift;
189 #define PROP_BATCH (8*(1+ilog2(nr_cpu_ids)))
196 return percpu_counter_init(&pl->
events, 0);
215 unsigned long period_mask = ~(period - 1);
216 unsigned long global_period;
219 global_period = percpu_counter_read(&pg->
events);
220 global_period &= period_mask;
226 if (pl->
period == global_period)
237 period = (global_period - pl->
period) >> (pg->
shift - 1);
242 val = percpu_counter_sum(&pl->
events);
249 pl->
period = global_period;
260 prop_norm_percpu(pg, pl);
262 percpu_counter_add(&pg->
events, 1);
263 prop_put_global(pd, pg);
275 prop_norm_percpu(pg, pl);
278 unsigned long period_2 = 1
UL << (pg->
shift - 1);
279 unsigned long counter_mask = period_2 - 1;
280 unsigned long global_count;
281 long numerator, denominator;
283 numerator = percpu_counter_read_positive(&pl->
events);
284 global_count = percpu_counter_read(&pg->
events);
285 denominator = period_2 + (global_count & counter_mask);
291 percpu_counter_add(&pl->
events, 1);
292 percpu_counter_add(&pg->
events, 1);
295 prop_put_global(pd, pg);
305 long *numerator,
long *denominator)
308 unsigned long period_2 = 1
UL << (pg->
shift - 1);
309 unsigned long counter_mask = period_2 - 1;
310 unsigned long global_count;
312 prop_norm_percpu(pg, pl);
313 *numerator = percpu_counter_read_positive(&pl->
events);
315 global_count = percpu_counter_read(&pg->
events);
316 *denominator = period_2 + (global_count & counter_mask);
318 prop_put_global(pd, pg);
344 unsigned long period = 1
UL << (pg->
shift - 1);
345 unsigned long period_mask = ~(period - 1);
346 unsigned long global_period;
349 global_period = percpu_counter_read(&pg->
events);
350 global_period &= period_mask;
356 if (pl->
period == global_period)
364 period = (global_period - pl->
period) >> (pg->
shift - 1);
369 pl->
period = global_period;
380 prop_norm_single(pg, pl);
382 percpu_counter_add(&pg->
events, 1);
383 prop_put_global(pd, pg);
393 long *numerator,
long *denominator)
396 unsigned long period_2 = 1
UL << (pg->
shift - 1);
397 unsigned long counter_mask = period_2 - 1;
398 unsigned long global_count;
400 prop_norm_single(pg, pl);
403 global_count = percpu_counter_read(&pg->
events);
404 *denominator = period_2 + (global_count & counter_mask);
406 prop_put_global(pd, pg);