blob: 214806deca993809ebe9675c35fddbe5ac1178d6 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * kernel/cpuset.c
3 *
4 * Processor and Memory placement constraints for sets of tasks.
5 *
6 * Copyright (C) 2003 BULL SA.
7 * Copyright (C) 2004 Silicon Graphics, Inc.
8 *
9 * Portions derived from Patrick Mochel's sysfs code.
10 * sysfs is Copyright (c) 2001-3 Patrick Mochel
11 * Portions Copyright (c) 2004 Silicon Graphics, Inc.
12 *
13 * 2003-10-10 Written by Simon Derr <simon.derr@bull.net>
14 * 2003-10-22 Updates by Stephen Hemminger.
15 * 2004 May-July Rework by Paul Jackson <pj@sgi.com>
16 *
17 * This file is subject to the terms and conditions of the GNU General Public
18 * License. See the file COPYING in the main directory of the Linux
19 * distribution for more details.
20 */
21
22#include <linux/config.h>
23#include <linux/cpu.h>
24#include <linux/cpumask.h>
25#include <linux/cpuset.h>
26#include <linux/err.h>
27#include <linux/errno.h>
28#include <linux/file.h>
29#include <linux/fs.h>
30#include <linux/init.h>
31#include <linux/interrupt.h>
32#include <linux/kernel.h>
33#include <linux/kmod.h>
34#include <linux/list.h>
35#include <linux/mm.h>
36#include <linux/module.h>
37#include <linux/mount.h>
38#include <linux/namei.h>
39#include <linux/pagemap.h>
40#include <linux/proc_fs.h>
41#include <linux/sched.h>
42#include <linux/seq_file.h>
43#include <linux/slab.h>
44#include <linux/smp_lock.h>
45#include <linux/spinlock.h>
46#include <linux/stat.h>
47#include <linux/string.h>
48#include <linux/time.h>
49#include <linux/backing-dev.h>
50#include <linux/sort.h>
51
52#include <asm/uaccess.h>
53#include <asm/atomic.h>
54#include <asm/semaphore.h>
55
56#define CPUSET_SUPER_MAGIC 0x27e0eb
57
58struct cpuset {
59 unsigned long flags; /* "unsigned long" so bitops work */
60 cpumask_t cpus_allowed; /* CPUs allowed to tasks in cpuset */
61 nodemask_t mems_allowed; /* Memory Nodes allowed to tasks */
62
63 atomic_t count; /* count tasks using this cpuset */
64
65 /*
66 * We link our 'sibling' struct into our parents 'children'.
67 * Our children link their 'sibling' into our 'children'.
68 */
69 struct list_head sibling; /* my parents children */
70 struct list_head children; /* my children */
71
72 struct cpuset *parent; /* my parent */
73 struct dentry *dentry; /* cpuset fs entry */
74
75 /*
76 * Copy of global cpuset_mems_generation as of the most
77 * recent time this cpuset changed its mems_allowed.
78 */
79 int mems_generation;
80};
81
82/* bits in struct cpuset flags field */
83typedef enum {
84 CS_CPU_EXCLUSIVE,
85 CS_MEM_EXCLUSIVE,
86 CS_REMOVED,
87 CS_NOTIFY_ON_RELEASE
88} cpuset_flagbits_t;
89
90/* convenient tests for these bits */
91static inline int is_cpu_exclusive(const struct cpuset *cs)
92{
93 return !!test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
94}
95
96static inline int is_mem_exclusive(const struct cpuset *cs)
97{
98 return !!test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
99}
100
101static inline int is_removed(const struct cpuset *cs)
102{
103 return !!test_bit(CS_REMOVED, &cs->flags);
104}
105
106static inline int notify_on_release(const struct cpuset *cs)
107{
108 return !!test_bit(CS_NOTIFY_ON_RELEASE, &cs->flags);
109}
110
111/*
112 * Increment this atomic integer everytime any cpuset changes its
113 * mems_allowed value. Users of cpusets can track this generation
114 * number, and avoid having to lock and reload mems_allowed unless
115 * the cpuset they're using changes generation.
116 *
117 * A single, global generation is needed because attach_task() could
118 * reattach a task to a different cpuset, which must not have its
119 * generation numbers aliased with those of that tasks previous cpuset.
120 *
121 * Generations are needed for mems_allowed because one task cannot
122 * modify anothers memory placement. So we must enable every task,
123 * on every visit to __alloc_pages(), to efficiently check whether
124 * its current->cpuset->mems_allowed has changed, requiring an update
125 * of its current->mems_allowed.
126 */
127static atomic_t cpuset_mems_generation = ATOMIC_INIT(1);
128
129static struct cpuset top_cpuset = {
130 .flags = ((1 << CS_CPU_EXCLUSIVE) | (1 << CS_MEM_EXCLUSIVE)),
131 .cpus_allowed = CPU_MASK_ALL,
132 .mems_allowed = NODE_MASK_ALL,
133 .count = ATOMIC_INIT(0),
134 .sibling = LIST_HEAD_INIT(top_cpuset.sibling),
135 .children = LIST_HEAD_INIT(top_cpuset.children),
136 .parent = NULL,
137 .dentry = NULL,
138 .mems_generation = 0,
139};
140
141static struct vfsmount *cpuset_mount;
142static struct super_block *cpuset_sb = NULL;
143
144/*
145 * cpuset_sem should be held by anyone who is depending on the children
146 * or sibling lists of any cpuset, or performing non-atomic operations
147 * on the flags or *_allowed values of a cpuset, such as raising the
148 * CS_REMOVED flag bit iff it is not already raised, or reading and
149 * conditionally modifying the *_allowed values. One kernel global
150 * cpuset semaphore should be sufficient - these things don't change
151 * that much.
152 *
153 * The code that modifies cpusets holds cpuset_sem across the entire
154 * operation, from cpuset_common_file_write() down, single threading
155 * all cpuset modifications (except for counter manipulations from
156 * fork and exit) across the system. This presumes that cpuset
157 * modifications are rare - better kept simple and safe, even if slow.
158 *
159 * The code that reads cpusets, such as in cpuset_common_file_read()
160 * and below, only holds cpuset_sem across small pieces of code, such
161 * as when reading out possibly multi-word cpumasks and nodemasks, as
162 * the risks are less, and the desire for performance a little greater.
163 * The proc_cpuset_show() routine needs to hold cpuset_sem to insure
164 * that no cs->dentry is NULL, as it walks up the cpuset tree to root.
165 *
166 * The hooks from fork and exit, cpuset_fork() and cpuset_exit(), don't
167 * (usually) grab cpuset_sem. These are the two most performance
168 * critical pieces of code here. The exception occurs on exit(),
Paul Jackson2efe86b2005-05-27 02:02:43 -0700169 * when a task in a notify_on_release cpuset exits. Then cpuset_sem
170 * is taken, and if the cpuset count is zero, a usermode call made
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 * to /sbin/cpuset_release_agent with the name of the cpuset (path
172 * relative to the root of cpuset file system) as the argument.
173 *
174 * A cpuset can only be deleted if both its 'count' of using tasks is
175 * zero, and its list of 'children' cpusets is empty. Since all tasks
176 * in the system use _some_ cpuset, and since there is always at least
177 * one task in the system (init, pid == 1), therefore, top_cpuset
178 * always has either children cpusets and/or using tasks. So no need
179 * for any special hack to ensure that top_cpuset cannot be deleted.
180 */
181
182static DECLARE_MUTEX(cpuset_sem);
183
184/*
185 * A couple of forward declarations required, due to cyclic reference loop:
186 * cpuset_mkdir -> cpuset_create -> cpuset_populate_dir -> cpuset_add_file
187 * -> cpuset_create_file -> cpuset_dir_inode_operations -> cpuset_mkdir.
188 */
189
190static int cpuset_mkdir(struct inode *dir, struct dentry *dentry, int mode);
191static int cpuset_rmdir(struct inode *unused_dir, struct dentry *dentry);
192
193static struct backing_dev_info cpuset_backing_dev_info = {
194 .ra_pages = 0, /* No readahead */
195 .capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK,
196};
197
198static struct inode *cpuset_new_inode(mode_t mode)
199{
200 struct inode *inode = new_inode(cpuset_sb);
201
202 if (inode) {
203 inode->i_mode = mode;
204 inode->i_uid = current->fsuid;
205 inode->i_gid = current->fsgid;
206 inode->i_blksize = PAGE_CACHE_SIZE;
207 inode->i_blocks = 0;
208 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
209 inode->i_mapping->backing_dev_info = &cpuset_backing_dev_info;
210 }
211 return inode;
212}
213
214static void cpuset_diput(struct dentry *dentry, struct inode *inode)
215{
216 /* is dentry a directory ? if so, kfree() associated cpuset */
217 if (S_ISDIR(inode->i_mode)) {
218 struct cpuset *cs = dentry->d_fsdata;
219 BUG_ON(!(is_removed(cs)));
220 kfree(cs);
221 }
222 iput(inode);
223}
224
225static struct dentry_operations cpuset_dops = {
226 .d_iput = cpuset_diput,
227};
228
229static struct dentry *cpuset_get_dentry(struct dentry *parent, const char *name)
230{
Christoph Hellwig5f45f1a2005-06-23 00:09:12 -0700231 struct dentry *d = lookup_one_len(name, parent, strlen(name));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 if (!IS_ERR(d))
233 d->d_op = &cpuset_dops;
234 return d;
235}
236
237static void remove_dir(struct dentry *d)
238{
239 struct dentry *parent = dget(d->d_parent);
240
241 d_delete(d);
242 simple_rmdir(parent->d_inode, d);
243 dput(parent);
244}
245
246/*
247 * NOTE : the dentry must have been dget()'ed
248 */
249static void cpuset_d_remove_dir(struct dentry *dentry)
250{
251 struct list_head *node;
252
253 spin_lock(&dcache_lock);
254 node = dentry->d_subdirs.next;
255 while (node != &dentry->d_subdirs) {
256 struct dentry *d = list_entry(node, struct dentry, d_child);
257 list_del_init(node);
258 if (d->d_inode) {
259 d = dget_locked(d);
260 spin_unlock(&dcache_lock);
261 d_delete(d);
262 simple_unlink(dentry->d_inode, d);
263 dput(d);
264 spin_lock(&dcache_lock);
265 }
266 node = dentry->d_subdirs.next;
267 }
268 list_del_init(&dentry->d_child);
269 spin_unlock(&dcache_lock);
270 remove_dir(dentry);
271}
272
273static struct super_operations cpuset_ops = {
274 .statfs = simple_statfs,
275 .drop_inode = generic_delete_inode,
276};
277
278static int cpuset_fill_super(struct super_block *sb, void *unused_data,
279 int unused_silent)
280{
281 struct inode *inode;
282 struct dentry *root;
283
284 sb->s_blocksize = PAGE_CACHE_SIZE;
285 sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
286 sb->s_magic = CPUSET_SUPER_MAGIC;
287 sb->s_op = &cpuset_ops;
288 cpuset_sb = sb;
289
290 inode = cpuset_new_inode(S_IFDIR | S_IRUGO | S_IXUGO | S_IWUSR);
291 if (inode) {
292 inode->i_op = &simple_dir_inode_operations;
293 inode->i_fop = &simple_dir_operations;
294 /* directories start off with i_nlink == 2 (for "." entry) */
295 inode->i_nlink++;
296 } else {
297 return -ENOMEM;
298 }
299
300 root = d_alloc_root(inode);
301 if (!root) {
302 iput(inode);
303 return -ENOMEM;
304 }
305 sb->s_root = root;
306 return 0;
307}
308
309static struct super_block *cpuset_get_sb(struct file_system_type *fs_type,
310 int flags, const char *unused_dev_name,
311 void *data)
312{
313 return get_sb_single(fs_type, flags, data, cpuset_fill_super);
314}
315
316static struct file_system_type cpuset_fs_type = {
317 .name = "cpuset",
318 .get_sb = cpuset_get_sb,
319 .kill_sb = kill_litter_super,
320};
321
322/* struct cftype:
323 *
324 * The files in the cpuset filesystem mostly have a very simple read/write
325 * handling, some common function will take care of it. Nevertheless some cases
326 * (read tasks) are special and therefore I define this structure for every
327 * kind of file.
328 *
329 *
330 * When reading/writing to a file:
331 * - the cpuset to use in file->f_dentry->d_parent->d_fsdata
332 * - the 'cftype' of the file is file->f_dentry->d_fsdata
333 */
334
335struct cftype {
336 char *name;
337 int private;
338 int (*open) (struct inode *inode, struct file *file);
339 ssize_t (*read) (struct file *file, char __user *buf, size_t nbytes,
340 loff_t *ppos);
341 int (*write) (struct file *file, const char __user *buf, size_t nbytes,
342 loff_t *ppos);
343 int (*release) (struct inode *inode, struct file *file);
344};
345
346static inline struct cpuset *__d_cs(struct dentry *dentry)
347{
348 return dentry->d_fsdata;
349}
350
351static inline struct cftype *__d_cft(struct dentry *dentry)
352{
353 return dentry->d_fsdata;
354}
355
356/*
357 * Call with cpuset_sem held. Writes path of cpuset into buf.
358 * Returns 0 on success, -errno on error.
359 */
360
361static int cpuset_path(const struct cpuset *cs, char *buf, int buflen)
362{
363 char *start;
364
365 start = buf + buflen;
366
367 *--start = '\0';
368 for (;;) {
369 int len = cs->dentry->d_name.len;
370 if ((start -= len) < buf)
371 return -ENAMETOOLONG;
372 memcpy(start, cs->dentry->d_name.name, len);
373 cs = cs->parent;
374 if (!cs)
375 break;
376 if (!cs->parent)
377 continue;
378 if (--start < buf)
379 return -ENAMETOOLONG;
380 *start = '/';
381 }
382 memmove(buf, start, buf + buflen - start);
383 return 0;
384}
385
386/*
387 * Notify userspace when a cpuset is released, by running
388 * /sbin/cpuset_release_agent with the name of the cpuset (path
389 * relative to the root of cpuset file system) as the argument.
390 *
391 * Most likely, this user command will try to rmdir this cpuset.
392 *
393 * This races with the possibility that some other task will be
394 * attached to this cpuset before it is removed, or that some other
395 * user task will 'mkdir' a child cpuset of this cpuset. That's ok.
396 * The presumed 'rmdir' will fail quietly if this cpuset is no longer
397 * unused, and this cpuset will be reprieved from its death sentence,
398 * to continue to serve a useful existence. Next time it's released,
399 * we will get notified again, if it still has 'notify_on_release' set.
400 *
Paul Jackson3077a262005-08-09 10:07:59 -0700401 * The final arg to call_usermodehelper() is 0, which means don't
402 * wait. The separate /sbin/cpuset_release_agent task is forked by
403 * call_usermodehelper(), then control in this thread returns here,
404 * without waiting for the release agent task. We don't bother to
405 * wait because the caller of this routine has no use for the exit
406 * status of the /sbin/cpuset_release_agent task, so no sense holding
407 * our caller up for that.
408 *
409 * The simple act of forking that task might require more memory,
410 * which might need cpuset_sem. So this routine must be called while
411 * cpuset_sem is not held, to avoid a possible deadlock. See also
412 * comments for check_for_release(), below.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 */
414
Paul Jackson3077a262005-08-09 10:07:59 -0700415static void cpuset_release_agent(const char *pathbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416{
417 char *argv[3], *envp[3];
418 int i;
419
Paul Jackson3077a262005-08-09 10:07:59 -0700420 if (!pathbuf)
421 return;
422
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 i = 0;
424 argv[i++] = "/sbin/cpuset_release_agent";
Paul Jackson3077a262005-08-09 10:07:59 -0700425 argv[i++] = (char *)pathbuf;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 argv[i] = NULL;
427
428 i = 0;
429 /* minimal command environment */
430 envp[i++] = "HOME=/";
431 envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
432 envp[i] = NULL;
433
Paul Jackson3077a262005-08-09 10:07:59 -0700434 call_usermodehelper(argv[0], argv, envp, 0);
435 kfree(pathbuf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436}
437
438/*
439 * Either cs->count of using tasks transitioned to zero, or the
440 * cs->children list of child cpusets just became empty. If this
441 * cs is notify_on_release() and now both the user count is zero and
Paul Jackson3077a262005-08-09 10:07:59 -0700442 * the list of children is empty, prepare cpuset path in a kmalloc'd
443 * buffer, to be returned via ppathbuf, so that the caller can invoke
444 * cpuset_release_agent() with it later on, once cpuset_sem is dropped.
445 * Call here with cpuset_sem held.
446 *
447 * This check_for_release() routine is responsible for kmalloc'ing
448 * pathbuf. The above cpuset_release_agent() is responsible for
449 * kfree'ing pathbuf. The caller of these routines is responsible
450 * for providing a pathbuf pointer, initialized to NULL, then
451 * calling check_for_release() with cpuset_sem held and the address
452 * of the pathbuf pointer, then dropping cpuset_sem, then calling
453 * cpuset_release_agent() with pathbuf, as set by check_for_release().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 */
455
Paul Jackson3077a262005-08-09 10:07:59 -0700456static void check_for_release(struct cpuset *cs, char **ppathbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457{
458 if (notify_on_release(cs) && atomic_read(&cs->count) == 0 &&
459 list_empty(&cs->children)) {
460 char *buf;
461
462 buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
463 if (!buf)
464 return;
465 if (cpuset_path(cs, buf, PAGE_SIZE) < 0)
Paul Jackson3077a262005-08-09 10:07:59 -0700466 kfree(buf);
467 else
468 *ppathbuf = buf;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 }
470}
471
472/*
473 * Return in *pmask the portion of a cpusets's cpus_allowed that
474 * are online. If none are online, walk up the cpuset hierarchy
475 * until we find one that does have some online cpus. If we get
476 * all the way to the top and still haven't found any online cpus,
477 * return cpu_online_map. Or if passed a NULL cs from an exit'ing
478 * task, return cpu_online_map.
479 *
480 * One way or another, we guarantee to return some non-empty subset
481 * of cpu_online_map.
482 *
483 * Call with cpuset_sem held.
484 */
485
486static void guarantee_online_cpus(const struct cpuset *cs, cpumask_t *pmask)
487{
488 while (cs && !cpus_intersects(cs->cpus_allowed, cpu_online_map))
489 cs = cs->parent;
490 if (cs)
491 cpus_and(*pmask, cs->cpus_allowed, cpu_online_map);
492 else
493 *pmask = cpu_online_map;
494 BUG_ON(!cpus_intersects(*pmask, cpu_online_map));
495}
496
497/*
498 * Return in *pmask the portion of a cpusets's mems_allowed that
499 * are online. If none are online, walk up the cpuset hierarchy
500 * until we find one that does have some online mems. If we get
501 * all the way to the top and still haven't found any online mems,
502 * return node_online_map.
503 *
504 * One way or another, we guarantee to return some non-empty subset
505 * of node_online_map.
506 *
507 * Call with cpuset_sem held.
508 */
509
510static void guarantee_online_mems(const struct cpuset *cs, nodemask_t *pmask)
511{
512 while (cs && !nodes_intersects(cs->mems_allowed, node_online_map))
513 cs = cs->parent;
514 if (cs)
515 nodes_and(*pmask, cs->mems_allowed, node_online_map);
516 else
517 *pmask = node_online_map;
518 BUG_ON(!nodes_intersects(*pmask, node_online_map));
519}
520
521/*
522 * Refresh current tasks mems_allowed and mems_generation from
523 * current tasks cpuset. Call with cpuset_sem held.
524 *
525 * Be sure to call refresh_mems() on any cpuset operation which
526 * (1) holds cpuset_sem, and (2) might possibly alloc memory.
527 * Call after obtaining cpuset_sem lock, before any possible
528 * allocation. Otherwise one risks trying to allocate memory
529 * while the task cpuset_mems_generation is not the same as
530 * the mems_generation in its cpuset, which would deadlock on
531 * cpuset_sem in cpuset_update_current_mems_allowed().
532 *
533 * Since we hold cpuset_sem, once refresh_mems() is called, the
534 * test (current->cpuset_mems_generation != cs->mems_generation)
535 * in cpuset_update_current_mems_allowed() will remain false,
536 * until we drop cpuset_sem. Anyone else who would change our
537 * cpusets mems_generation needs to lock cpuset_sem first.
538 */
539
540static void refresh_mems(void)
541{
542 struct cpuset *cs = current->cpuset;
543
544 if (current->cpuset_mems_generation != cs->mems_generation) {
545 guarantee_online_mems(cs, &current->mems_allowed);
546 current->cpuset_mems_generation = cs->mems_generation;
547 }
548}
549
550/*
551 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
552 *
553 * One cpuset is a subset of another if all its allowed CPUs and
554 * Memory Nodes are a subset of the other, and its exclusive flags
555 * are only set if the other's are set.
556 */
557
558static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
559{
560 return cpus_subset(p->cpus_allowed, q->cpus_allowed) &&
561 nodes_subset(p->mems_allowed, q->mems_allowed) &&
562 is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
563 is_mem_exclusive(p) <= is_mem_exclusive(q);
564}
565
566/*
567 * validate_change() - Used to validate that any proposed cpuset change
568 * follows the structural rules for cpusets.
569 *
570 * If we replaced the flag and mask values of the current cpuset
571 * (cur) with those values in the trial cpuset (trial), would
572 * our various subset and exclusive rules still be valid? Presumes
573 * cpuset_sem held.
574 *
575 * 'cur' is the address of an actual, in-use cpuset. Operations
576 * such as list traversal that depend on the actual address of the
577 * cpuset in the list must use cur below, not trial.
578 *
579 * 'trial' is the address of bulk structure copy of cur, with
580 * perhaps one or more of the fields cpus_allowed, mems_allowed,
581 * or flags changed to new, trial values.
582 *
583 * Return 0 if valid, -errno if not.
584 */
585
586static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
587{
588 struct cpuset *c, *par;
589
590 /* Each of our child cpusets must be a subset of us */
591 list_for_each_entry(c, &cur->children, sibling) {
592 if (!is_cpuset_subset(c, trial))
593 return -EBUSY;
594 }
595
596 /* Remaining checks don't apply to root cpuset */
597 if ((par = cur->parent) == NULL)
598 return 0;
599
600 /* We must be a subset of our parent cpuset */
601 if (!is_cpuset_subset(trial, par))
602 return -EACCES;
603
604 /* If either I or some sibling (!= me) is exclusive, we can't overlap */
605 list_for_each_entry(c, &par->children, sibling) {
606 if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
607 c != cur &&
608 cpus_intersects(trial->cpus_allowed, c->cpus_allowed))
609 return -EINVAL;
610 if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
611 c != cur &&
612 nodes_intersects(trial->mems_allowed, c->mems_allowed))
613 return -EINVAL;
614 }
615
616 return 0;
617}
618
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700619/*
620 * For a given cpuset cur, partition the system as follows
621 * a. All cpus in the parent cpuset's cpus_allowed that are not part of any
622 * exclusive child cpusets
623 * b. All cpus in the current cpuset's cpus_allowed that are not part of any
624 * exclusive child cpusets
625 * Build these two partitions by calling partition_sched_domains
626 *
627 * Call with cpuset_sem held. May nest a call to the
628 * lock_cpu_hotplug()/unlock_cpu_hotplug() pair.
629 */
Paul Jackson212d6d22005-08-25 12:47:56 -0700630
631/*
632 * Hack to avoid 2.6.13 partial node dynamic sched domain bug.
633 * Disable letting 'cpu_exclusive' cpusets define dynamic sched
634 * domains, until the sched domain can handle partial nodes.
635 * Remove this #if hackery when sched domains fixed.
636 */
637#if 0
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700638static void update_cpu_domains(struct cpuset *cur)
639{
640 struct cpuset *c, *par = cur->parent;
641 cpumask_t pspan, cspan;
642
643 if (par == NULL || cpus_empty(cur->cpus_allowed))
644 return;
645
646 /*
647 * Get all cpus from parent's cpus_allowed not part of exclusive
648 * children
649 */
650 pspan = par->cpus_allowed;
651 list_for_each_entry(c, &par->children, sibling) {
652 if (is_cpu_exclusive(c))
653 cpus_andnot(pspan, pspan, c->cpus_allowed);
654 }
655 if (is_removed(cur) || !is_cpu_exclusive(cur)) {
656 cpus_or(pspan, pspan, cur->cpus_allowed);
657 if (cpus_equal(pspan, cur->cpus_allowed))
658 return;
659 cspan = CPU_MASK_NONE;
660 } else {
661 if (cpus_empty(pspan))
662 return;
663 cspan = cur->cpus_allowed;
664 /*
665 * Get all cpus from current cpuset's cpus_allowed not part
666 * of exclusive children
667 */
668 list_for_each_entry(c, &cur->children, sibling) {
669 if (is_cpu_exclusive(c))
670 cpus_andnot(cspan, cspan, c->cpus_allowed);
671 }
672 }
673
674 lock_cpu_hotplug();
675 partition_sched_domains(&pspan, &cspan);
676 unlock_cpu_hotplug();
677}
Paul Jackson212d6d22005-08-25 12:47:56 -0700678#else
679static void update_cpu_domains(struct cpuset *cur)
680{
681}
682#endif
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700683
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684static int update_cpumask(struct cpuset *cs, char *buf)
685{
686 struct cpuset trialcs;
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700687 int retval, cpus_unchanged;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688
689 trialcs = *cs;
690 retval = cpulist_parse(buf, trialcs.cpus_allowed);
691 if (retval < 0)
692 return retval;
693 cpus_and(trialcs.cpus_allowed, trialcs.cpus_allowed, cpu_online_map);
694 if (cpus_empty(trialcs.cpus_allowed))
695 return -ENOSPC;
696 retval = validate_change(cs, &trialcs);
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700697 if (retval < 0)
698 return retval;
699 cpus_unchanged = cpus_equal(cs->cpus_allowed, trialcs.cpus_allowed);
700 cs->cpus_allowed = trialcs.cpus_allowed;
701 if (is_cpu_exclusive(cs) && !cpus_unchanged)
702 update_cpu_domains(cs);
703 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704}
705
706static int update_nodemask(struct cpuset *cs, char *buf)
707{
708 struct cpuset trialcs;
709 int retval;
710
711 trialcs = *cs;
712 retval = nodelist_parse(buf, trialcs.mems_allowed);
713 if (retval < 0)
714 return retval;
715 nodes_and(trialcs.mems_allowed, trialcs.mems_allowed, node_online_map);
716 if (nodes_empty(trialcs.mems_allowed))
717 return -ENOSPC;
718 retval = validate_change(cs, &trialcs);
719 if (retval == 0) {
720 cs->mems_allowed = trialcs.mems_allowed;
721 atomic_inc(&cpuset_mems_generation);
722 cs->mems_generation = atomic_read(&cpuset_mems_generation);
723 }
724 return retval;
725}
726
727/*
728 * update_flag - read a 0 or a 1 in a file and update associated flag
729 * bit: the bit to update (CS_CPU_EXCLUSIVE, CS_MEM_EXCLUSIVE,
730 * CS_NOTIFY_ON_RELEASE)
731 * cs: the cpuset to update
732 * buf: the buffer where we read the 0 or 1
733 */
734
735static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, char *buf)
736{
737 int turning_on;
738 struct cpuset trialcs;
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700739 int err, cpu_exclusive_changed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740
741 turning_on = (simple_strtoul(buf, NULL, 10) != 0);
742
743 trialcs = *cs;
744 if (turning_on)
745 set_bit(bit, &trialcs.flags);
746 else
747 clear_bit(bit, &trialcs.flags);
748
749 err = validate_change(cs, &trialcs);
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -0700750 if (err < 0)
751 return err;
752 cpu_exclusive_changed =
753 (is_cpu_exclusive(cs) != is_cpu_exclusive(&trialcs));
754 if (turning_on)
755 set_bit(bit, &cs->flags);
756 else
757 clear_bit(bit, &cs->flags);
758
759 if (cpu_exclusive_changed)
760 update_cpu_domains(cs);
761 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762}
763
Paul Jackson3077a262005-08-09 10:07:59 -0700764static int attach_task(struct cpuset *cs, char *pidbuf, char **ppathbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765{
766 pid_t pid;
767 struct task_struct *tsk;
768 struct cpuset *oldcs;
769 cpumask_t cpus;
770
Paul Jackson3077a262005-08-09 10:07:59 -0700771 if (sscanf(pidbuf, "%d", &pid) != 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 return -EIO;
773 if (cpus_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))
774 return -ENOSPC;
775
776 if (pid) {
777 read_lock(&tasklist_lock);
778
779 tsk = find_task_by_pid(pid);
780 if (!tsk) {
781 read_unlock(&tasklist_lock);
782 return -ESRCH;
783 }
784
785 get_task_struct(tsk);
786 read_unlock(&tasklist_lock);
787
788 if ((current->euid) && (current->euid != tsk->uid)
789 && (current->euid != tsk->suid)) {
790 put_task_struct(tsk);
791 return -EACCES;
792 }
793 } else {
794 tsk = current;
795 get_task_struct(tsk);
796 }
797
798 task_lock(tsk);
799 oldcs = tsk->cpuset;
800 if (!oldcs) {
801 task_unlock(tsk);
802 put_task_struct(tsk);
803 return -ESRCH;
804 }
805 atomic_inc(&cs->count);
806 tsk->cpuset = cs;
807 task_unlock(tsk);
808
809 guarantee_online_cpus(cs, &cpus);
810 set_cpus_allowed(tsk, cpus);
811
812 put_task_struct(tsk);
813 if (atomic_dec_and_test(&oldcs->count))
Paul Jackson3077a262005-08-09 10:07:59 -0700814 check_for_release(oldcs, ppathbuf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 return 0;
816}
817
818/* The various types of files and directories in a cpuset file system */
819
820typedef enum {
821 FILE_ROOT,
822 FILE_DIR,
823 FILE_CPULIST,
824 FILE_MEMLIST,
825 FILE_CPU_EXCLUSIVE,
826 FILE_MEM_EXCLUSIVE,
827 FILE_NOTIFY_ON_RELEASE,
828 FILE_TASKLIST,
829} cpuset_filetype_t;
830
831static ssize_t cpuset_common_file_write(struct file *file, const char __user *userbuf,
832 size_t nbytes, loff_t *unused_ppos)
833{
834 struct cpuset *cs = __d_cs(file->f_dentry->d_parent);
835 struct cftype *cft = __d_cft(file->f_dentry);
836 cpuset_filetype_t type = cft->private;
837 char *buffer;
Paul Jackson3077a262005-08-09 10:07:59 -0700838 char *pathbuf = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 int retval = 0;
840
841 /* Crude upper limit on largest legitimate cpulist user might write. */
842 if (nbytes > 100 + 6 * NR_CPUS)
843 return -E2BIG;
844
845 /* +1 for nul-terminator */
846 if ((buffer = kmalloc(nbytes + 1, GFP_KERNEL)) == 0)
847 return -ENOMEM;
848
849 if (copy_from_user(buffer, userbuf, nbytes)) {
850 retval = -EFAULT;
851 goto out1;
852 }
853 buffer[nbytes] = 0; /* nul-terminate */
854
855 down(&cpuset_sem);
856
857 if (is_removed(cs)) {
858 retval = -ENODEV;
859 goto out2;
860 }
861
862 switch (type) {
863 case FILE_CPULIST:
864 retval = update_cpumask(cs, buffer);
865 break;
866 case FILE_MEMLIST:
867 retval = update_nodemask(cs, buffer);
868 break;
869 case FILE_CPU_EXCLUSIVE:
870 retval = update_flag(CS_CPU_EXCLUSIVE, cs, buffer);
871 break;
872 case FILE_MEM_EXCLUSIVE:
873 retval = update_flag(CS_MEM_EXCLUSIVE, cs, buffer);
874 break;
875 case FILE_NOTIFY_ON_RELEASE:
876 retval = update_flag(CS_NOTIFY_ON_RELEASE, cs, buffer);
877 break;
878 case FILE_TASKLIST:
Paul Jackson3077a262005-08-09 10:07:59 -0700879 retval = attach_task(cs, buffer, &pathbuf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 break;
881 default:
882 retval = -EINVAL;
883 goto out2;
884 }
885
886 if (retval == 0)
887 retval = nbytes;
888out2:
889 up(&cpuset_sem);
Paul Jackson3077a262005-08-09 10:07:59 -0700890 cpuset_release_agent(pathbuf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891out1:
892 kfree(buffer);
893 return retval;
894}
895
896static ssize_t cpuset_file_write(struct file *file, const char __user *buf,
897 size_t nbytes, loff_t *ppos)
898{
899 ssize_t retval = 0;
900 struct cftype *cft = __d_cft(file->f_dentry);
901 if (!cft)
902 return -ENODEV;
903
904 /* special function ? */
905 if (cft->write)
906 retval = cft->write(file, buf, nbytes, ppos);
907 else
908 retval = cpuset_common_file_write(file, buf, nbytes, ppos);
909
910 return retval;
911}
912
913/*
914 * These ascii lists should be read in a single call, by using a user
915 * buffer large enough to hold the entire map. If read in smaller
916 * chunks, there is no guarantee of atomicity. Since the display format
917 * used, list of ranges of sequential numbers, is variable length,
918 * and since these maps can change value dynamically, one could read
919 * gibberish by doing partial reads while a list was changing.
920 * A single large read to a buffer that crosses a page boundary is
921 * ok, because the result being copied to user land is not recomputed
922 * across a page fault.
923 */
924
925static int cpuset_sprintf_cpulist(char *page, struct cpuset *cs)
926{
927 cpumask_t mask;
928
929 down(&cpuset_sem);
930 mask = cs->cpus_allowed;
931 up(&cpuset_sem);
932
933 return cpulist_scnprintf(page, PAGE_SIZE, mask);
934}
935
936static int cpuset_sprintf_memlist(char *page, struct cpuset *cs)
937{
938 nodemask_t mask;
939
940 down(&cpuset_sem);
941 mask = cs->mems_allowed;
942 up(&cpuset_sem);
943
944 return nodelist_scnprintf(page, PAGE_SIZE, mask);
945}
946
947static ssize_t cpuset_common_file_read(struct file *file, char __user *buf,
948 size_t nbytes, loff_t *ppos)
949{
950 struct cftype *cft = __d_cft(file->f_dentry);
951 struct cpuset *cs = __d_cs(file->f_dentry->d_parent);
952 cpuset_filetype_t type = cft->private;
953 char *page;
954 ssize_t retval = 0;
955 char *s;
956 char *start;
957 size_t n;
958
959 if (!(page = (char *)__get_free_page(GFP_KERNEL)))
960 return -ENOMEM;
961
962 s = page;
963
964 switch (type) {
965 case FILE_CPULIST:
966 s += cpuset_sprintf_cpulist(s, cs);
967 break;
968 case FILE_MEMLIST:
969 s += cpuset_sprintf_memlist(s, cs);
970 break;
971 case FILE_CPU_EXCLUSIVE:
972 *s++ = is_cpu_exclusive(cs) ? '1' : '0';
973 break;
974 case FILE_MEM_EXCLUSIVE:
975 *s++ = is_mem_exclusive(cs) ? '1' : '0';
976 break;
977 case FILE_NOTIFY_ON_RELEASE:
978 *s++ = notify_on_release(cs) ? '1' : '0';
979 break;
980 default:
981 retval = -EINVAL;
982 goto out;
983 }
984 *s++ = '\n';
985 *s = '\0';
986
987 start = page + *ppos;
988 n = s - start;
989 retval = n - copy_to_user(buf, start, min(n, nbytes));
990 *ppos += retval;
991out:
992 free_page((unsigned long)page);
993 return retval;
994}
995
996static ssize_t cpuset_file_read(struct file *file, char __user *buf, size_t nbytes,
997 loff_t *ppos)
998{
999 ssize_t retval = 0;
1000 struct cftype *cft = __d_cft(file->f_dentry);
1001 if (!cft)
1002 return -ENODEV;
1003
1004 /* special function ? */
1005 if (cft->read)
1006 retval = cft->read(file, buf, nbytes, ppos);
1007 else
1008 retval = cpuset_common_file_read(file, buf, nbytes, ppos);
1009
1010 return retval;
1011}
1012
1013static int cpuset_file_open(struct inode *inode, struct file *file)
1014{
1015 int err;
1016 struct cftype *cft;
1017
1018 err = generic_file_open(inode, file);
1019 if (err)
1020 return err;
1021
1022 cft = __d_cft(file->f_dentry);
1023 if (!cft)
1024 return -ENODEV;
1025 if (cft->open)
1026 err = cft->open(inode, file);
1027 else
1028 err = 0;
1029
1030 return err;
1031}
1032
1033static int cpuset_file_release(struct inode *inode, struct file *file)
1034{
1035 struct cftype *cft = __d_cft(file->f_dentry);
1036 if (cft->release)
1037 return cft->release(inode, file);
1038 return 0;
1039}
1040
1041static struct file_operations cpuset_file_operations = {
1042 .read = cpuset_file_read,
1043 .write = cpuset_file_write,
1044 .llseek = generic_file_llseek,
1045 .open = cpuset_file_open,
1046 .release = cpuset_file_release,
1047};
1048
1049static struct inode_operations cpuset_dir_inode_operations = {
1050 .lookup = simple_lookup,
1051 .mkdir = cpuset_mkdir,
1052 .rmdir = cpuset_rmdir,
1053};
1054
1055static int cpuset_create_file(struct dentry *dentry, int mode)
1056{
1057 struct inode *inode;
1058
1059 if (!dentry)
1060 return -ENOENT;
1061 if (dentry->d_inode)
1062 return -EEXIST;
1063
1064 inode = cpuset_new_inode(mode);
1065 if (!inode)
1066 return -ENOMEM;
1067
1068 if (S_ISDIR(mode)) {
1069 inode->i_op = &cpuset_dir_inode_operations;
1070 inode->i_fop = &simple_dir_operations;
1071
1072 /* start off with i_nlink == 2 (for "." entry) */
1073 inode->i_nlink++;
1074 } else if (S_ISREG(mode)) {
1075 inode->i_size = 0;
1076 inode->i_fop = &cpuset_file_operations;
1077 }
1078
1079 d_instantiate(dentry, inode);
1080 dget(dentry); /* Extra count - pin the dentry in core */
1081 return 0;
1082}
1083
1084/*
1085 * cpuset_create_dir - create a directory for an object.
1086 * cs: the cpuset we create the directory for.
1087 * It must have a valid ->parent field
1088 * And we are going to fill its ->dentry field.
1089 * name: The name to give to the cpuset directory. Will be copied.
1090 * mode: mode to set on new directory.
1091 */
1092
1093static int cpuset_create_dir(struct cpuset *cs, const char *name, int mode)
1094{
1095 struct dentry *dentry = NULL;
1096 struct dentry *parent;
1097 int error = 0;
1098
1099 parent = cs->parent->dentry;
1100 dentry = cpuset_get_dentry(parent, name);
1101 if (IS_ERR(dentry))
1102 return PTR_ERR(dentry);
1103 error = cpuset_create_file(dentry, S_IFDIR | mode);
1104 if (!error) {
1105 dentry->d_fsdata = cs;
1106 parent->d_inode->i_nlink++;
1107 cs->dentry = dentry;
1108 }
1109 dput(dentry);
1110
1111 return error;
1112}
1113
1114static int cpuset_add_file(struct dentry *dir, const struct cftype *cft)
1115{
1116 struct dentry *dentry;
1117 int error;
1118
1119 down(&dir->d_inode->i_sem);
1120 dentry = cpuset_get_dentry(dir, cft->name);
1121 if (!IS_ERR(dentry)) {
1122 error = cpuset_create_file(dentry, 0644 | S_IFREG);
1123 if (!error)
1124 dentry->d_fsdata = (void *)cft;
1125 dput(dentry);
1126 } else
1127 error = PTR_ERR(dentry);
1128 up(&dir->d_inode->i_sem);
1129 return error;
1130}
1131
1132/*
1133 * Stuff for reading the 'tasks' file.
1134 *
1135 * Reading this file can return large amounts of data if a cpuset has
1136 * *lots* of attached tasks. So it may need several calls to read(),
1137 * but we cannot guarantee that the information we produce is correct
1138 * unless we produce it entirely atomically.
1139 *
1140 * Upon tasks file open(), a struct ctr_struct is allocated, that
1141 * will have a pointer to an array (also allocated here). The struct
1142 * ctr_struct * is stored in file->private_data. Its resources will
1143 * be freed by release() when the file is closed. The array is used
1144 * to sprintf the PIDs and then used by read().
1145 */
1146
1147/* cpusets_tasks_read array */
1148
1149struct ctr_struct {
1150 char *buf;
1151 int bufsz;
1152};
1153
1154/*
1155 * Load into 'pidarray' up to 'npids' of the tasks using cpuset 'cs'.
1156 * Return actual number of pids loaded.
1157 */
1158static inline int pid_array_load(pid_t *pidarray, int npids, struct cpuset *cs)
1159{
1160 int n = 0;
1161 struct task_struct *g, *p;
1162
1163 read_lock(&tasklist_lock);
1164
1165 do_each_thread(g, p) {
1166 if (p->cpuset == cs) {
1167 pidarray[n++] = p->pid;
1168 if (unlikely(n == npids))
1169 goto array_full;
1170 }
1171 } while_each_thread(g, p);
1172
1173array_full:
1174 read_unlock(&tasklist_lock);
1175 return n;
1176}
1177
1178static int cmppid(const void *a, const void *b)
1179{
1180 return *(pid_t *)a - *(pid_t *)b;
1181}
1182
1183/*
1184 * Convert array 'a' of 'npids' pid_t's to a string of newline separated
1185 * decimal pids in 'buf'. Don't write more than 'sz' chars, but return
1186 * count 'cnt' of how many chars would be written if buf were large enough.
1187 */
1188static int pid_array_to_buf(char *buf, int sz, pid_t *a, int npids)
1189{
1190 int cnt = 0;
1191 int i;
1192
1193 for (i = 0; i < npids; i++)
1194 cnt += snprintf(buf + cnt, max(sz - cnt, 0), "%d\n", a[i]);
1195 return cnt;
1196}
1197
1198static int cpuset_tasks_open(struct inode *unused, struct file *file)
1199{
1200 struct cpuset *cs = __d_cs(file->f_dentry->d_parent);
1201 struct ctr_struct *ctr;
1202 pid_t *pidarray;
1203 int npids;
1204 char c;
1205
1206 if (!(file->f_mode & FMODE_READ))
1207 return 0;
1208
1209 ctr = kmalloc(sizeof(*ctr), GFP_KERNEL);
1210 if (!ctr)
1211 goto err0;
1212
1213 /*
1214 * If cpuset gets more users after we read count, we won't have
1215 * enough space - tough. This race is indistinguishable to the
1216 * caller from the case that the additional cpuset users didn't
1217 * show up until sometime later on.
1218 */
1219 npids = atomic_read(&cs->count);
1220 pidarray = kmalloc(npids * sizeof(pid_t), GFP_KERNEL);
1221 if (!pidarray)
1222 goto err1;
1223
1224 npids = pid_array_load(pidarray, npids, cs);
1225 sort(pidarray, npids, sizeof(pid_t), cmppid, NULL);
1226
1227 /* Call pid_array_to_buf() twice, first just to get bufsz */
1228 ctr->bufsz = pid_array_to_buf(&c, sizeof(c), pidarray, npids) + 1;
1229 ctr->buf = kmalloc(ctr->bufsz, GFP_KERNEL);
1230 if (!ctr->buf)
1231 goto err2;
1232 ctr->bufsz = pid_array_to_buf(ctr->buf, ctr->bufsz, pidarray, npids);
1233
1234 kfree(pidarray);
1235 file->private_data = ctr;
1236 return 0;
1237
1238err2:
1239 kfree(pidarray);
1240err1:
1241 kfree(ctr);
1242err0:
1243 return -ENOMEM;
1244}
1245
1246static ssize_t cpuset_tasks_read(struct file *file, char __user *buf,
1247 size_t nbytes, loff_t *ppos)
1248{
1249 struct ctr_struct *ctr = file->private_data;
1250
1251 if (*ppos + nbytes > ctr->bufsz)
1252 nbytes = ctr->bufsz - *ppos;
1253 if (copy_to_user(buf, ctr->buf + *ppos, nbytes))
1254 return -EFAULT;
1255 *ppos += nbytes;
1256 return nbytes;
1257}
1258
1259static int cpuset_tasks_release(struct inode *unused_inode, struct file *file)
1260{
1261 struct ctr_struct *ctr;
1262
1263 if (file->f_mode & FMODE_READ) {
1264 ctr = file->private_data;
1265 kfree(ctr->buf);
1266 kfree(ctr);
1267 }
1268 return 0;
1269}
1270
1271/*
1272 * for the common functions, 'private' gives the type of file
1273 */
1274
1275static struct cftype cft_tasks = {
1276 .name = "tasks",
1277 .open = cpuset_tasks_open,
1278 .read = cpuset_tasks_read,
1279 .release = cpuset_tasks_release,
1280 .private = FILE_TASKLIST,
1281};
1282
1283static struct cftype cft_cpus = {
1284 .name = "cpus",
1285 .private = FILE_CPULIST,
1286};
1287
1288static struct cftype cft_mems = {
1289 .name = "mems",
1290 .private = FILE_MEMLIST,
1291};
1292
1293static struct cftype cft_cpu_exclusive = {
1294 .name = "cpu_exclusive",
1295 .private = FILE_CPU_EXCLUSIVE,
1296};
1297
1298static struct cftype cft_mem_exclusive = {
1299 .name = "mem_exclusive",
1300 .private = FILE_MEM_EXCLUSIVE,
1301};
1302
1303static struct cftype cft_notify_on_release = {
1304 .name = "notify_on_release",
1305 .private = FILE_NOTIFY_ON_RELEASE,
1306};
1307
1308static int cpuset_populate_dir(struct dentry *cs_dentry)
1309{
1310 int err;
1311
1312 if ((err = cpuset_add_file(cs_dentry, &cft_cpus)) < 0)
1313 return err;
1314 if ((err = cpuset_add_file(cs_dentry, &cft_mems)) < 0)
1315 return err;
1316 if ((err = cpuset_add_file(cs_dentry, &cft_cpu_exclusive)) < 0)
1317 return err;
1318 if ((err = cpuset_add_file(cs_dentry, &cft_mem_exclusive)) < 0)
1319 return err;
1320 if ((err = cpuset_add_file(cs_dentry, &cft_notify_on_release)) < 0)
1321 return err;
1322 if ((err = cpuset_add_file(cs_dentry, &cft_tasks)) < 0)
1323 return err;
1324 return 0;
1325}
1326
1327/*
1328 * cpuset_create - create a cpuset
1329 * parent: cpuset that will be parent of the new cpuset.
1330 * name: name of the new cpuset. Will be strcpy'ed.
1331 * mode: mode to set on new inode
1332 *
1333 * Must be called with the semaphore on the parent inode held
1334 */
1335
1336static long cpuset_create(struct cpuset *parent, const char *name, int mode)
1337{
1338 struct cpuset *cs;
1339 int err;
1340
1341 cs = kmalloc(sizeof(*cs), GFP_KERNEL);
1342 if (!cs)
1343 return -ENOMEM;
1344
1345 down(&cpuset_sem);
1346 refresh_mems();
1347 cs->flags = 0;
1348 if (notify_on_release(parent))
1349 set_bit(CS_NOTIFY_ON_RELEASE, &cs->flags);
1350 cs->cpus_allowed = CPU_MASK_NONE;
1351 cs->mems_allowed = NODE_MASK_NONE;
1352 atomic_set(&cs->count, 0);
1353 INIT_LIST_HEAD(&cs->sibling);
1354 INIT_LIST_HEAD(&cs->children);
1355 atomic_inc(&cpuset_mems_generation);
1356 cs->mems_generation = atomic_read(&cpuset_mems_generation);
1357
1358 cs->parent = parent;
1359
1360 list_add(&cs->sibling, &cs->parent->children);
1361
1362 err = cpuset_create_dir(cs, name, mode);
1363 if (err < 0)
1364 goto err;
1365
1366 /*
1367 * Release cpuset_sem before cpuset_populate_dir() because it
1368 * will down() this new directory's i_sem and if we race with
1369 * another mkdir, we might deadlock.
1370 */
1371 up(&cpuset_sem);
1372
1373 err = cpuset_populate_dir(cs->dentry);
1374 /* If err < 0, we have a half-filled directory - oh well ;) */
1375 return 0;
1376err:
1377 list_del(&cs->sibling);
1378 up(&cpuset_sem);
1379 kfree(cs);
1380 return err;
1381}
1382
1383static int cpuset_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1384{
1385 struct cpuset *c_parent = dentry->d_parent->d_fsdata;
1386
1387 /* the vfs holds inode->i_sem already */
1388 return cpuset_create(c_parent, dentry->d_name.name, mode | S_IFDIR);
1389}
1390
1391static int cpuset_rmdir(struct inode *unused_dir, struct dentry *dentry)
1392{
1393 struct cpuset *cs = dentry->d_fsdata;
1394 struct dentry *d;
1395 struct cpuset *parent;
Paul Jackson3077a262005-08-09 10:07:59 -07001396 char *pathbuf = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397
1398 /* the vfs holds both inode->i_sem already */
1399
1400 down(&cpuset_sem);
1401 refresh_mems();
1402 if (atomic_read(&cs->count) > 0) {
1403 up(&cpuset_sem);
1404 return -EBUSY;
1405 }
1406 if (!list_empty(&cs->children)) {
1407 up(&cpuset_sem);
1408 return -EBUSY;
1409 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 parent = cs->parent;
1411 set_bit(CS_REMOVED, &cs->flags);
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -07001412 if (is_cpu_exclusive(cs))
1413 update_cpu_domains(cs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001414 list_del(&cs->sibling); /* delete my sibling from parent->children */
1415 if (list_empty(&parent->children))
Paul Jackson3077a262005-08-09 10:07:59 -07001416 check_for_release(parent, &pathbuf);
Dinakar Guniguntala85d7b942005-06-25 14:57:34 -07001417 spin_lock(&cs->dentry->d_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 d = dget(cs->dentry);
1419 cs->dentry = NULL;
1420 spin_unlock(&d->d_lock);
1421 cpuset_d_remove_dir(d);
1422 dput(d);
1423 up(&cpuset_sem);
Paul Jackson3077a262005-08-09 10:07:59 -07001424 cpuset_release_agent(pathbuf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425 return 0;
1426}
1427
1428/**
1429 * cpuset_init - initialize cpusets at system boot
1430 *
1431 * Description: Initialize top_cpuset and the cpuset internal file system,
1432 **/
1433
1434int __init cpuset_init(void)
1435{
1436 struct dentry *root;
1437 int err;
1438
1439 top_cpuset.cpus_allowed = CPU_MASK_ALL;
1440 top_cpuset.mems_allowed = NODE_MASK_ALL;
1441
1442 atomic_inc(&cpuset_mems_generation);
1443 top_cpuset.mems_generation = atomic_read(&cpuset_mems_generation);
1444
1445 init_task.cpuset = &top_cpuset;
1446
1447 err = register_filesystem(&cpuset_fs_type);
1448 if (err < 0)
1449 goto out;
1450 cpuset_mount = kern_mount(&cpuset_fs_type);
1451 if (IS_ERR(cpuset_mount)) {
1452 printk(KERN_ERR "cpuset: could not mount!\n");
1453 err = PTR_ERR(cpuset_mount);
1454 cpuset_mount = NULL;
1455 goto out;
1456 }
1457 root = cpuset_mount->mnt_sb->s_root;
1458 root->d_fsdata = &top_cpuset;
1459 root->d_inode->i_nlink++;
1460 top_cpuset.dentry = root;
1461 root->d_inode->i_op = &cpuset_dir_inode_operations;
1462 err = cpuset_populate_dir(root);
1463out:
1464 return err;
1465}
1466
1467/**
1468 * cpuset_init_smp - initialize cpus_allowed
1469 *
1470 * Description: Finish top cpuset after cpu, node maps are initialized
1471 **/
1472
1473void __init cpuset_init_smp(void)
1474{
1475 top_cpuset.cpus_allowed = cpu_online_map;
1476 top_cpuset.mems_allowed = node_online_map;
1477}
1478
1479/**
1480 * cpuset_fork - attach newly forked task to its parents cpuset.
Randy Dunlapd9fd8a62005-07-27 11:45:11 -07001481 * @tsk: pointer to task_struct of forking parent process.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001482 *
1483 * Description: By default, on fork, a task inherits its
Randy Dunlapd9fd8a62005-07-27 11:45:11 -07001484 * parent's cpuset. The pointer to the shared cpuset is
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485 * automatically copied in fork.c by dup_task_struct().
1486 * This cpuset_fork() routine need only increment the usage
1487 * counter in that cpuset.
1488 **/
1489
1490void cpuset_fork(struct task_struct *tsk)
1491{
1492 atomic_inc(&tsk->cpuset->count);
1493}
1494
1495/**
1496 * cpuset_exit - detach cpuset from exiting task
1497 * @tsk: pointer to task_struct of exiting process
1498 *
1499 * Description: Detach cpuset from @tsk and release it.
1500 *
Paul Jackson2efe86b2005-05-27 02:02:43 -07001501 * Note that cpusets marked notify_on_release force every task
1502 * in them to take the global cpuset_sem semaphore when exiting.
1503 * This could impact scaling on very large systems. Be reluctant
1504 * to use notify_on_release cpusets where very high task exit
1505 * scaling is required on large systems.
1506 *
1507 * Don't even think about derefencing 'cs' after the cpuset use
1508 * count goes to zero, except inside a critical section guarded
1509 * by the cpuset_sem semaphore. If you don't hold cpuset_sem,
1510 * then a zero cpuset use count is a license to any other task to
1511 * nuke the cpuset immediately.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512 **/
1513
1514void cpuset_exit(struct task_struct *tsk)
1515{
1516 struct cpuset *cs;
1517
1518 task_lock(tsk);
1519 cs = tsk->cpuset;
1520 tsk->cpuset = NULL;
1521 task_unlock(tsk);
1522
Paul Jackson2efe86b2005-05-27 02:02:43 -07001523 if (notify_on_release(cs)) {
Paul Jackson3077a262005-08-09 10:07:59 -07001524 char *pathbuf = NULL;
1525
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 down(&cpuset_sem);
Paul Jackson2efe86b2005-05-27 02:02:43 -07001527 if (atomic_dec_and_test(&cs->count))
Paul Jackson3077a262005-08-09 10:07:59 -07001528 check_for_release(cs, &pathbuf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 up(&cpuset_sem);
Paul Jackson3077a262005-08-09 10:07:59 -07001530 cpuset_release_agent(pathbuf);
Paul Jackson2efe86b2005-05-27 02:02:43 -07001531 } else {
1532 atomic_dec(&cs->count);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001533 }
1534}
1535
1536/**
1537 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
1538 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
1539 *
1540 * Description: Returns the cpumask_t cpus_allowed of the cpuset
1541 * attached to the specified @tsk. Guaranteed to return some non-empty
1542 * subset of cpu_online_map, even if this means going outside the
1543 * tasks cpuset.
1544 **/
1545
Benoit Boissinot9a848892005-04-16 15:25:59 -07001546cpumask_t cpuset_cpus_allowed(const struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547{
1548 cpumask_t mask;
1549
1550 down(&cpuset_sem);
1551 task_lock((struct task_struct *)tsk);
1552 guarantee_online_cpus(tsk->cpuset, &mask);
1553 task_unlock((struct task_struct *)tsk);
1554 up(&cpuset_sem);
1555
1556 return mask;
1557}
1558
1559void cpuset_init_current_mems_allowed(void)
1560{
1561 current->mems_allowed = NODE_MASK_ALL;
1562}
1563
Randy Dunlapd9fd8a62005-07-27 11:45:11 -07001564/**
1565 * cpuset_update_current_mems_allowed - update mems parameters to new values
1566 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567 * If the current tasks cpusets mems_allowed changed behind our backs,
1568 * update current->mems_allowed and mems_generation to the new value.
1569 * Do not call this routine if in_interrupt().
1570 */
1571
1572void cpuset_update_current_mems_allowed(void)
1573{
1574 struct cpuset *cs = current->cpuset;
1575
1576 if (!cs)
1577 return; /* task is exiting */
1578 if (current->cpuset_mems_generation != cs->mems_generation) {
1579 down(&cpuset_sem);
1580 refresh_mems();
1581 up(&cpuset_sem);
1582 }
1583}
1584
Randy Dunlapd9fd8a62005-07-27 11:45:11 -07001585/**
1586 * cpuset_restrict_to_mems_allowed - limit nodes to current mems_allowed
1587 * @nodes: pointer to a node bitmap that is and-ed with mems_allowed
1588 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589void cpuset_restrict_to_mems_allowed(unsigned long *nodes)
1590{
1591 bitmap_and(nodes, nodes, nodes_addr(current->mems_allowed),
1592 MAX_NUMNODES);
1593}
1594
Randy Dunlapd9fd8a62005-07-27 11:45:11 -07001595/**
1596 * cpuset_zonelist_valid_mems_allowed - check zonelist vs. curremt mems_allowed
1597 * @zl: the zonelist to be checked
1598 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599 * Are any of the nodes on zonelist zl allowed in current->mems_allowed?
1600 */
1601int cpuset_zonelist_valid_mems_allowed(struct zonelist *zl)
1602{
1603 int i;
1604
1605 for (i = 0; zl->zones[i]; i++) {
1606 int nid = zl->zones[i]->zone_pgdat->node_id;
1607
1608 if (node_isset(nid, current->mems_allowed))
1609 return 1;
1610 }
1611 return 0;
1612}
1613
Paul Jackson9bf22292005-09-06 15:18:12 -07001614/*
1615 * nearest_exclusive_ancestor() - Returns the nearest mem_exclusive
1616 * ancestor to the specified cpuset. Call while holding cpuset_sem.
1617 * If no ancestor is mem_exclusive (an unusual configuration), then
1618 * returns the root cpuset.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619 */
Paul Jackson9bf22292005-09-06 15:18:12 -07001620static const struct cpuset *nearest_exclusive_ancestor(const struct cpuset *cs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001621{
Paul Jackson9bf22292005-09-06 15:18:12 -07001622 while (!is_mem_exclusive(cs) && cs->parent)
1623 cs = cs->parent;
1624 return cs;
1625}
1626
1627/**
1628 * cpuset_zone_allowed - Can we allocate memory on zone z's memory node?
1629 * @z: is this zone on an allowed node?
1630 * @gfp_mask: memory allocation flags (we use __GFP_HARDWALL)
1631 *
1632 * If we're in interrupt, yes, we can always allocate. If zone
1633 * z's node is in our tasks mems_allowed, yes. If it's not a
1634 * __GFP_HARDWALL request and this zone's nodes is in the nearest
1635 * mem_exclusive cpuset ancestor to this tasks cpuset, yes.
1636 * Otherwise, no.
1637 *
1638 * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
1639 * and do not allow allocations outside the current tasks cpuset.
1640 * GFP_KERNEL allocations are not so marked, so can escape to the
1641 * nearest mem_exclusive ancestor cpuset.
1642 *
1643 * Scanning up parent cpusets requires cpuset_sem. The __alloc_pages()
1644 * routine only calls here with __GFP_HARDWALL bit _not_ set if
1645 * it's a GFP_KERNEL allocation, and all nodes in the current tasks
1646 * mems_allowed came up empty on the first pass over the zonelist.
1647 * So only GFP_KERNEL allocations, if all nodes in the cpuset are
1648 * short of memory, might require taking the cpuset_sem semaphore.
1649 *
1650 * The first loop over the zonelist in mm/page_alloc.c:__alloc_pages()
1651 * calls here with __GFP_HARDWALL always set in gfp_mask, enforcing
1652 * hardwall cpusets - no allocation on a node outside the cpuset is
1653 * allowed (unless in interrupt, of course).
1654 *
1655 * The second loop doesn't even call here for GFP_ATOMIC requests
1656 * (if the __alloc_pages() local variable 'wait' is set). That check
1657 * and the checks below have the combined affect in the second loop of
1658 * the __alloc_pages() routine that:
1659 * in_interrupt - any node ok (current task context irrelevant)
1660 * GFP_ATOMIC - any node ok
1661 * GFP_KERNEL - any node in enclosing mem_exclusive cpuset ok
1662 * GFP_USER - only nodes in current tasks mems allowed ok.
1663 **/
1664
1665int cpuset_zone_allowed(struct zone *z, unsigned int __nocast gfp_mask)
1666{
1667 int node; /* node that zone z is on */
1668 const struct cpuset *cs; /* current cpuset ancestors */
1669 int allowed = 1; /* is allocation in zone z allowed? */
1670
1671 if (in_interrupt())
1672 return 1;
1673 node = z->zone_pgdat->node_id;
1674 if (node_isset(node, current->mems_allowed))
1675 return 1;
1676 if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
1677 return 0;
1678
1679 /* Not hardwall and node outside mems_allowed: scan up cpusets */
1680 down(&cpuset_sem);
1681 cs = current->cpuset;
1682 if (!cs)
1683 goto done; /* current task exiting */
1684 cs = nearest_exclusive_ancestor(cs);
1685 allowed = node_isset(node, cs->mems_allowed);
1686done:
1687 up(&cpuset_sem);
1688 return allowed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689}
1690
1691/*
1692 * proc_cpuset_show()
1693 * - Print tasks cpuset path into seq_file.
1694 * - Used for /proc/<pid>/cpuset.
1695 */
1696
1697static int proc_cpuset_show(struct seq_file *m, void *v)
1698{
1699 struct cpuset *cs;
1700 struct task_struct *tsk;
1701 char *buf;
1702 int retval = 0;
1703
1704 buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
1705 if (!buf)
1706 return -ENOMEM;
1707
1708 tsk = m->private;
1709 down(&cpuset_sem);
1710 task_lock(tsk);
1711 cs = tsk->cpuset;
1712 task_unlock(tsk);
1713 if (!cs) {
1714 retval = -EINVAL;
1715 goto out;
1716 }
1717
1718 retval = cpuset_path(cs, buf, PAGE_SIZE);
1719 if (retval < 0)
1720 goto out;
1721 seq_puts(m, buf);
1722 seq_putc(m, '\n');
1723out:
1724 up(&cpuset_sem);
1725 kfree(buf);
1726 return retval;
1727}
1728
1729static int cpuset_open(struct inode *inode, struct file *file)
1730{
1731 struct task_struct *tsk = PROC_I(inode)->task;
1732 return single_open(file, proc_cpuset_show, tsk);
1733}
1734
1735struct file_operations proc_cpuset_operations = {
1736 .open = cpuset_open,
1737 .read = seq_read,
1738 .llseek = seq_lseek,
1739 .release = single_release,
1740};
1741
1742/* Display task cpus_allowed, mems_allowed in /proc/<pid>/status file. */
1743char *cpuset_task_status_allowed(struct task_struct *task, char *buffer)
1744{
1745 buffer += sprintf(buffer, "Cpus_allowed:\t");
1746 buffer += cpumask_scnprintf(buffer, PAGE_SIZE, task->cpus_allowed);
1747 buffer += sprintf(buffer, "\n");
1748 buffer += sprintf(buffer, "Mems_allowed:\t");
1749 buffer += nodemask_scnprintf(buffer, PAGE_SIZE, task->mems_allowed);
1750 buffer += sprintf(buffer, "\n");
1751 return buffer;
1752}