GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

No menu could be recovered. Follow the blue links in the page.

This view

  • Original page
  • New address

GeezerVison is showing codebrowser.dev (truncated) — fork.c source code [linux/kernel/fork.c] - Codebrowser

Codebrowser logo
About
Contact
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/kernel/fork.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8/*
9 * 'fork.c' contains the help-routines for the 'fork' system call
10 * (see also entry.S and others).
11 * Fork is rather simple, once you get the hang of it, but the memory
12 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
13 */
14
15#include <linux/anon_inodes.h>
16#include <linux/slab.h>
17#include <linux/sched/autogroup.h>
18#include <linux/sched/mm.h>
19#include <linux/sched/user.h>
20#include <linux/sched/numa_balancing.h>
21#include <linux/sched/stat.h>
22#include <linux/sched/task.h>
23#include <linux/sched/task_stack.h>
24#include <linux/sched/cputime.h>
25#include <linux/sched/ext.h>
26#include <linux/seq_file.h>
27#include <linux/rtmutex.h>
28#include <linux/init.h>
29#include <linux/unistd.h>
30#include <linux/module.h>
31#include <linux/vmalloc.h>
32#include <linux/completion.h>
33#include <linux/personality.h>
34#include <linux/mempolicy.h>
35#include <linux/sem.h>
36#include <linux/file.h>
37#include <linux/fdtable.h>
38#include <linux/iocontext.h>
39#include <linux/key.h>
40#include <linux/kmsan.h>
41#include <linux/binfmts.h>
42#include <linux/mman.h>
43#include <linux/mmu_notifier.h>
44#include <linux/fs.h>
45#include <linux/mm.h>
46#include <linux/mm_inline.h>
47#include <linux/memblock.h>
48#include <linux/nsproxy.h>
49#include <linux/capability.h>
50#include <linux/cpu.h>
51#include <linux/cgroup.h>
52#include <linux/security.h>
53#include <linux/hugetlb.h>
54#include <linux/seccomp.h>
55#include <linux/swap.h>
56#include <linux/syscalls.h>
57#include <linux/syscall_user_dispatch.h>
58#include <linux/jiffies.h>
59#include <linux/futex.h>
60#include <linux/compat.h>
61#include <linux/kthread.h>
62#include <linux/task_io_accounting_ops.h>
63#include <linux/rcupdate.h>
64#include <linux/ptrace.h>
65#include <linux/mount.h>
66#include <linux/audit.h>
67#include <linux/memcontrol.h>
68#include <linux/ftrace.h>
69#include <linux/proc_fs.h>
70#include <linux/profile.h>
71#include <linux/rmap.h>
72#include <linux/ksm.h>
73#include <linux/acct.h>
74#include <linux/userfaultfd_k.h>
75#include <linux/tsacct_kern.h>
76#include <linux/cn_proc.h>
77#include <linux/freezer.h>
78#include <linux/delayacct.h>
79#include <linux/taskstats_kern.h>
80#include <linux/tty.h>
81#include <linux/fs_struct.h>
82#include <linux/magic.h>
83#include <linux/perf_event.h>
84#include <linux/posix-timers.h>
85#include <linux/user-return-notifier.h>
86#include <linux/oom.h>
87#include <linux/khugepaged.h>
88#include <linux/signalfd.h>
89#include <linux/uprobes.h>
90#include <linux/aio.h>
91#include <linux/compiler.h>
92#include <linux/sysctl.h>
93#include <linux/kcov.h>
94#include <linux/livepatch.h>
95#include <linux/thread_info.h>
96#include <linux/kstack_erase.h>
97#include <linux/kasan.h>
98#include <linux/scs.h>
99#include <linux/io_uring.h>
100#include <linux/bpf.h>
101#include <linux/stackprotector.h>
102#include <linux/user_events.h>
103#include <linux/iommu.h>
104#include <linux/rseq.h>
105#include <uapi/linux/pidfd.h>
106#include <linux/pidfs.h>
107#include <linux/tick.h>
108#include <linux/unwind_deferred.h>
109#include <linux/pgalloc.h>
110#include <linux/uaccess.h>
111
112#include <asm/mmu_context.h>
113#include <asm/cacheflush.h>
114#include <asm/tlbflush.h>
115
116/* For dup_mmap(). */
117#include "../mm/internal.h"
118
119#include <trace/events/sched.h>
120
121#define CREATE_TRACE_POINTS
122#include <trace/events/task.h>
123
124#include <kunit/visibility.h>
125
126/*
127 * Minimum number of threads to boot the kernel
128 */
129#define MIN_THREADS 20
130
131/*
132 * Maximum number of threads
133 */
134#define MAX_THREADS FUTEX_TID_MASK
135
136/*
137 * Protected counters by write_lock_irq(&tasklist_lock)
138 */
139unsigned long total_forks; /* Handle normal Linux uptimes. */
140int nr_threads; /* The idle threads do not count.. */
141
142static int max_threads; /* tunable limit on nr_threads */
143
144#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x)
145
146static const char * const resident_page_types[] = {
147 NAMED_ARRAY_INDEX(MM_FILEPAGES),
148 NAMED_ARRAY_INDEX(MM_ANONPAGES),
149 NAMED_ARRAY_INDEX(MM_SWAPENTS),
150 NAMED_ARRAY_INDEX(MM_SHMEMPAGES),
151};
152
153DEFINE_PER_CPU(unsigned long, process_counts) = 0;
154
155__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
156
157#ifdef CONFIG_PROVE_RCU
158int lockdep_tasklist_lock_is_held(void)
159{
160 return lockdep_is_held(&tasklist_lock);
161}
162EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
163#endif /* #ifdef CONFIG_PROVE_RCU */
164
165int nr_processes(void)
166{
167 int cpu;
168 int total = 0;
169
170 for_each_possible_cpu(cpu)
171 total += per_cpu(process_counts, cpu);
172
173 return total;
174}
175
176void __weak arch_release_task_struct(struct task_struct *tsk)
177{
178}
179
180static struct kmem_cache *task_struct_cachep;
181
182static inline struct task_struct *alloc_task_struct_node(int node)
183{
184 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
185}
186
187static inline void free_task_struct(struct task_struct *tsk)
188{
189 kmem_cache_free(s: task_struct_cachep, objp: tsk);
190}
191
192#ifdef CONFIG_VMAP_STACK
193/*
194 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
195 * flush. Try to minimize the number of calls by caching stacks.
196 */
197#define NR_CACHED_STACKS 2
198static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
199/*
200 * Allocated stacks are cached and later reused by new threads, so memcg
201 * accounting is performed by the code assigning/releasing stacks to tasks.
202 * We need a zeroed memory without __GFP_ACCOUNT.
203 */
204#define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO)
205
206struct vm_stack {
207 struct rcu_head rcu;
208 struct vm_struct *stack_vm_area;
209};
210
211static struct vm_struct *alloc_thread_stack_node_from_cache(struct task_struct *tsk, int node)
212{
213 struct vm_struct *vm_area;
214 unsigned int i;
215
216 /*
217 * If the node has memory, we are guaranteed the stacks are backed by local pages.
218 * Otherwise the pages are arbitrary.
219 *
220 * Note that depending on cpuset it is possible we will get migrated to a different
221 * node immediately after allocating here, so this does *not* guarantee locality for
222 * arbitrary callers.
223 */
224 scoped_guard(preempt) {
225 if (node != NUMA_NO_NODE && numa_node_id() != node)
226 return NULL;
227
228 for (i = 0; i < NR_CACHED_STACKS; i++) {
229 vm_area = this_cpu_xchg(cached_stacks[i], NULL);
230 if (vm_area)
231 return vm_area;
232 }
233 }
234
235 return NULL;
236}
237
238static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area)
239{
240 unsigned int i;
241 int nid;
242
243 /*
244 * Don't cache stacks if any of the pages don't match the local domain, unless
245 * there is no local memory to begin with.
246 *
247 * Note that lack of local memory does not automatically mean it makes no difference
248 * performance-wise which other domain backs the stack. In this case we are merely
249 * trying to avoid constantly going to vmalloc.
250 */
251 scoped_guard(preempt) {
252 nid = numa_node_id();
253 if (node_state(node: nid, state: N_MEMORY)) {
254 for (i = 0; i < vm_area->nr_pages; i++) {
255 struct page *page = vm_area->pages[i];
256 if (page_to_nid(page) != nid)
257 return false;
258 }
259 }
260
261 for (i = 0; i < NR_CACHED_STACKS; i++) {
262 struct vm_struct *tmp = NULL;
263
264 if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm_area))
265 return true;
266 }
267 }
268 return false;
269}
270
271static void thread_stack_free_rcu(struct rcu_head *rh)
272{
273 struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu);
274 struct vm_struct *vm_area = vm_stack->stack_vm_area;
275
276 if (try_release_thread_stack_to_cache(vm_area: vm_stack->stack_vm_area))
277 return;
278
279 vfree(addr: vm_area->addr);
280}
281
282static void thread_stack_delayed_free(struct task_struct *tsk)
283{
284 struct vm_stack *vm_stack = tsk->stack;
285
286 vm_stack->stack_vm_area = tsk->stack_vm_area;
287 call_rcu(head: &vm_stack->rcu, func: thread_stack_free_rcu);
288}
289
290static int free_vm_stack_cache(unsigned int cpu)
291{
292 struct vm_struct **cached_vm_stack_areas = per_cpu_ptr(cached_stacks, cpu);
293 int i;
294
295 for (i = 0; i < NR_CACHED_STACKS; i++) {
296 struct vm_struct *vm_area = cached_vm_stack_areas[i];
297
298 if (!vm_area)
299 continue;
300
301 vfree(addr: vm_area->addr);
302 cached_vm_stack_areas[i] = NULL;
303 }
304
305 return 0;
306}
307
308static int memcg_charge_kernel_stack(struct vm_struct *vm_area)
309{
310 int i;
311 int ret;
312 int nr_charged = 0;
313
314 BUG_ON(vm_area->nr_pages != THREAD_SIZE / PAGE_SIZE);
315
316 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
317 ret = memcg_kmem_charge_page(page: vm_area->pages[i], GFP_KERNEL, order: 0);
318 if (ret)
319 goto err;
320 nr_charged++;
321 }
322 return 0;
323err:
324 for (i = 0; i < nr_charged; i++)
325 memcg_kmem_uncharge_page(page: vm_area->pages[i], order: 0);
326 return ret;
327}
328
329static int alloc_thread_stack_node(struct task_struct *tsk, int node)
330{
331 struct vm_struct *vm_area;
332 void *stack;
333
334 vm_area = alloc_thread_stack_node_from_cache(tsk, node);
335 if (vm_area) {
336 if (memcg_charge_kernel_stack(vm_area)) {
337 vfree(addr: vm_area->addr);
338 return -ENOMEM;
339 }
340
341 /* Reset stack metadata. */
342 kasan_unpoison_range(addr: vm_area->addr, THREAD_SIZE);
343
344 stack = kasan_reset_tag(addr: vm_area->addr);
345
346 /* Clear stale pointers from reused stack. */
347 memset(stack, 0, THREAD_SIZE);
348
349 tsk->stack_vm_area = vm_area;
350 tsk->stack = stack;
351 return 0;
352 }
353
354 stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,
355 GFP_VMAP_STACK,
356 node, __builtin_return_address(0));
357 if (!stack)
358 return -ENOMEM;
359
360 vm_area = find_vm_area(addr: stack);
361 if (memcg_charge_kernel_stack(vm_area)) {
362 vfree(addr: stack);
363 return -ENOMEM;
364 }
365 /*
366 * We can't call find_vm_area() in interrupt context, and
367 * free_thread_stack() can be called in interrupt context,
368 * so cache the vm_struct.
369 */
370 tsk->stack_vm_area = vm_area;
371 stack = kasan_reset_tag(addr: stack);
372 tsk->stack = stack;
373 return 0;
374}
375
376static void free_thread_stack(struct task_struct *tsk)
377{
378 if (!try_release_thread_stack_to_cache(vm_area: tsk->stack_vm_area))
379 thread_stack_delayed_free(tsk);
380
381 tsk->stack = NULL;
382 tsk->stack_vm_area = NULL;
383}
384
385#else /* !CONFIG_VMAP_STACK */
386
387/*
388 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
389 * kmemcache based allocator.
390 */
391#if THREAD_SIZE >= PAGE_SIZE
392
393static void thread_stack_free_rcu(struct rcu_head *rh)
394{
395 __free_pages(virt_to_page(rh), THREAD_SIZE_ORDER);
396}
397
398static void thread_stack_delayed_free(struct task_struct *tsk)
399{
400 struct rcu_head *rh = tsk->stack;
401
402 call_rcu(rh, thread_stack_free_rcu);
403}
404
405static int alloc_thread_stack_node(struct task_struct *tsk, int node)
406{
407 struct page *page = alloc_pages_node(node, THREADINFO_GFP,
408 THREAD_SIZE_ORDER);
409
410 if (likely(page)) {
411 tsk->stack = kasan_reset_tag(page_address(page));
412 return 0;
413 }
414 return -ENOMEM;
415}
416
417static void free_thread_stack(struct task_struct *tsk)
418{
419 thread_stack_delayed_free(tsk);
420 tsk->stack = NULL;
421}
422
423#else /* !(THREAD_SIZE >= PAGE_SIZE) */
424
425static struct kmem_cache *thread_stack_cache;
426
427static void thread_stack_free_rcu(struct rcu_head *rh)
428{
429 kmem_cache_free(thread_stack_cache, rh);
430}
431
432static void thread_stack_delayed_free(struct task_struct *tsk)
433{
434 struct rcu_head *rh = tsk->stack;
435
436 call_rcu(rh, thread_stack_free_rcu);
437}
438
439static int alloc_thread_stack_node(struct task_struct *tsk, int node)
440{
441 unsigned long *stack;
442 stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
443 stack = kasan_reset_tag(stack);
444 tsk->stack = stack;
445 return stack ? 0 : -ENOMEM;
446}
447
448static void free_thread_stack(struct task_struct *tsk)
449{
450 thread_stack_delayed_free(tsk);
451 tsk->stack = NULL;
452}
453
454void thread_stack_cache_init(void)
455{
456 thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
457 THREAD_SIZE, THREAD_SIZE, 0, 0,
458 THREAD_SIZE, NULL);
459 BUG_ON(thread_stack_cache == NULL);
460}
461
462#endif /* THREAD_SIZE >= PAGE_SIZE */
463#endif /* CONFIG_VMAP_STACK */
464
465/* SLAB cache for signal_struct structures (tsk->signal) */
466static struct kmem_cache *signal_cachep;
467
468/* SLAB cache for sighand_struct structures (tsk->sighand) */
469struct kmem_cache *sighand_cachep;
470
471/* SLAB cache for files_struct structures (tsk->files) */
472struct kmem_cache *files_cachep;
473
474/* SLAB cache for fs_struct structures (tsk->fs) */
475struct kmem_cache *fs_cachep;
476
477/* SLAB cache for mm_struct structures (tsk->mm) */
478static struct kmem_cache *mm_cachep;
479
480static void account_kernel_stack(struct task_struct *tsk, int account)
481{
482 if (IS_ENABLED(CONFIG_VMAP_STACK)) {
483 struct vm_struct *vm_area = task_stack_vm_area(t: tsk);
484 int i;
485
486 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
487 mod_lruvec_page_state(page: vm_area->pages[i], idx: NR_KERNEL_STACK_KB,
488 val: account * (PAGE_SIZE / 1024));
489 } else {
490 void *stack = task_stack_page(task: tsk);
491
492 /* All stack pages are in the same node. */
493 mod_lruvec_kmem_state(p: stack, idx: NR_KERNEL_STACK_KB,
494 val: account * (THREAD_SIZE / 1024));
495 }
496}
497
498void exit_task_stack_account(struct task_struct *tsk)
499{
500 account_kernel_stack(tsk, account: -1);
501
502 if (IS_ENABLED(CONFIG_VMAP_STACK)) {
503 struct vm_struct *vm_area;
504 int i;
505
506 vm_area = task_stack_vm_area(t: tsk);
507 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
508 memcg_kmem_uncharge_page(page: vm_area->pages[i], order: 0);
509 }
510}
511
512static void release_task_stack(struct task_struct *tsk)
513{
514 if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))
515 return; /* Better to leak the stack than to free prematurely */
516
517 free_thread_stack(tsk);
518}
519
520#ifdef CONFIG_THREAD_INFO_IN_TASK
521void put_task_stack(struct task_struct *tsk)
522{
523 if (refcount_dec_and_test(r: &tsk->stack_refcount))
524 release_task_stack(tsk);
525}
526#endif
527
528void free_task(struct task_struct *tsk)
529{
530#ifdef CONFIG_SECCOMP
531 WARN_ON_ONCE(tsk->seccomp.filter);
532#endif
533 release_user_cpus_ptr(p: tsk);
534 scs_release(tsk);
535
536#ifndef CONFIG_THREAD_INFO_IN_TASK
537 /*
538 * The task is finally done with both the stack and thread_info,
539 * so free both.
540 */
541 release_task_stack(tsk);
542#else
543 /*
544 * If the task had a separate stack allocation, it should be gone
545 * by now.
546 */
547 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
548#endif
549 rt_mutex_debug_task_free(tsk);
550 ftrace_graph_exit_task(t: tsk);
551 arch_release_task_struct(tsk);
552 if (tsk->flags & PF_KTHREAD)
553 free_kthread_struct(k: tsk);
554 bpf_task_storage_free(task: tsk);
555 free_task_struct(tsk);
556}
557EXPORT_SYMBOL(free_task);
558
559void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm)
560{
561 struct file *exe_file;
562
563 exe_file = get_mm_exe_file(mm: oldmm);
564 RCU_INIT_POINTER(mm->exe_file, exe_file);
565 /*
566 * We depend on the oldmm having properly denied write access to the
567 * exe_file already.
568 */
569 if (exe_file && exe_file_deny_write_access(exe_file))
570 pr_warn_once("exe_file_deny_write_access() failed in %s\n", __func__);
571}
572
573#ifdef CONFIG_MMU
574static inline int mm_alloc_pgd(struct mm_struct *mm)
575{
576 mm->pgd = pgd_alloc(mm);
577 if (unlikely(!mm->pgd))
578 return -ENOMEM;
579 return 0;
580}
581
582static inline void mm_free_pgd(struct mm_struct *mm)
583{
584 pgd_free(mm, pgd: mm->pgd);
585}
586#else
587#define mm_alloc_pgd(mm) (0)
588#define mm_free_pgd(mm)
589#endif /* CONFIG_MMU */
590
591#ifdef CONFIG_MM_ID
592static DEFINE_IDA(mm_ida);
593
594static inline int mm_alloc_id(struct mm_struct *mm)
595{
596 int ret;
597
598 ret = ida_alloc_range(&mm_ida, MM_ID_MIN, MM_ID_MAX, GFP_KERNEL);
599 if (ret < 0)
600 return ret;
601 mm->mm_id = ret;
602 return 0;
603}
604
605static inline void mm_free_id(struct mm_struct *mm)
606{
607 const mm_id_t id = mm->mm_id;
608
609 mm->mm_id = MM_ID_DUMMY;
610 if (id == MM_ID_DUMMY)
611 return;
612 if (WARN_ON_ONCE(id < MM_ID_MIN || id > MM_ID_MAX))
613 return;
614 ida_free(&mm_ida, id);
615}
616#else /* !CONFIG_MM_ID */
617static inline int mm_alloc_id(struct mm_struct *mm) { return 0; }
618static inline void mm_free_id(struct mm_struct *mm) {}
619#endif /* CONFIG_MM_ID */
620
621static void check_mm(struct mm_struct *mm)
622{
623 int i;
624
625 BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
626 "Please make sure 'struct resident_page_types[]' is updated as well");
627
628 for (i = 0; i < NR_MM_COUNTERS; i++) {
629 long x = percpu_counter_sum(fbc: &mm->rss_stat[i]);
630
631 if (unlikely(x)) {
632 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n",
633 mm, resident_page_types[i], x,
634 current->comm,
635 task_pid_nr(current));
636 }
637 }
638
639 if (mm_pgtables_bytes(mm))
640 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
641 mm_pgtables_bytes(mm));
642
643#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
644 VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
645#endif
646}
647
648#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
649#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
650
651static void do_check_lazy_tlb(void *arg)
652{
653 struct mm_struct *mm = arg;
654
655 WARN_ON_ONCE(current->active_mm == mm);
656}
657
658static void do_shoot_lazy_tlb(void *arg)
659{
660 struct mm_struct *mm = arg;
661
662 if (current->active_mm == mm) {
663 WARN_ON_ONCE(current->mm);
664 current->active_mm = &init_mm;
665 switch_mm(prev: mm, next: &init_mm, current);
666 }
667}
668
669static void cleanup_lazy_tlbs(struct mm_struct *mm)
670{
671 if (!IS_ENABLED(CONFIG_MMU_LAZY_TLB_SHOOTDOWN)) {
672 /*
673 * In this case, lazy tlb mms are refounted and would not reach
674 * __mmdrop until all CPUs have switched away and mmdrop()ed.
675 */
676 return;
677 }
678
679 /*
680 * Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it
681 * requires lazy mm users to switch to another mm when the refcount
682 * drops to zero, before the mm is freed. This requires IPIs here to
683 * switch kernel threads to init_mm.
684 *
685 * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm
686 * switch with the final userspace teardown TLB flush which leaves the
687 * mm lazy on this CPU but no others, reducing the need for additional
688 * IPIs here. There are cases where a final IPI is still required here,
689 * such as the final mmdrop being performed on a different CPU than the
690 * one exiting, or kernel threads using the mm when userspace exits.
691 *
692 * IPI overheads have not found to be expensive, but they could be
693 * reduced in a number of possible ways, for example (roughly
694 * increasing order of complexity):
695 * - The last lazy reference created by exit_mm() could instead switch
696 * to init_mm, however it's probable this will run on the same CPU
697 * immediately afterwards, so this may not reduce IPIs much.
698 * - A batch of mms requiring IPIs could be gathered and freed at once.
699 * - CPUs store active_mm where it can be remotely checked without a
700 * lock, to filter out false-positives in the cpumask.
701 * - After mm_users or mm_count reaches zero, switching away from the
702 * mm could clear mm_cpumask to reduce some IPIs, perhaps together
703 * with some batching or delaying of the final IPIs.
704 * - A delayed freeing and RCU-like quiescing sequence based on mm
705 * switching to avoid IPIs completely.
706 */
707 on_each_cpu_mask(mask: mm_cpumask(mm), func: do_shoot_lazy_tlb, info: (void *)mm, wait: 1);
708 if (IS_ENABLED(CONFIG_DEBUG_VM_SHOOT_LAZIES))
709 on_each_cpu(func: do_check_lazy_tlb, info: (void *)mm, wait: 1);
710}
711
712/*
713 * Called when the last reference to the mm
714 * is dropped: either by a lazy thread or by
715 * mmput. Free the page directory and the mm.
716 */
717void __mmdrop(struct mm_struct *mm)
718{
719 BUG_ON(mm == &init_mm);
720 WARN_ON_ONCE(mm == current->mm);
721
722 /* Ensure no CPUs are using this as their lazy tlb mm */
723 cleanup_lazy_tlbs(mm);
724
725 WARN_ON_ONCE(mm == current->active_mm);
726 mm_free_pgd(mm);
727 mm_free_id(mm);
728 destroy_context(mm);
729 mmu_notifier_subscriptions_destroy(mm);
730 check_mm(mm);
731 put_user_ns(ns: mm->user_ns);
732 mm_pasid_drop(mm);
733 mm_destroy_cid(mm);
734 percpu_counter_destroy_many(fbc: mm->rss_stat, nr_counters: NR_MM_COUNTERS);
735
736 free_mm(mm);
737}
738EXPORT_SYMBOL_GPL(__mmdrop);
739
740static void mmdrop_async_fn(struct work_struct *work)
741{
742 struct mm_struct *mm;
743
744 mm = container_of(work, struct mm_struct, async_put_work);
745 __mmdrop(mm);
746}
747
748static void mmdrop_async(struct mm_struct *mm)
749{
750 if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
751 INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
752 schedule_work(work: &mm->async_put_work);
753 }
754}
755
756static inline void free_signal_struct(struct signal_struct *sig)
757{
758 taskstats_tgid_free(sig);
759 sched_autogroup_exit(sig);
760 /*
761 * __mmdrop is not safe to call from softirq context on x86 due to
762 * pgd_dtor so postpone it to the async context
763 */
764 if (sig->oom_mm)
765 mmdrop_async(mm: sig->oom_mm);
766 kmem_cache_free(s: signal_cachep, objp: sig);
767}
768
769static inline void put_signal_struct(struct signal_struct *sig)
770{
771 if (refcount_dec_and_test(r: &sig->sigcnt))
772 free_signal_struct(sig);
773}
774
775void __put_task_struct(struct task_struct *tsk)
776{
777 WARN_ON(!tsk->exit_state);
778 WARN_ON(refcount_read(&tsk->usage));
779 WARN_ON(tsk == current);
780
781 unwind_task_free(task: tsk);
782 io_uring_free(tsk);
783 cgroup_task_free(p: tsk);
784 task_numa_free(p: tsk, final: true);
785 security_task_free(task: tsk);
786 exit_creds(tsk);
787 delayacct_tsk_free(tsk);
788 put_signal_struct(sig: tsk->signal);
789 sched_core_free(tsk);
790 free_task(tsk);
791}
792EXPORT_SYMBOL_GPL(__put_task_struct);
793
794void __put_task_struct_rcu_cb(struct rcu_head *rhp)
795{
796 struct task_struct *task = container_of(rhp, struct task_struct, rcu);
797
798 __put_task_struct(task);
799}
800EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb);
801
802void __init __weak arch_task_cache_init(void) { }
803
804/*
805 * set_max_threads
806 */
807static void __init set_max_threads(unsigned int max_threads_suggested)
808{
809 u64 threads;
810 unsigned long nr_pages = memblock_estimated_nr_free_pages();
811
812 /*
813 * The number of threads shall be limited such that the thread
814 * structures may only consume a small part of the available memory.
815 */
816 if (fls64(x: nr_pages) + fls64(PAGE_SIZE) > 64)
817 threads = MAX_THREADS;
818 else
819 threads = div64_u64(dividend: (u64) nr_pages * (u64) PAGE_SIZE,
820 divisor: (u64) THREAD_SIZE * 8UL);
821
822 if (threads > max_threads_suggested)
823 threads = max_threads_suggested;
824
825 max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
826}
827
828#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
829/* Initialized by the architecture: */
830int arch_task_struct_size __read_mostly;
831#endif
832
833static void __init task_struct_whitelist(unsigned long *offset, unsigned long *size)
834{
835 /* Fetch thread_struct whitelist for the architecture. */
836 arch_thread_struct_whitelist(offset, size);
837
838 /*
839 * Handle zero-sized whitelist or empty thread_struct, otherwise
840 * adjust offset to position of thread_struct in task_struct.
841 */
842 if (unlikely(*size == 0))
843 *offset = 0;
844 else
845 *offset += offsetof(struct task_struct, thread);
846}
847
848void __init fork_init(void)
849{
850 int i;
851#ifndef ARCH_MIN_TASKALIGN
852#define ARCH_MIN_TASKALIGN 0
853#endif
854 int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
855 unsigned long useroffset, usersize;
856
857 /* create a slab on which task_structs can be allocated */
858 task_struct_whitelist(offset: &useroffset, size: &usersize);
859 task_struct_cachep = kmem_cache_create_usercopy(name: "task_struct",
860 size: arch_task_struct_size, align,
861 SLAB_PANIC|SLAB_ACCOUNT,
862 useroffset, usersize, NULL);
863
864 /* do the arch specific task caches init */
865 arch_task_cache_init();
866
867 set_max_threads(MAX_THREADS);
868
869 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
870 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
871 init_task.signal->rlim[RLIMIT_SIGPENDING] =
872 init_task.signal->rlim[RLIMIT_NPROC];
873
874 for (i = 0; i < UCOUNT_COUNTS; i++)
875 init_user_ns.ucount_max[i] = max_threads/2;
876
877 set_userns_rlimit_max(ns: &init_user_ns, type: UCOUNT_RLIMIT_NPROC, RLIM_INFINITY);
878 set_userns_rlimit_max(ns: &init_user_ns, type: UCOUNT_RLIMIT_MSGQUEUE, RLIM_INFINITY);
879 set_userns_rlimit_max(ns: &init_user_ns, type: UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY);
880 set_userns_rlimit_max(ns: &init_user_ns, type: UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY);
881
882#ifdef CONFIG_VMAP_STACK
883 cpuhp_setup_state(state: CPUHP_BP_PREPARE_DYN, name: "fork:vm_stack_cache",
884 NULL, teardown: free_vm_stack_cache);
885#endif
886
887 scs_init();
888
889 lockdep_init_task(task: &init_task);
890 uprobes_init();
891}
892
893int __weak arch_dup_task_struct(struct task_struct *dst,
894 struct task_struct *src)
895{
896 *dst = *src;
897 return 0;
898}
899
900void set_task_stack_end_magic(struct task_struct *tsk)
901{
902 unsigned long *stackend;
903
904 stackend = end_of_stack(task: tsk);
905 *stackend = STACK_END_MAGIC; /* for overflow detection */
906}
907
908static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
909{
910 struct task_struct *tsk;
911 int err;
912
913 if (node == NUMA_NO_NODE)
914 node = tsk_fork_get_node(tsk: orig);
915 tsk = alloc_task_struct_node(node);
916 if (!tsk)
917 return NULL;
918
919 err = arch_dup_task_struct(dst: tsk, src: orig);
920 if (err)
921 goto free_tsk;
922
923 err = alloc_thread_stack_node(tsk, node);
924 if (err)
925 goto free_tsk;
926
927#ifdef CONFIG_THREAD_INFO_IN_TASK
928 refcount_set(r: &tsk->stack_refcount, n: 1);
929#endif
930 account_kernel_stack(tsk, account: 1);
931
932 err = scs_prepare(tsk, node);
933 if (err)
934 goto free_stack;
935
936#ifdef CONFIG_SECCOMP
937 /*
938 * We must handle setting up seccomp filters once we're under
939 * the sighand lock in case orig has changed between now and
940 * then. Until then, filter must be NULL to avoid messing up
941 * the usage counts on the error path calling free_task.
942 */
943 tsk->seccomp.filter = NULL;
944#endif
945
946 setup_thread_stack(tsk, orig);
947 clear_user_return_notifier(p: tsk);
948 clear_tsk_need_resched(tsk);
949 set_task_stack_end_magic(tsk);
950 clear_syscall_work_syscall_user_dispatch(tsk);
951
952#ifdef CONFIG_STACKPROTECTOR
953 tsk->stack_canary = get_random_canary();
954#endif
955 if (orig->cpus_ptr == &orig->cpus_mask)
956 tsk->cpus_ptr = &tsk->cpus_mask;
957 dup_user_cpus_ptr(dst: tsk, src: orig, node);
958
959 /*
960 * One for the user space visible state that goes away when reaped.
961 * One for the scheduler.
962 */
963 refcount_set(r: &tsk->rcu_users, n: 2);
964 /* One for the rcu users */
965 refcount_set(r: &tsk->usage, n: 1);
966#ifdef CONFIG_BLK_DEV_IO_TRACE
967 tsk->btrace_seq = 0;
968#endif
969 tsk->splice_pipe = NULL;
970 tsk->task_frag.page = NULL;
971 tsk->wake_q.next = NULL;
972 tsk->worker_private = NULL;
973
974 kcov_task_init(t: tsk);
975 kmsan_task_create(task: tsk);
976 kmap_local_fork(tsk);
977
978#ifdef CONFIG_FAULT_INJECTION
979 tsk->fail_nth = 0;
980#endif
981
982#ifdef CONFIG_BLK_CGROUP
983 tsk->throttle_disk = NULL;
984 tsk->use_memdelay = 0;
985#endif
986
987#ifdef CONFIG_ARCH_HAS_CPU_PASID
988 tsk->pasid_activated = 0;
989#endif
990
991#ifdef CONFIG_MEMCG
992 tsk->active_memcg = NULL;
993#endif
994
995#ifdef CONFIG_X86_BUS_LOCK_DETECT
996 tsk->reported_split_lock = 0;
997#endif
998
999#ifdef CONFIG_SCHED_MM_CID
1000 tsk->mm_cid.cid = MM_CID_UNSET;
1001 tsk->mm_cid.active = 0;
1002#endif
1003 return tsk;
1004
1005free_stack:
1006 exit_task_stack_account(tsk);
1007 free_thread_stack(tsk);
1008free_tsk:
1009 free_task_struct(tsk);
1010 return NULL;
1011}
1012
1013__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
1014
1015static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
1016
1017static int __init coredump_filter_setup(char *s)
1018{
1019 default_dump_filter =
1020 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
1021 MMF_DUMP_FILTER_MASK;
1022 return 1;
1023}
1024
1025__setup("coredump_filter=", coredump_filter_setup);
1026
1027#include <linux/init_task.h>
1028
1029static void mm_init_aio(struct mm_struct *mm)
1030{
1031#ifdef CONFIG_AIO
1032 spin_lock_init(&mm->ioctx_lock);
1033 mm->ioctx_table = NULL;
1034#endif
1035}
1036
1037static __always_inline void mm_clear_owner(struct mm_struct *mm,
1038 struct task_struct *p)
1039{
1040#ifdef CONFIG_MEMCG
1041 if (mm->owner == p)
1042 WRITE_ONCE(mm->owner, NULL);
1043#endif
1044}
1045
1046static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
1047{
1048#ifdef CONFIG_MEMCG
1049 mm->owner = p;
1050#endif
1051}
1052
1053static void mm_init_uprobes_state(struct mm_struct *mm)
1054{
1055#ifdef CONFIG_UPROBES
1056 mm->uprobes_state.xol_area = NULL;
1057 arch_uprobe_init_state(mm);
1058#endif
1059}
1060
1061static void mmap_init_lock(struct mm_struct *mm)
1062{
1063 init_rwsem(&mm->mmap_lock);
1064 mm_lock_seqcount_init(mm);
1065#ifdef CONFIG_PER_VMA_LOCK
1066 rcuwait_init(w: &mm->vma_writer_wait);
1067#endif
1068}
1069
1070static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
1071 struct user_namespace *user_ns)
1072{
1073 mt_init_flags(mt: &mm->mm_mt, MM_MT_FLAGS);
1074 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
1075 atomic_set(v: &mm->mm_users, i: 1);
1076 atomic_set(v: &mm->mm_count, i: 1);
1077 seqcount_init(&mm->write_protect_seq);
1078 mmap_init_lock(mm);
1079 INIT_LIST_HEAD(list: &mm->mmlist);
1080 mm_pgtables_bytes_init(mm);
1081 mm->map_count = 0;
1082 mm->locked_vm = 0;
1083 atomic64_set(v: &mm->pinned_vm, i: 0);
1084 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
1085 spin_lock_init(&mm->page_table_lock);
1086 spin_lock_init(&mm->arg_lock);
1087 mm_init_cpumask(mm);
1088 mm_init_aio(mm);
1089 mm_init_owner(mm, p);
1090 mm_pasid_init(mm);
1091 RCU_INIT_POINTER(mm->exe_file, NULL);
1092 mmu_notifier_subscriptions_init(mm);
1093 init_tlb_flush_pending(mm);
1094#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
1095 mm->pmd_huge_pte = NULL;
1096#endif
1097 mm_init_uprobes_state(mm);
1098 hugetlb_count_init(mm);
1099
1100 mm_flags_clear_all(mm);
1101 if (current->mm) {
1102 unsigned long flags = __mm_flags_get_word(current->mm);
1103
1104 __mm_flags_overwrite_word(mm, value: mmf_init_legacy_flags(flags));
1105 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
1106 } else {
1107 __mm_flags_overwrite_word(mm, value: default_dump_filter);
1108 mm->def_flags = 0;
1109 }
1110
1111 if (futex_mm_init(mm))
1112 goto fail_mm_init;
1113
1114 if (mm_alloc_pgd(mm))
1115 goto fail_nopgd;
1116
1117 if (mm_alloc_id(mm))
1118 goto fail_noid;
1119
1120 if (init_new_context(tsk: p, mm))
1121 goto fail_nocontext;
1122
1123 if (mm_alloc_cid(mm, p))
1124 goto fail_cid;
1125
1126 if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
1127 NR_MM_COUNTERS))
1128 goto fail_pcpu;
1129
1130 mm->user_ns = get_user_ns(ns: user_ns);
1131 lru_gen_init_mm(mm);
1132 return mm;
1133
1134fail_pcpu:
1135 mm_destroy_cid(mm);
1136fail_cid:
1137 destroy_context(mm);
1138fail_nocontext:
1139 mm_free_id(mm);
1140fail_noid:
1141 mm_free_pgd(mm);
1142fail_nopgd:
1143 futex_hash_free(mm);
1144fail_mm_init:
1145 free_mm(mm);
1146 return NULL;
1147}
1148
1149/*
1150 * Allocate and initialize an mm_struct.
1151 */
1152struct mm_struct *mm_alloc(void)
1153{
1154 struct mm_struct *mm;
1155
1156 mm = allocate_mm();
1157 if (!mm)
1158 return NULL;
1159
1160 memset(mm, 0, sizeof(*mm));
1161 return mm_init(mm, current, current_user_ns());
1162}
1163EXPORT_SYMBOL_IF_KUNIT(mm_alloc);
1164
1165static inline void __mmput(struct mm_struct *mm)
1166{
1167 VM_BUG_ON(atomic_read(&mm->mm_users));
1168
1169 uprobe_clear_state(mm);
1170 exit_aio(mm);
1171 ksm_exit(mm);
1172 khugepaged_exit(mm); /* must run before exit_mmap */
1173 exit_mmap(mm);
1174 mm_put_huge_zero_folio(mm);
1175 set_mm_exe_file(mm, NULL);
1176 if (!list_empty(head: &mm->mmlist)) {
1177 spin_lock(lock: &mmlist_lock);
1178 list_del(entry: &mm->mmlist);
1179 spin_unlock(lock: &mmlist_lock);
1180 }
1181 if (mm->binfmt)
1182 module_put(module: mm->binfmt->module);
1183 lru_gen_del_mm(mm);
1184 futex_hash_free(mm);
1185 mmdrop(mm);
1186}
1187
1188/*
1189 * Decrement the use count and release all resources for an mm.
1190 */
1191void mmput(struct mm_struct *mm)
1192{
1193 might_sleep();
1194
1195 if (atomic_dec_and_test(v: &mm->mm_users))
1196 __mmput(mm);
1197}
1198EXPORT_SYMBOL_GPL(mmput);
1199
1200#if defined(CONFIG_MMU) || defined(CONFIG_FUTEX_PRIVATE_HASH)
1201static void mmput_async_fn(struct work_struct *work)
1202{
1203 struct mm_struct *mm = container_of(work, struct mm_struct,
1204 async_put_work);
1205
1206 __mmput(mm);
1207}
1208
1209void mmput_async(struct mm_struct *mm)
1210{
1211 if (atomic_dec_and_test(v: &mm->mm_users)) {
1212 INIT_WORK(&mm->async_put_work, mmput_async_fn);
1213 schedule_work(work: &mm->async_put_work);
1214 }
1215}
1216EXPORT_SYMBOL_GPL(mmput_async);
1217#endif
1218
1219/**
1220 * set_mm_exe_file - change a reference to the mm's executable file
1221 * @mm: The mm to change.
1222 * @new_exe_file: The new file to use.
1223 *
1224 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1225 *
1226 * Main users are mmput() and sys_execve(). Callers prevent concurrent
1227 * invocations: in mmput() nobody alive left, in execve it happens before
1228 * the new mm is made visible to anyone.
1229 *
1230 * Can only fail if new_exe_file != NULL.
1231 */
1232int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
1233{
1234 struct file *old_exe_file;
1235
1236 /*
1237 * It is safe to dereference the exe_file without RCU as
1238 * this function is only called if nobody else can access
1239 * this mm -- see comment above for justification.
1240 */
1241 old_exe_file = rcu_dereference_raw(mm->exe_file);
1242
1243 if (new_exe_file) {
1244 /*
1245 * We expect the caller (i.e., sys_execve) to already denied
1246 * write access, so this is unlikely to fail.
1247 */
1248 if (unlikely(exe_file_deny_write_access(new_exe_file)))
1249 return -EACCES;
1250 get_file(f: new_exe_file);
1251 }
1252 rcu_assign_pointer(mm->exe_file, new_exe_file);
1253 if (old_exe_file) {
1254 exe_file_allow_write_access(exe_file: old_exe_file);
1255 fput(old_exe_file);
1256 }
1257 return 0;
1258}
1259
1260/**
1261 * replace_mm_exe_file - replace a reference to the mm's executable file
1262 * @mm: The mm to change.
1263 * @new_exe_file: The new file to use.
1264 *
1265 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1266 *
1267 * Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE).
1268 */
1269int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
1270{
1271 struct vm_area_struct *vma;
1272 struct file *old_exe_file;
1273 int ret = 0;
1274
1275 /* Forbid mm->exe_file change if old file still mapped. */
1276 old_exe_file = get_mm_exe_file(mm);
1277 if (old_exe_file) {
1278 VMA_ITERATOR(vmi, mm, 0);
1279 mmap_read_lock(mm);
1280 for_each_vma(vmi, vma) {
1281 if (!vma->vm_file)
1282 continue;
1283 if (path_equal(path1: &vma->vm_file->f_path,
1284 path2: &old_exe_file->f_path)) {
1285 ret = -EBUSY;
1286 break;
1287 }
1288 }
1289 mmap_read_unlock(mm);
1290 fput(old_exe_file);
1291 if (ret)
1292 return ret;
1293 }
1294
1295 ret = exe_file_deny_write_access(exe_file: new_exe_file);
1296 if (ret)
1297 return -EACCES;
1298 get_file(f: new_exe_file);
1299
1300 /* set the new file */
1301 mmap_write_lock(mm);
1302 old_exe_file = rcu_dereference_raw(mm->exe_file);
1303 rcu_assign_pointer(mm->exe_file, new_exe_file);
1304 mmap_write_unlock(mm);
1305
1306 if (old_exe_file) {
1307 exe_file_allow_write_access(exe_file: old_exe_file);
1308 fput(old_exe_file);
1309 }
1310 return 0;
1311}
1312
1313/**
1314 * get_mm_exe_file - acquire a reference to the mm's executable file
1315 * @mm: The mm of interest.
1316 *
1317 * Returns %NULL if mm has no associated executable file.
1318 * User must release file via fput().
1319 */
1320struct file *get_mm_exe_file(struct mm_struct *mm)
1321{
1322 struct file *exe_file;
1323
1324 rcu_read_lock();
1325 exe_file = get_file_rcu(f: &mm->exe_file);
1326 rcu_read_unlock();
1327 return exe_file;
1328}
1329
1330/**
1331 * get_task_exe_file - acquire a reference to the task's executable file
1332 * @task: The task.
1333 *
1334 * Returns %NULL if task's mm (if any) has no associated executable file or
1335 * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1336 * User must release file via fput().
1337 */
1338struct file *get_task_exe_file(struct task_struct *task)
1339{
1340 struct file *exe_file = NULL;
1341 struct mm_struct *mm;
1342
1343 if (task->flags & PF_KTHREAD)
1344 return NULL;
1345
1346 task_lock(p: task);
1347 mm = task->mm;
1348 if (mm)
1349 exe_file = get_mm_exe_file(mm);
1350 task_unlock(p: task);
1351 return exe_file;
1352}
1353
1354/**
1355 * get_task_mm - acquire a reference to the task's mm
1356 * @task: The task.
1357 *
1358 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
1359 * this kernel workthread has transiently adopted a user mm with use_mm,
1360 * to do its AIO) is not set and if so returns a reference to it, after
1361 * bumping up the use count. User must release the mm via mmput()
1362 * after use. Typically used by /proc and ptrace.
1363 */
1364struct mm_struct *get_task_mm(struct task_struct *task)
1365{
1366 struct mm_struct *mm;
1367
1368 if (task->flags & PF_KTHREAD)
1369 return NULL;
1370
1371 task_lock(p: task);
1372 mm = task->mm;
1373 if (mm)
1374 mmget(mm);
1375 task_unlock(p: task);
1376 return mm;
1377}
1378EXPORT_SYMBOL_GPL(get_task_mm);
1379
1380static bool may_access_mm(struct mm_struct *mm, struct task_struct *task, unsigned int mode)
1381{
1382 if (mm == current->mm)
1383 return true;
1384 if (ptrace_may_access(task, mode))
1385 return true;
1386 if ((mode & PTRACE_MODE_READ) && perfmon_capable())
1387 return true;
1388 return false;
1389}
1390
1391struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
1392{
1393 struct mm_struct *mm;
1394 int err;
1395
1396 err = down_read_killable(sem: &task->signal->exec_update_lock);
1397 if (err)
1398 return ERR_PTR(error: err);
1399
1400 mm = get_task_mm(task);
1401 if (!mm) {
1402 mm = ERR_PTR(error: -ESRCH);
1403 } else if (!may_access_mm(mm, task, mode)) {
1404 mmput(mm);
1405 mm = ERR_PTR(error: -EACCES);
1406 }
1407 up_read(sem: &task->signal->exec_update_lock);
1408
1409 return mm;
1410}
1411
1412static void complete_vfork_done(struct task_struct *tsk)
1413{
1414 struct completion *vfork;
1415
1416 task_lock(p: tsk);
1417 vfork = tsk->vfork_done;
1418 if (likely(vfork)) {
1419 tsk->vfork_done = NULL;
1420 complete(vfork);
1421 }
1422 task_unlock(p: tsk);
1423}
1424
1425static int wait_for_vfork_done(struct task_struct *child,
1426 struct completion *vfork)
1427{
1428 unsigned int state = TASK_KILLABLE|TASK_FREEZABLE;
1429 int killed;
1430
1431 cgroup_enter_frozen();
1432 killed = wait_for_completion_state(x: vfork, state);
1433 cgroup_leave_frozen(always_leave: false);
1434
1435 if (killed) {
1436 task_lock(p: child);
1437 child->vfork_done = NULL;
1438 task_unlock(p: child);
1439 }
1440
1441 put_task_struct(t: child);
1442 return killed;
1443}
1444
1445/* Please note the differences between mmput and mm_release.
1446 * mmput is called whenever we stop holding onto a mm_struct,
1447 * error success whatever.
1448 *
1449 * mm_release is called after a mm_struct has been removed
1450 * from the current process.
1451 *
1452 * This difference is important for error handling, when we
1453 * only half set up a mm_struct for a new process and need to restore
1454 * the old one. Because we mmput the new mm_struct before
1455 * restoring the old one. . .
1456 * Eric Biederman 10 January 1998
1457 */
1458static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
1459{
1460 uprobe_free_utask(t: tsk);
1461
1462 /* Get rid of any cached register state */
1463 deactivate_mm(tsk, mm);
1464
1465 /*
1466 * Signal userspace if we're not exiting with a core dump
1467 * because we want to leave the value intact for debugging
1468 * purposes.
1469 */
1470 if (tsk->clear_child_tid) {
1471 if (atomic_read(v: &mm->mm_users) > 1) {
1472 /*
1473 * We don't check the error code - if userspace has
1474 * not set up a proper pointer then tough luck.
1475 */
1476 put_user(0, tsk->clear_child_tid);
1477 do_futex(uaddr: tsk->clear_child_tid, FUTEX_WAKE,
1478 val: 1, NULL, NULL, val2: 0, val3: 0);
1479 }
1480 tsk->clear_child_tid = NULL;
1481 }
1482
1483 /*
1484 * All done, finally we can wake up parent and return this mm to him.
1485 * Also kthread_stop() uses this completion for synchronization.
1486 */
1487 if (tsk->vfork_done)
1488 complete_vfork_done(tsk);
1489}
1490
1491void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
1492{
1493 futex_exit_release(tsk);
1494 mm_release(tsk, mm);
1495}
1496
1497void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
1498{
1499 futex_exec_release(tsk);
1500 mm_release(tsk, mm);
1501}
1502
1503/**
1504 * dup_mm() - duplicates an existing mm structure
1505 * @tsk: the task_struct with which the new mm will be associated.
1506 * @oldmm: the mm to duplicate.
1507 *
1508 * Allocates a new mm structure and duplicates the provided @oldmm structure
1509 * content into it.
1510 *
1511 * Return: the duplicated mm or NULL on failure.
1512 */
1513static struct mm_struct *dup_mm(struct task_struct *tsk,
1514 struct mm_struct *oldmm)
1515{
1516 struct mm_struct *mm;
1517 int err;
1518
1519 mm = allocate_mm();
1520 if (!mm)
1521 goto fail_nomem;
1522
1523 memcpy(mm, oldmm, sizeof(*mm));
1524
1525 if (!mm_init(mm, p: tsk, user_ns: mm->user_ns))
1526 goto fail_nomem;
1527
1528 uprobe_start_dup_mmap();
1529 err = dup_mmap(mm, oldmm);
1530 if (err)
1531 goto free_pt;
1532 uprobe_end_dup_mmap();
1533
1534 mm->hiwater_rss = get_mm_rss(mm);
1535 mm->hiwater_vm = mm->total_vm;
1536
1537 if (mm->binfmt && !try_module_get(module: mm->binfmt->module))
1538 goto free_pt;
1539
1540 return mm;
1541
1542free_pt:
1543 /* don't put binfmt in mmput, we haven't got module yet */
1544 mm->binfmt = NULL;
1545 mm_init_owner(mm, NULL);
1546 mmput(mm);
1547 if (err)
1548 uprobe_end_dup_mmap();
1549
1550fail_nomem:
1551 return NULL;
1552}
1553
1554static int copy_mm(u64 clone_flags, struct task_struct *tsk)
1555{
1556 struct mm_struct *mm, *oldmm;
1557
1558 tsk->min_flt = tsk->maj_flt = 0;
1559 tsk->nvcsw = tsk->nivcsw = 0;
1560#ifdef CONFIG_DETECT_HUNG_TASK
1561 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
1562 tsk->last_switch_time = 0;
1563#endif
1564
1565 tsk->mm = NULL;
1566 tsk->active_mm = NULL;
1567
1568 /*
1569 * Are we cloning a kernel thread?
1570 *
1571 * We need to steal a active VM for that..
1572 */
1573 oldmm = current->mm;
1574 if (!oldmm)
1575 return 0;
1576
1577 if (clone_flags & CLONE_VM) {
1578 mmget(mm: oldmm);
1579 mm = oldmm;
1580 } else {
1581 mm = dup_mm(tsk, current->mm);
1582 if (!mm)
1583 return -ENOMEM;
1584 }
1585
1586 tsk->mm = mm;
1587 tsk->active_mm = mm;
1588 sched_mm_cid_fork(t: tsk);
1589 return 0;
1590}
1591
1592static int copy_fs(u64 clone_flags, struct task_struct *tsk)
1593{
1594 struct fs_struct *fs = current->fs;
1595 if (clone_flags & CLONE_FS) {
1596 /* tsk->fs is already what we want */
1597 read_seqlock_excl(sl: &fs->seq);
1598 /* "users" and "in_exec" locked for check_unsafe_exec() */
1599 if (fs->in_exec) {
1600 read_sequnlock_excl(sl: &fs->seq);
1601 return -EAGAIN;
1602 }
1603 fs->users++;
1604 read_sequnlock_excl(sl: &fs->seq);
1605 return 0;
1606 }
1607 tsk->fs = copy_fs_struct(fs);
1608 if (!tsk->fs)
1609 return -ENOMEM;
1610 return 0;
1611}
1612
1613static int copy_files(u64 clone_flags, struct task_struct *tsk,
1614 int no_files)
1615{
1616 struct files_struct *oldf, *newf;
1617
1618 /*
1619 * A background process may not have any files ...
1620 */
1621 oldf = current->files;
1622 if (!oldf)
1623 return 0;
1624
1625 if (no_files) {
1626 tsk->files = NULL;
1627 return 0;
1628 }
1629
1630 if (clone_flags & CLONE_FILES) {
1631 atomic_inc(v: &oldf->count);
1632 return 0;
1633 }
1634
1635 newf = dup_fd(oldf, NULL);
1636 if (IS_ERR(ptr: newf))
1637 return PTR_ERR(ptr: newf);
1638
1639 tsk->files = newf;
1640 return 0;
1641}
1642
1643static int copy_sighand(u64 clone_flags, struct task_struct *tsk)
1644{
1645 struct sighand_struct *sig;
1646
1647 if (clone_flags & CLONE_SIGHAND) {
1648 refcount_inc(r: &current->sighand->count);
1649 return 0;
1650 }
1651 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1652 RCU_INIT_POINTER(tsk->sighand, sig);
1653 if (!sig)
1654 return -ENOMEM;
1655
1656 refcount_set(r: &sig->count, n: 1);
1657 spin_lock_irq(lock: &current->sighand->siglock);
1658 memcpy(sig->action, current->sighand->action, sizeof(sig->action));
1659 spin_unlock_irq(lock: &current->sighand->siglock);
1660
1661 /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
1662 if (clone_flags & CLONE_CLEAR_SIGHAND)
1663 flush_signal_handlers(tsk, force_default: 0);
1664
1665 return 0;
1666}
1667
1668void __cleanup_sighand(struct sighand_struct *sighand)
1669{
1670 if (refcount_dec_and_test(r: &sighand->count)) {
1671 signalfd_cleanup(sighand);
1672 /*
1673 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
1674 * without an RCU grace period, see __lock_task_sighand().
1675 */
1676 kmem_cache_free(s: sighand_cachep, objp: sighand);
1677 }
1678}
1679
1680/*
1681 * Initialize POSIX timer handling for a thread group.
1682 */
1683static void posix_cpu_timers_init_group(struct signal_struct *sig)
1684{
1685 struct posix_cputimers *pct = &sig->posix_cputimers;
1686 unsigned long cpu_limit;
1687
1688 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1689 posix_cputimers_group_init(pct, cpu_limit);
1690}
1691
1692static int copy_signal(u64 clone_flags, struct task_struct *tsk)
1693{
1694 struct signal_struct *sig;
1695
1696 if (clone_flags & CLONE_THREAD)
1697 return 0;
1698
1699 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
1700 tsk->signal = sig;
1701 if (!sig)
1702 return -ENOMEM;
1703
1704 sig->nr_threads = 1;
1705 sig->quick_threads = 1;
1706 atomic_set(v: &sig->live, i: 1);
1707 refcount_set(r: &sig->sigcnt, n: 1);
1708
1709 /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
1710 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
1711 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
1712
1713 init_waitqueue_head(&sig->wait_chldexit);
1714 sig->curr_target = tsk;
1715 init_sigpending(sig: &sig->shared_pending);
1716 INIT_HLIST_HEAD(&sig->multiprocess);
1717 seqlock_init(&sig->stats_lock);
1718 prev_cputime_init(prev: &sig->prev_cputime);
1719
1720#ifdef CONFIG_POSIX_TIMERS
1721 INIT_HLIST_HEAD(&sig->posix_timers);
1722 INIT_HLIST_HEAD(&sig->ignored_posix_timers);
1723 hrtimer_setup(timer: &sig->real_timer, function: it_real_fn, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
1724#endif
1725
1726 task_lock(current->group_leader);
1727 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
1728 task_unlock(current->group_leader);
1729
1730 posix_cpu_timers_init_group(sig);
1731
1732 tty_audit_fork(sig);
1733 sched_autogroup_fork(sig);
1734
1735#ifdef CONFIG_CGROUPS
1736 init_rwsem(&sig->cgroup_threadgroup_rwsem);
1737#endif
1738
1739 sig->oom_score_adj = current->signal->oom_score_adj;
1740 sig->oom_score_adj_min = current->signal->oom_score_adj_min;
1741
1742 mutex_init(&sig->cred_guard_mutex);
1743 init_rwsem(&sig->exec_update_lock);
1744
1745 return 0;
1746}
1747
1748static void copy_seccomp(struct task_struct *p)
1749{
1750#ifdef CONFIG_SECCOMP
1751 /*
1752 * Must be called with sighand->lock held, which is common to
1753 * all threads in the group. Holding cred_guard_mutex is not
1754 * needed because this new task is not yet running and cannot
1755 * be racing exec.
1756 */
1757 assert_spin_locked(&current->sighand->siglock);
1758
1759 /* Ref-count the new filter user, and assign it. */
1760 get_seccomp_filter(current);
1761 p->seccomp = current->seccomp;
1762
1763 /*
1764 * Explicitly enable no_new_privs here in case it got set
1765 * between the task_struct being duplicated and holding the
1766 * sighand lock. The seccomp state and nnp must be in sync.
1767 */
1768 if (task_no_new_privs(current))
1769 task_set_no_new_privs(p);
1770
1771 /*
1772 * If the parent gained a seccomp mode after copying thread
1773 * flags and between before we held the sighand lock, we have
1774 * to manually enable the seccomp thread flag here.
1775 */
1776 if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1777 set_task_syscall_work(p, SECCOMP);
1778#endif
1779}
1780
1781SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
1782{
1783 current->clear_child_tid = tidptr;
1784
1785 return task_pid_vnr(current);
1786}
1787
1788static void rt_mutex_init_task(struct task_struct *p)
1789{
1790 raw_spin_lock_init(&p->pi_lock);
1791#ifdef CONFIG_RT_MUTEXES
1792 p->pi_waiters = RB_ROOT_CACHED;
1793 p->pi_top_task = NULL;
1794 p->pi_blocked_on = NULL;
1795#endif
1796}
1797
1798static inline void init_task_pid_links(struct task_struct *task)
1799{
1800 enum pid_type type;
1801
1802 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type)
1803 INIT_HLIST_NODE(h: &task->pid_links[type]);
1804}
1805
1806static inline void
1807init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
1808{
1809 if (type == PIDTYPE_PID)
1810 task->thread_pid = pid;
1811 else
1812 task->signal->pids[type] = pid;
1813}
1814
1815static inline void rcu_copy_process(struct task_struct *p)
1816{
1817#ifdef CONFIG_PREEMPT_RCU
1818 p->rcu_read_lock_nesting = 0;
1819 p->rcu_read_unlock_special.s = 0;
1820 p->rcu_blocked_node = NULL;
1821 INIT_LIST_HEAD(list: &p->rcu_node_entry);
1822#endif /* #ifdef CONFIG_PREEMPT_RCU */
1823#ifdef CONFIG_TASKS_RCU
1824 p->rcu_tasks_holdout = false;
1825 INIT_LIST_HEAD(list: &p->rcu_tasks_holdout_list);
1826 p->rcu_tasks_idle_cpu = -1;
1827 INIT_LIST_HEAD(list: &p->rcu_tasks_exit_list);
1828#endif /* #ifdef CONFIG_TASKS_RCU */
1829#ifdef CONFIG_TASKS_TRACE_RCU
1830 p->trc_reader_nesting = 0;
1831 p->trc_reader_special.s = 0;
1832 INIT_LIST_HEAD(list: &p->trc_holdout_list);
1833 INIT_LIST_HEAD(list: &p->trc_blkd_node);
1834#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
1835}
1836
1837/**
1838 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
1839 * @pid: the struct pid for which to create a pidfd
1840 * @flags: flags of the new @pidfd
1841 * @ret_file: return the new pidfs file
1842 *
1843 * Allocate a new file that stashes @pid and reserve a new pidfd number in the
1844 * caller's file descriptor table. The pidfd is reserved but not installed yet.
1845 *
1846 * The helper verifies that @pid is still in use, without PIDFD_THREAD the
1847 * task identified by @pid must be a thread-group leader.
1848 *
1849 * If this function returns successfully the caller is responsible to either
1850 * call fd_install() passing the returned pidfd and pidfd file as arguments in
1851 * order to install the pidfd into its file descriptor table or they must use
1852 * put_unused_fd() and fput() on the returned pidfd and pidfd file
1853 * respectively.
1854 *
1855 * This function is useful when a pidfd must already be reserved but there
1856 * might still be points of failure afterwards and the caller wants to ensure
1857 * that no pidfd is leaked into its file descriptor table.
1858 *
1859 * Return: On success, a reserved pidfd is returned from the function and a new
1860 * pidfd file is returned in the last argument to the function. On
1861 * error, a negative error code is returned from the function and the
1862 * last argument remains unchanged.
1863 */
1864int pidfd_prepare(struct pid *pid, unsigned int flags, struct file **ret_file)
1865{
1866 struct file *pidfs_file;
1867
1868 /*
1869 * PIDFD_STALE is only allowed to be passed if the caller knows
1870 * that @pid is already registered in pidfs and thus
1871 * PIDFD_INFO_EXIT information is guaranteed to be available.
1872 */
1873 if (!(flags & PIDFD_STALE)) {
1874 /*
1875 * While holding the pidfd waitqueue lock removing the
1876 * task linkage for the thread-group leader pid
1877 * (PIDTYPE_TGID) isn't possible. Thus, if there's still
1878 * task linkage for PIDTYPE_PID not having thread-group
1879 * leader linkage for the pid means it wasn't a
1880 * thread-group leader in the first place.
1881 */
1882 guard(spinlock_irq)(l: &pid->wait_pidfd.lock);
1883
1884 /* Task has already been reaped. */
1885 if (!pid_has_task(pid, type: PIDTYPE_PID))
1886 return -ESRCH;
1887 /*
1888 * If this struct pid isn't used as a thread-group
1889 * leader but the caller requested to create a
1890 * thread-group leader pidfd then report ENOENT.
1891 */
1892 if (!(flags & PIDFD_THREAD) && !pid_has_task(pid, type: PIDTYPE_TGID))
1893 return -ENOENT;
1894 }
1895
1896 CLASS(get_unused_fd, pidfd)(O_CLOEXEC);
1897 if (pidfd < 0)
1898 return pidfd;
1899
1900 pidfs_file = pidfs_alloc_file(pid, flags: flags | O_RDWR);
1901 if (IS_ERR(ptr: pidfs_file))
1902 return PTR_ERR(ptr: pidfs_file);
1903
1904 *ret_file = pidfs_file;
1905 return take_fd(pidfd);
1906}
1907
1908static void __delayed_free_task(struct rcu_head *rhp)
1909{
1910 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
1911
1912 free_task(tsk);
1913}
1914
1915static __always_inline void delayed_free_task(struct task_struct *tsk)
1916{
1917 if (IS_ENABLED(CONFIG_MEMCG))
1918 call_rcu(head: &tsk->rcu, func: __delayed_free_task);
1919 else
1920 free_task(tsk);
1921}
1922
1923static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
1924{
1925 /* Skip if kernel thread */
1926 if (!tsk->mm)
1927 return;
1928
1929 /* Skip if spawning a thread or using vfork */
1930 if ((clone_flags & (CLONE_VM | CLONE_THREAD | CLONE_VFORK)) != CLONE_VM)
1931 return;
1932
1933 /* We need to synchronize with __set_oom_adj */
1934 mutex_lock(&oom_adj_mutex);
1935 mm_flags_set(MMF_MULTIPROCESS, mm: tsk->mm);
1936 /* Update the values in case they were changed after copy_signal */
1937 tsk->signal->oom_score_adj = current->signal->oom_score_adj;
1938 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min;
1939 mutex_unlock(lock: &oom_adj_mutex);
1940}
1941
1942#ifdef CONFIG_RV
1943static void rv_task_fork(struct task_struct *p)
1944{
1945 memset(&p->rv, 0, sizeof(p->rv));
1946}
1947#else
1948#define rv_task_fork(p) do {} while (0)
1949#endif
1950
1951static bool need_futex_hash_allocate_default(u64 clone_flags)
1952{
1953 if ((clone_flags & (CLONE_THREAD | CLONE_VM)) != (CLONE_THREAD | CLONE_VM))
1954 return false;
1955 return true;
1956}
1957
1958/*
1959 * This creates a new process as a copy of the old one,
1960 * but does not actually start it yet.
1961 *
1962 * It copies the registers, and all the appropriate
1963 * parts of the process environment (as per the clone
1964 * flags). The actual kick-off is left to the caller.
1965 */
1966__latent_entropy struct task_struct *copy_process(
1967 struct pid *pid,
1968 int trace,
1969 int node,
1970 struct kernel_clone_args *args)
1971{
1972 int pidfd = -1, retval;
1973 struct task_struct *p;
1974 struct multiprocess_signals delayed;
1975 struct file *pidfile = NULL;
1976 const u64 clone_flags = args->flags;
1977 struct nsproxy *nsp = current->nsproxy;
1978
1979 /*
1980 * Don't allow sharing the root directory with processes in a different
1981 * namespace
1982 */
1983 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
1984 return ERR_PTR(error: -EINVAL);
1985
1986 if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
1987 return ERR_PTR(error: -EINVAL);
1988
1989 /*
1990 * Thread groups must share signals as well, and detached threads
1991 * can only be started up within the thread group.
1992 */
1993 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
1994 return ERR_PTR(error: -EINVAL);
1995
1996 /*
1997 * Shared signal handlers imply shared VM. By way of the above,
1998 * thread groups also imply shared VM. Blocking this case allows
1999 * for various simplifications in other code.
2000 */
2001 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
2002 return ERR_PTR(error: -EINVAL);
2003
2004 /*
2005 * Siblings of global init remain as zombies on exit since they are
2006 * not reaped by their parent (swapper). To solve this and to avoid
2007 * multi-rooted process trees, prevent global and container-inits
2008 * from creating siblings.
2009 */
2010 if ((clone_flags & CLONE_PARENT) &&
2011 current->signal->flags & SIGNAL_UNKILLABLE)
2012 return ERR_PTR(error: -EINVAL);
2013
2014 /*
2015 * If the new process will be in a different pid or user namespace
2016 * do not allow it to share a thread group with the forking task.
2017 */
2018 if (clone_flags & CLONE_THREAD) {
2019 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
2020 (task_active_pid_ns(current) != nsp->pid_ns_for_children))
2021 return ERR_PTR(error: -EINVAL);
2022 }
2023
2024 if (clone_flags & CLONE_PIDFD) {
2025 /*
2026 * - CLONE_DETACHED is blocked so that we can potentially
2027 * reuse it later for CLONE_PIDFD.
2028 */
2029 if (clone_flags & CLONE_DETACHED)
2030 return ERR_PTR(error: -EINVAL);
2031 }
2032
2033 /*
2034 * Force any signals received before this point to be delivered
2035 * before the fork happens. Collect up signals sent to multiple
2036 * processes that happen during the fork and delay them so that
2037 * they appear to happen after the fork.
2038 */
2039 sigemptyset(set: &delayed.signal);
2040 INIT_HLIST_NODE(h: &delayed.node);
2041
2042 spin_lock_irq(lock: &current->sighand->siglock);
2043 if (!(clone_flags & CLONE_THREAD))
2044 hlist_add_head(n: &delayed.node, h: &current->signal->multiprocess);
2045 recalc_sigpending();
2046 spin_unlock_irq(lock: &current->sighand->siglock);
2047 retval = -ERESTARTNOINTR;
2048 if (task_sigpending(current))
2049 goto fork_out;
2050
2051 retval = -ENOMEM;
2052 p = dup_task_struct(current, node);
2053 if (!p)
2054 goto fork_out;
2055 p->flags &= ~PF_KTHREAD;
2056 if (args->kthread)
2057 p->flags |= PF_KTHREAD;
2058 if (args->user_worker) {
2059 /*
2060 * Mark us a user worker, and block any signal that isn't
2061 * fatal or STOP
2062 */
2063 p->flags |= PF_USER_WORKER;
2064 siginitsetinv(set: &p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP));
2065 }
2066 if (args->io_thread)
2067 p->flags |= PF_IO_WORKER;
2068
2069 if (args->You are visitor 01402503 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.