/**********************************************************************/

/* This file contains much of the glue between the Solaris code in */

/* dtrace.c and the linux kernel. We emulate much of the missing */

/* functionality, or map into the kernel. */

/* */

/* Date: April 2008 */

/* Author: Paul D. Fox */

/* */

/* License: CDDL */

/* */

/* $Header: Last edited: 04-Feb-2013 1.14 $ */

/**********************************************************************/

#include <linux/mm.h>

# undef zone

# define zone linux_zone

#include "dtrace_linux.h"

#include <sys/dtrace_impl.h>

#include "dtrace_proto.h"

#include "proc_compat.h"

#include <linux/cpumask.h>

#include <linux/errno.h>

#include <linux/miscdevice.h>

#include <linux/fs.h>

#include <linux/proc_fs.h>

#include <linux/seq_file.h>

#include <linux/module.h>

#include <linux/slab.h>

#include <linux/sys.h>

#include <linux/thread_info.h>

#include <linux/profile.h>

#include <linux/vmalloc.h>

#include <asm/tlbflush.h>

#include <asm/current.h>

# if defined(__i386) || defined(__amd64)

# include <asm/desc.h>

# endif

#include <sys/rwlock.h>

#include <sys/privregs.h>

//#include <asm/pgtable.h>

//#include <asm/pgalloc.h>

MODULE_AUTHOR("Paul D. Fox");

MODULE_LICENSE(DRIVER_LICENSE);

MODULE_DESCRIPTION("DTRACEDRV Driver");

# define TRACE_ALLOC 0

/**********************************************************************/

/* Turn on HERE() macro tracing. */

/**********************************************************************/

int dtrace_here;

module_param(dtrace_here, int, 0);

int dtrace_mem_alloc;

module_param(dtrace_mem_alloc, int, 0);

int dtrace_unhandled;

module_param(dtrace_unhandled, int, 0);

int fbt_name_opcodes;

module_param(fbt_name_opcodes, int, 0);

int grab_panic;

module_param(grab_panic, int, 0);

char *arg_kallsyms_lookup_name; /* Done as a string, because kernel doesnt */

/* like 0xfffffff12345678 as a number. */

module_param(arg_kallsyms_lookup_name, charp, 0);

extern char dtrace_buf[];

extern const int log_bufsiz;

extern int dbuf_i;

extern int dtrace_safe;

/**********************************************************************/

/* TRUE when we have called dtrace_linux_init(). After that point, */

/* xcalls are valid, but until then, we need to wait for the */

/* externally loaded symbol patchups. */

/**********************************************************************/

int driver_initted;

/**********************************************************************/

/* Stuff we stash away from /proc/kallsyms. */

/**********************************************************************/

enum {

OFFSET_kallsyms_lookup_name,

OFFSET_modules,

OFFSET_sys_call_table,

OFFSET_access_process_vm,

OFFSET_syscall_call,

OFFSET_print_modules,

OFFSET_task_exit_notifier,

OFFSET_xtime,

OFFSET_ia32_sys_call_table,

OFFSET_old_rsp,

OFFSET_END_SYMS,

};

static struct map {

char *m_name;

unsigned long *m_ptr;

} syms[] = {

{"kallsyms_lookup_name", NULL},

{"modules", NULL},

{"sys_call_table", NULL},

{"access_process_vm", NULL},

{"syscall_call", NULL}, /* Backup for i386 2.6.23 kernel to help */

/* find the sys_call_table. */

{"print_modules", NULL}, /* Backup for i386 2.6.23 kernel to help */

/* find the modules table. */

{"task_exit_notifier", NULL},

{"xtime", NULL}, /* Needed for dtrace_gethrtime, if 2.6.9 */

{"ia32_sys_call_table", NULL}, /* On 64b kernel, the 32b syscall table. */

{"old_rsp", NULL}, /* This has moved to System.map in 3.7 or above */

{"END_SYMS", NULL}, /* This is a sentinel so we know we are done. */

{0}

};

static unsigned long (*xkallsyms_lookup_name)(char *);

static void *xmodules;

static void **xsys_call_table;

static void **xia32_sys_call_table;

static void (*fn_sysrq_showregs_othercpus)(void *);

int (*kernel_text_address_fn)(unsigned long);

char *(*dentry_path_fn)(struct dentry *, char *, int);

static struct module *(*fn__module_text_address)(unsigned long);

void *(*fn_pid_task)(void *, int);

void *(*fn_find_get_pid)(int);

/**********************************************************************/

/* Stats counters for ad hoc debugging; exposed via */

/* /proc/dtrace/stats. */

/**********************************************************************/

unsigned long dcnt[MAX_DCNT];

/**********************************************************************/

/* The security profiles, loaded via /dev/dtrace. See comment in */

/* dtrace_linux.h for details on how this works. */

/**********************************************************************/

# define MAX_SEC_LIST 64

int di_cnt;

dsec_item_t di_list[MAX_SEC_LIST];

/**********************************************************************/

/* The kernel can be compiled with a lot of potential CPUs, e.g. */

/* 64 is not untypical, but we only have a dual core cpu. We */

/* allocate buffers for each cpu - which can mushroom the memory */

/* needed (4MB+ per core), and can OOM for small systems or put */

/* other systems into mortal danger as we eat all the memory. */

/**********************************************************************/

cpu_t *cpu_list;

cpu_core_t *cpu_core;

cpu_t *cpu_table;

cred_t *cpu_cred;

int nr_cpus = 1;

MUTEX_DEFINE(mod_lock);

/**********************************************************************/

/* Set to true by debug code that wants to immediately disable */

/* probes so we can diagnose what was happening. */

/**********************************************************************/

int dtrace_shutdown;

/**********************************************************************/

/* We need one of these for every process on the system. Linux */

/* doesnt provide a way to find a process being created, and even */

/* if it did, we need to allocate memory when that happens, which */

/* is not viable. So we preallocate all the space we need up front */

/* during driver init. This isnt nice, since that max_pid variable */

/* can change, but typically doesnt. */

/**********************************************************************/

sol_proc_t *shadow_procs;

MUTEX_DEFINE(cpu_lock);

int panic_quiesce;

sol_proc_t *curthread;

dtrace_vtime_state_t dtrace_vtime_active = 0;

dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1;

/**********************************************************************/

/* For dtrace_gethrtime. */

/**********************************************************************/

static int tsc_max_delta;

static struct timespec *xtime_cache_ptr;

ktime_t (*ktime_get_ptr)(void);

struct timespec (*__current_kernel_time_ptr)(void);

static u64 (*native_sched_clock_ptr)(void);

/**********************************************************************/

/* Ensure we are at the head of the chains. Unfortunately, kprobes */

/* puts itself at the front of the chain and the notifier calls */

/* wont let us go first - which we need in order to avoid */

/* re-entrancy issues. We have to work around that in */

/* dtrace_linux_init(). */

/**********************************************************************/

# define NOTIFIER_MAX_PRIO 0x7fffffff

/**********************************************************************/

/* Stuff for INT3 interception. */

/**********************************************************************/

static int (*fn_profile_event_register)(enum profile_type type, struct notifier_block *n);

static int (*fn_profile_event_unregister)(enum profile_type type, struct notifier_block *n);

/**********************************************************************/

/* Notifier for invalid instruction trap. */

/**********************************************************************/

# if 0

static int proc_notifier_trap_illop(struct notifier_block *, unsigned long, void *);

static struct notifier_block n_trap_illop = {

.notifier_call = proc_notifier_trap_illop,

.priority = NOTIFIER_MAX_PRIO, // notify us first - before kprobes

};

# endif

/**********************************************************************/

/* Process exiting notifier. */

/**********************************************************************/

static int proc_exit_notifier(struct notifier_block *, unsigned long, void *);

static struct notifier_block n_exit = {

.notifier_call = proc_exit_notifier,

.priority = NOTIFIER_MAX_PRIO, // notify us first - before kprobes

};

/**********************************************************************/

/* Used when we want to intercept panics for final autopsy of what */

/* we did wrong. */

/**********************************************************************/

#if 0

int dtrace_kernel_panic(struct notifier_block *this, unsigned long event, void *ptr);

static struct notifier_block panic_notifier = {

.notifier_call = dtrace_kernel_panic,

};

#endif

/**********************************************************************/

/* Externs. */

/**********************************************************************/

extern unsigned long long cnt_timer_add;

extern unsigned long long cnt_timer_remove;

extern unsigned long long cnt_syscall1;

extern unsigned long long cnt_syscall2;

extern unsigned long long cnt_syscall3;

extern unsigned long cnt_xcall0;

extern unsigned long cnt_xcall1;

extern unsigned long cnt_xcall2;

extern unsigned long cnt_xcall3;

extern unsigned long cnt_xcall4;

extern unsigned long cnt_xcall5;

extern unsigned long long cnt_xcall6;

extern unsigned long long cnt_xcall7;

extern unsigned long cnt_xcall8;

extern unsigned long cnt_xcall_slave;

extern unsigned long cnt_nmi1;

extern unsigned long cnt_nmi2;

extern unsigned long long cnt_timer1;

extern unsigned long long cnt_timer2;

extern unsigned long long cnt_timer3;

/**********************************************************************/

/* Prototypes. */

/**********************************************************************/

void ctf_setup(void);

static void * par_lookup(void *ptr);

# define cas32 dtrace_cas32

uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new);

int ctf_init(void);

void ctf_exit(void);

int ctl_init(void);

void ctl_exit(void);

int dcpc_init(void);

void dcpc_exit(void);

int dtrace_profile_init(void);

int dtrace_profile_fini(void);

int fasttrap_init(void);

void fasttrap_exit(void);

int fbt_init(void);

int fbt_init2(void);

void fbt_exit(void);

int instr_init(void);

void instr_exit(void);

int intr_init(void);

void intr_exit(void);

int dtrace_prcom_init(void);

void dtrace_prcom_exit(void);

int sdt_init(void);

void sdt_exit(void);

int signal_init(void);

void signal_fini(void);

int systrace_init(void);

void systrace_exit(void);

void io_prov_init(void);

void xcall_init(void);

void xcall_fini(void);

//static void print_pte(pte_t *pte, int level);

/**********************************************************************/

/* Avoid problems with old kernels which have conflicting */

/* definitions. */

/**********************************************************************/

void

dtrace_clflush(void *ptr)

{

# if defined(__arm__)

local_flush_tlb_kernel_page(ptr);

# else

__asm__("clflush %0\n" : "+m" (*(char *)ptr));

# endif

}

/**********************************************************************/

/* Return the credentials of the current process. Solaris assumes */

/* we are embedded inside the proc/user struct, but in Linux, we */

/* have different linages. Earlier kernels had explicit */

/* uid/euid/... fields, and from 2.6.29, a separate cred */

/* structure. We need to hide that from the kernel. */

/* */

/* In addition, because we are encapsulating, we need to be */

/* careful of SMP systems - we cannot use a static, so we use a */

/* per-cpu array so that any CPU wont disturb the cached */

/* credentials we are picking up. */

/**********************************************************************/

cred_t *

CRED()

{ cred_t *cr = &cpu_cred[cpu_get_id()];

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)

cr->cr_uid = current->cred->uid;

cr->cr_gid = current->cred->gid;

#else

cr->cr_uid = current->uid;

cr->cr_gid = current->gid;

#endif

//printk("get cred end %d %d\n", cr->cr_uid, cr->cr_gid);

return cr;

}

# if 0

/**********************************************************************/

/* Placeholder code -- we need to avoid a linear tomax/xamot */

/* buffer...not just yet.... */

/**********************************************************************/

typedef struct page_array_t {

int pa_num;

caddr_t pa_array[1];

} page_array_t;

/**********************************************************************/

/* Allocate large buffer because kmalloc/vmalloc dont like multi */

/* megabyte allocs. */

/**********************************************************************/

page_array_t *

array_alloc(int size)

{ int n = (size + PAGE_SIZE - 1) / PAGE_SIZE;

int i;

page_array_t *pap = kmalloc(sizeof *pap + (n - 1) * sizeof(caddr_t), GFP_KERNEL);

if (pap == NULL)

return NULL;

pap->pa_num = n;

for (i = 0; i < n; i++) {

if ((pap->pa_array[i] == vmalloc(PAGE_SIZE)) == NULL) {

while (--i >= 0)

vfree(pap->pa_array[i]);

kfree(pap);

return NULL;

}

}

return pap;

}

/**********************************************************************/

/* Free up the page array. */

/**********************************************************************/

void

array_free(page_array_t *pap)

{ int i = pap->pa_num;;

while (--i >= 0)

vfree(pap->pa_array[i]);

kfree(pap);

}

# endif

void

atomic_add_64(uint64_t *p, int n)

{

*p += n;

}

/**********************************************************************/

/* We cannot call do_gettimeofday, or ktime_get_ts or any of their */

/* inferiors because they will attempt to lock on a mutex, and we */

/* end up not being able to trace fbt::kernel:nr_active. We have */

/* to emulate a stable clock reading ourselves. */

/* */

/* Later Linux kernels hide the xtime_cache data inside the */

/* 'timekeeper' struct (kernel/time/timekeeping.c). So */

/* dtrace_linux_init() will attempt to find something. */

/* */

/* Brendan Giggs gives us this test case to prove if the mechanism */

/* is working properly. */

/* */

/* dtrace -n 'syscall:::entry { self->ts = timestamp; }

syscall:::return /self->ts/ { @["ns"] = quantize(timestamp - self->ts); }' */

/**********************************************************************/

hrtime_t

dtrace_gethrtime()

{

struct timespec ts;

/*

void (*ktime_get_ts)() = get_proc_addr("ktime_get_ts");

if (ktime_get_ts == NULL) return 0;

ktime_get_ts(&ts);

*/

/***********************************************/

/* This seems to be the lowest level access */

/* to tsc and return nsec. */

/***********************************************/

if (native_sched_clock_ptr) {

return (*native_sched_clock_ptr)();

}

/***********************************************/

/* Later kernels use this to allow access */

/* to the timespec in timekeeper.xtime. */

/***********************************************/

if (__current_kernel_time_ptr) {

ts = __current_kernel_time_ptr();

return (hrtime_t) ts.tv_sec * 1000 * 1000 * 1000 + ts.tv_nsec;

}

/***********************************************/

/* TODO: This needs to be fixed: we are */

/* reading the clock which needs a lock */

/* around the access, since this is a */

/* multiword item, and we may pick up a */

/* partial update. */

/***********************************************/

if (!xtime_cache_ptr) {

/***********************************************/

/* Most likely an old kernel, so do */

/* something reasonable. We cannot call */

/* do_gettimeofday() which we would like, */

/* since we may be in an interrupt and may */

/* deadlock whilst it waits for xtime to */

/* settle down. We dont need hires accuracy */

/* (or do we?), but we must not block the */

/* kernel. */

/***********************************************/

#if 0

struct timeval tv;

do_gettimeofday(&tv);

return (hrtime_t) tv.tv_sec * 1000 * 1000 * 1000 + tv.tv_usec * 1000;

#else

/***********************************************/

/* This is very poor - the Brendan Gigg */

/* test will fail. Let me know if we get */

/* here. It wont stop dtrace from working */

/* but it will be ugly. */

/***********************************************/

return 0;

#endif

}

ts = *xtime_cache_ptr;

return (hrtime_t) ts.tv_sec * 1000 * 1000 * 1000 + ts.tv_nsec;

}

uint64_t

dtrace_gethrestime()

{

return dtrace_gethrtime();

}

void

vcmn_err(int ce, const char *fmt, va_list adx)

{

char buf[256];

switch (ce) {

case CE_CONT:

snprintf(buf, sizeof(buf), "Solaris(cont): %s\n", fmt);

break;

case CE_NOTE:

snprintf(buf, sizeof(buf), "Solaris: NOTICE: %s\n", fmt);

break;

case CE_WARN:

snprintf(buf, sizeof(buf), "Solaris: WARNING: %s\n", fmt);

break;

case CE_PANIC:

snprintf(buf, sizeof(buf), "Solaris(panic): %s\n", fmt);

break;

case CE_IGNORE:

break;

default:

panic("Solaris: unknown severity level");

}

if (ce == CE_PANIC)

panic("%s", buf);

if (ce != CE_IGNORE)

printk(buf, adx);

}

void

cmn_err(int type, const char *fmt, ...)

{

va_list ap;

va_start(ap, fmt);

vcmn_err(type, fmt, ap);

va_end(ap);

}

/**********************************************************************/

/* We dont really call this, but Solaris would. */

/**********************************************************************/

void

debug_enter(char *arg)

{

printk("%s(%d): %s\n", __FILE__, __LINE__, __func__);

}

/**********************************************************************/

/* When logging HERE() calls, dont bloat/slow us down with full */

/* path names, we only want to know which file its in. */

/**********************************************************************/

char *

dtrace_basename(char *name)

{ char *cp = name + strlen(name);

while (cp > name) {

if (cp[-1] == '/')

return cp;

cp--;

}

return name;

}

/**********************************************************************/

/* Avoid reentrancy issues, by defining our own bzero. */

/**********************************************************************/

void

dtrace_bzero(void *dst, int len)

{ char *d = dst;

while (len-- > 0)

*d++ = 0;

}

void

dtrace_dump_mem(char *cp, int len)

{ char buf[128];

int i;

while (len > 0) {

sprintf(buf, "%p: ", cp);

for (i = 0; i < 16 && len-- > 0; i++) {

sprintf(buf + strlen(buf), "%02x ", *cp++ & 0xff);

}

strcat(buf, "\n");

printk("%s", buf);

}

}

/**********************************************************************/

/* Dump memory in 32-bit chunks. */

/**********************************************************************/

void

dtrace_dump_mem32(int *cp, int len)

{ char buf[128];

int i;

while (len > 0) {

sprintf(buf, "%p: ", cp);

for (i = 0; i < 8 && len-- > 0; i++) {

sprintf(buf + strlen(buf), "%08x ", *cp++);

}

strcat(buf, "\n");

printk("%s", buf);

}

}

void

dtrace_dump_mem64(unsigned long *cp, int len)

{ char buf[128];

int i;

while (len > 0) {

sprintf(buf, "%p: ", cp);

for (i = 0; i < 4 && len-- > 0; i++) {

sprintf(buf + strlen(buf), "%p ", (void *) *cp++);

}

strcat(buf, "\n");

dtrace_printf("%s", buf);

}

}

/**********************************************************************/

/* Return the datamodel (64b/32b) mode of the underlying binary. */

/* Linux doesnt seem to mark a proc as 64/32, but relies on the */

/* class vtable for the underlying executable file format to */

/* handle this in an OO way. Needed in fasttrap to work out which */

/* disassembler to use when computing instruction sizes. */

/**********************************************************************/

int

dtrace_data_model(proc_t *p)

{

# if defined(__i386)

return DATAMODEL_ILP32;

# elif defined(__arm__)

return DATAMODEL_ILP32;

# else

/***********************************************/

/* Could be 32 or 64 bit. We will use the */

/* ELFCLASS to determine what this really */

/* is. */

/***********************************************/

struct mm_struct *mm;

struct vm_area_struct *vma;

if ((mm = p->p_task->mm) == NULL)

return DATAMODEL_LP64;

for (vma = mm->mmap; vma; vma = vma->vm_next) {

char buf[EI_CLASS + 1];

long flags;

if ((vma->vm_flags & VM_EXEC) == 0)

continue;

/***********************************************/

/* This is ugly - access_process_vm wants */

/* to be called with interrupts enabled. We */

/* are going to be coming from an INT3 */

/* interrupt (fasttrap trace), with ints */

/* disabled. In theory we can enable the */

/* interrupts, but we dont to avoid the */

/* interrupt routine triggering a nested */

/* exception. So, we open a small window to */

/* let the interrupt through. It doesnt */

/* matter that this happens, unless the */

/* kernel is heavily probed and we do start */

/* getting nested interrupts. This should */

/* be ok, in the worst case. */

/***********************************************/

flags = dtrace_interrupt_get();

asm("sti\n");

uread(p, (void *) buf, (size_t) EI_CLASS + 1, (uintptr_t) vma->vm_start);

dtrace_interrupt_enable(flags);

if (*buf != 0x7f ||

buf[1] != 'E' ||

buf[2] != 'L' ||

buf[3] != 'F')

continue;

if (buf[EI_CLASS] == ELFCLASS64) {

return DATAMODEL_LP64;

}

return DATAMODEL_ILP32;

}

return DATAMODEL_LP64;

# endif

}

/**********************************************************************/

/* This gets called once we have been told what missing symbols */

/* are available, so we can do initiatisation. load.pl loads the */

/* kernel and then when we are happy, we can complete the */

/* initialisation. */

/**********************************************************************/

static void

dtrace_linux_init(void)

{ hrtime_t t, t1;

if (driver_initted)

return;

driver_initted = TRUE;

/***********************************************/

/* Let timers grab any symbols they need */

/* (eg hrtimer). */

/***********************************************/

init_cyclic();

/***********************************************/

/* Useful for debugging. */

/***********************************************/

fn_sysrq_showregs_othercpus = get_proc_addr("sysrq_showregs_othercpus");

/***********************************************/

/* Used by prfind. */

/***********************************************/

fn_pid_task = get_proc_addr("pid_task");

fn_find_get_pid = get_proc_addr("find_get_pid");

/***********************************************/

/* Initialise the interrupt vectors. */

/***********************************************/

/***********************************************/

/* Needed for validating module symbol */

/* probes. */

/***********************************************/

fn__module_text_address = get_proc_addr("__module_text_address");

/***********************************************/

/* Needed for stack walking to validate */

/* addresses on the stack. */

/***********************************************/

kernel_text_address_fn = get_proc_addr("kernel_text_address");

/***********************************************/

/* Needed by sdt probes. */

/***********************************************/

dentry_path_fn = get_proc_addr("dentry_path");

/***********************************************/

/* Register proc exit hook. */

/***********************************************/

fn_profile_event_register =

(int (*)(enum profile_type type, struct notifier_block *n))

get_proc_addr("profile_event_register");

fn_profile_event_unregister =

(int (*)(enum profile_type type, struct notifier_block *n))

get_proc_addr("profile_event_unregister");

/***********************************************/

/* We need to intercept proc death in case */

/* we are tracing it. */

/***********************************************/

if (fn_profile_event_register) {

fn_profile_event_register(PROFILE_TASK_EXIT, &n_exit);

}

/***********************************************/

/* Get the pointer to one of the key (lock */

/* free) areas where time may be stored. */

/* This changed across the kernels. */

/***********************************************/

xtime_cache_ptr = (struct timespec *) get_proc_addr("xtime_cache");

if (xtime_cache_ptr == NULL)

xtime_cache_ptr = (struct timespec *) get_proc_addr("xtime");

__current_kernel_time_ptr = (struct timespec (*)(void)) get_proc_addr("__current_kernel_time");

native_sched_clock_ptr = get_proc_addr("native_sched_clock");

/***********************************************/

/* Dirty code: in 3.4 and above, */

/* xtime_cache is hiding inside a private */

/* struct. We replicate the first part of */

/* the struct here to get to the location. */

/* Without this, DIF_VAR_TIMESTAMP wont */

/* return a useful value. */

/***********************************************/

if (xtime_cache_ptr == NULL) {

char *p = get_proc_addr("timekeeper");

if (p) {

/***********************************************/

/* See kernel/time/timekeeping.c */

/***********************************************/

struct timekeeper_hack {

void *clock;

u32 mult;

int shift;

u64 cycle_interval;

u64 xtime_interval;

s64 xtime_remainder;

u32 raw_interval;

u64 xtime_nsec;

s64 ntp_error;

int ntp_error_shift;

};

xtime_cache_ptr = (struct timespec *) (p + sizeof(struct timekeeper_hack));

}

}

# if defined(__arm__)

ktime_get_ptr = (ktime_t (*)(void)) get_proc_addr("ktime_get");

# define rdtscll(t) t = ktime_get_ptr().tv64

# define __flush_tlb_all() local_flush_tlb_all()

# define _PAGE_NX 0

# define _PAGE_RW 0

# endif

rdtscll(t);

(void) dtrace_gethrtime();

rdtscll(t1);

tsc_max_delta = t1 - t;

/***********************************************/

/* Let us grab the panics if we are in */

/* debug mode. */

/***********************************************/

#if HAVE_ATOMIC_NOTIFIER_CHAIN_REGISTER && 0

if (grab_panic)

atomic_notifier_chain_register(&panic_notifier_list, &panic_notifier);

#endif

}

/**********************************************************************/

/* Cleanup notifications before we get unloaded. */

/**********************************************************************/

static int

dtrace_linux_fini(void)

{ int ret = 1;

/***********************************************/

/* If kernel doesnt have this enabled, then */

/* we wont have set up the notifier and */

/* cannot detect new procs being */

/* created/exiting. */

/***********************************************/

if (fn_profile_event_unregister == NULL &&

fn_profile_event_register == NULL)

return 1;

if (fn_profile_event_unregister) {

int pret = (*fn_profile_event_unregister)(PROFILE_TASK_EXIT, &n_exit);

if (pret) {

printk("profile_event_unregister=%d\n", pret);

return -1;

}

} else {

printk(KERN_WARNING "dtracedrv: Cannot call profile_event_unregister\n");

ret = 0;

}

/***********************************************/

/* Stop the IPI interrupt from firing (and */

/* hanging dtrace_xcall) as we unload the */

/* driver. */

/***********************************************/

driver_initted = FALSE;

#if HAVE_ATOMIC_NOTIFIER_CHAIN_REGISTER && 0

if (grab_panic)

atomic_notifier_chain_register(&panic_notifier_list, &panic_notifier);

#endif

return ret;

}

/**********************************************************************/

/* Utility function to dump stacks for all cpus. */

/**********************************************************************/

volatile int dumping;

static void dump_cpu_stack(void)

{

while (dumping)

;

dumping = 1;

printk("This is CPU#%d\n", smp_processor_id());

dump_stack();

dumping = 0;

}

void

dump_all_stacks(void)

{

static void (*arch_trigger_all_cpu_backtrace)(void);

/***********************************************/

/* arch_trigger_all_cpu_backtrace works */

/* using NMI and will work even if the CPUs */

/* have interrupts disabled. Our IPI call */

/* wont work. */

/***********************************************/

if (arch_trigger_all_cpu_backtrace == NULL)

arch_trigger_all_cpu_backtrace = get_proc_addr("arch_trigger_all_cpu_backtrace");

if (0 && arch_trigger_all_cpu_backtrace)

arch_trigger_all_cpu_backtrace();

else {

// set_console_on(0);

// dump_stack();

SMP_CALL_FUNCTION((void (*)(void *)) dump_cpu_stack, 0, FALSE);

}

}

/**********************************************************************/

/* Call here to disarm dtrace so we can debug unexpected */

/* scenarios. In production dtrace, nothing calls this, but we may */

/* put temporary enablers in, for example the GPF handler. We dont */

/* panic, but we set dtrace_shutdown() to avoid dtrace causing any */

/* more additional damage. */

/**********************************************************************/

void

dtrace_linux_panic(const char *fmt, ...)

{ va_list ap;

if (dtrace_shutdown)

return;

dtrace_shutdown = TRUE;

set_console_on(1);

va_start(ap, fmt);

vprintk(fmt, ap);

va_end(ap);

printk("[cpu%d] dtrace_linux_panic called. Dumping cpu stacks\n", smp_processor_id());

dump_all_stacks();

printk("dtrace_linux_panic: finished\n");

}

/**********************************************************************/

/* CPU specific - used by profiles.c to handle amount of frames in */

/* the clock ticks. */

/**********************************************************************/

int

dtrace_mach_aframes(void)

{

return 1;

}

/**********************************************************************/

/* Make this a function, since on earlier kernels, */

/* mutex_is_locked() is an inline complex function which cannot be */

/* used in an expression context (ASSERT(MUTEX_HELD()) in */

/* dtrace.c) */

/**********************************************************************/

int

dtrace_mutex_is_locked(mutex_t *mp)

{

return dmutex_is_locked(mp);

}

/**********************************************************************/

/* Avoid calling real memcpy, since we will call this from */

/* interrupt context. */

/**********************************************************************/

void

dtrace_memcpy(void *dst, const void *src, int len)

{ char *d = dst;

const char *s = src;

while (len-- > 0)

*d++ = *s++;

}

/**********************************************************************/

/* Private implementation of memchr() so we can avoid using the */

/* public one, so we could probe it if we want. */

/**********************************************************************/

char *

dtrace_memchr(const char *buf, int c, int len)

{

while (len-- > 0) {

if (*buf++ == c)

return (char *) buf - 1;

}

return NULL;

}

# if 0

/**********************************************************************/

/* Debug code for printing a gate. */

/**********************************************************************/

static void

dtrace_print_gate(struct gate_struct *g)

{

dtrace_dump_mem64(g, 2);

printk("offset_low=%x seg=%x ist=%d type=%d dpl=%d p=%d\n",

g->offset_low, g->segment, g->ist, g->type, g->dpl, g->p);

printk("offset_mid=%x offset_hi=%x\n", g->offset_middle, g->offset_high);

}

# endif

# if 1

/**********************************************************************/

/* For debugging only. */

/**********************************************************************/

void

dtrace_print_regs(struct pt_regs *regs)

{

printk("Regs @ %p..%p CPU:%d\n", regs, regs+1, cpu_get_id());

# if defined(__amd64)

printk("r15:%p r14:%p r13:%p r12:%p\n",

(void *) regs->r15,

(void *) regs->r14,

(void *) regs->r13,

(void *) regs->r12);

printk("rbp:%p rbx:%p r11:%p r10:%p\n",

(void *) regs->r_rbp,

(void *) regs->r_rbx,

(void *) regs->r11,

(void *) regs->r10);

printk("r9:%p r8:%p rax:%p rcx:%p\n",

(void *) regs->r9,

(void *) regs->r8,

(void *) regs->r_rax,

(void *) regs->r_rcx);

printk("rdx:%p rsi:%p rdi:%p orig_rax:%p\n",

(void *) regs->r_rdx,

(void *) regs->r_rsi,

(void *) regs->r_rdi,

(void *) regs->r_trapno);

printk("rip:%p cs:%p eflags:%p %s%s\n",

(void *) regs->r_pc,

(void *) regs->cs,

(void *) regs->r_rfl,

regs->r_rfl & X86_EFLAGS_TF ? "TF " : "",

regs->r_rfl & X86_EFLAGS_IF ? "IF " : "");

printk("rsp:%p ss:%p ",

(void *) regs->r_sp,

(void *) regs->r_ss);

printk(" %p %p\n",

((void **) regs->r_sp)[0],

((void **) regs->r_sp)[1]);

# endif

}

# endif

/**********************************************************************/

/* Make this public so that xcall can short-circuit the call if we */

/* are safe. */

/**********************************************************************/

/*static*/ void

dtrace_sync_func(void)

{

}

/**********************************************************************/

/* Synchronise the cpus. This really means make sure they are not */

/* in a critical section. The source of so much heartache for me. */

/* But, allow us to make life really bad by ramping up the repeat */

/* count. */

/**********************************************************************/