/*

* CDDL HEADER START

*

* The contents of this file are subject to the terms of the

* Common Development and Distribution License (the "License").

* You may not use this file except in compliance with the License.

*

* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

* or http://www.opensolaris.org/os/licensing.

* See the License for the specific language governing permissions

* and limitations under the License.

*

* When distributing Covered Code, include this CDDL HEADER in each

* file and include the License file at usr/src/OPENSOLARIS.LICENSE.

* If applicable, add the following below this CDDL HEADER, with the

* fields enclosed by brackets "[]" replaced with your own identifying

* information: Portions Copyright [yyyy] [name of copyright owner]

*

* CDDL HEADER END

*/

/*

* Copyright 2008 Sun Microsystems, Inc. All rights reserved.

* Use is subject to license terms.

*/

//#pragma ident "@(#)fasttrap.c 1.26 08/04/21 SMI"

# if linux

#include "dtrace_linux.h"

#include "proc_compat.h"

#include <sys/zone.h>

#include <linux/seq_file.h>

#include <linux/proc_fs.h>

#include <sys/fasttrap.h>

#include <sys/fasttrap_impl.h>

#include <sys/fasttrap_isa.h>

#include <sys/dtrace.h>

#include <sys/dtrace_impl.h>

#include <sys/rwlock.h>

#include "dtrace_proto.h"

#if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 16)

void sort(void *base, size_t num, size_t size,

int (*cmp)(const void *, const void *),

void (*swap)(void *, void *, int));

#else

# include <linux/sort.h>

#endif

#include <linux/module.h>

extern int nr_cpus;

// Temporary definitions so we can compile.

proc_t *

sprlock(int pid)

{

proc_t *p = prfind(pid);

//printk("sprlock: pid=%d\n", pid);

if (!p)

return NULL;

dmutex_enter(&p->p_lock);

return p;

}

void

sprlock_proc(proc_t *p)

{

dmutex_enter(&p->p_lock);

}

void

sprunlock(proc_t *p)

{

dmutex_exit(&p->p_lock);

}

# define RW_WRITER 2

#define SEXITING 0x00000002 /* process is exiting */

#define SFORKING 0x00000008 /* tells called functions that we're forking */

#define SVFORK 0x00040000 /* child of vfork that has not yet exec'd */

#define VREAD 00400

#define VWRITE 00200

# define priv_proc_cred_perm(cr, p, ptr, flags) 0

static void swap_func(void *p1, void *p2, int size)

{

while (size-- > 0) {

char t = *(char *) p1;

*(char *) p1 = *(char *) p2;

*(char *) p2 = t;

p1 = p1 + 1;

p2 = p2 + 1;

}

}

# define qsort(base, num, size, cmp) sort(base, num, size, cmp, swap_func)

MUTEX_DEFINE(pidlock);

void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *);

void (*dtrace_fasttrap_exec_ptr)(proc_t *);

void (*dtrace_fasttrap_exit_ptr)(proc_t *);

# endif

# if defined(sun)

#include <sys/atomic.h>

#include <sys/errno.h>

#include <sys/stat.h>

#include <sys/modctl.h>

#include <sys/conf.h>

#include <sys/systm.h>

#include <sys/ddi.h>

#include <sys/sunddi.h>

#include <sys/cpuvar.h>

#include <sys/kmem.h>

#include <sys/strsubr.h>

#include <sys/fasttrap.h>

#include <sys/fasttrap_impl.h>

#include <sys/fasttrap_isa.h>

#include <sys/dtrace.h>

#include <sys/dtrace_impl.h>

#include <sys/sysmacros.h>

#include <sys/frame.h>

#include <sys/stack.h>

#include <sys/proc.h>

#include <sys/priv.h>

#include <sys/policy.h>

#include <sys/ontrap.h>

#include <sys/vmsystm.h>

#include <sys/prsystm.h>

#include <vm/as.h>

#include <vm/seg.h>

#include <vm/seg_dev.h>

#include <vm/seg_vn.h>

#include <vm/seg_spt.h>

#include <vm/seg_kmem.h>

# endif

/*

* User-Land Trap-Based Tracing

* ----------------------------

*

* The fasttrap provider allows DTrace consumers to instrument any user-level

* instruction to gather data; this includes probes with semantic

* signifigance like entry and return as well as simple offsets into the

* function. While the specific techniques used are very ISA specific, the

* methodology is generalizable to any architecture.

*

*

* The General Methodology

* -----------------------

*

* With the primary goal of tracing every user-land instruction and the

* limitation that we can't trust user space so don't want to rely on much

* information there, we begin by replacing the instructions we want to trace

* with trap instructions. Each instruction we overwrite is saved into a hash

* table keyed by process ID and pc address. When we enter the kernel due to

* this trap instruction, we need the effects of the replaced instruction to

* appear to have occurred before we proceed with the user thread's

* execution.

*

* Each user level thread is represented by a ulwp_t structure which is

* always easily accessible through a register. The most basic way to produce

* the effects of the instruction we replaced is to copy that instruction out

* to a bit of scratch space reserved in the user thread's ulwp_t structure

* (a sort of kernel-private thread local storage), set the PC to that

* scratch space and single step. When we reenter the kernel after single

* stepping the instruction we must then adjust the PC to point to what would

* normally be the next instruction. Of course, special care must be taken

* for branches and jumps, but these represent such a small fraction of any

* instruction set that writing the code to emulate these in the kernel is

* not too difficult.

*

* Return probes may require several tracepoints to trace every return site,

* and, conversely, each tracepoint may activate several probes (the entry

* and offset 0 probes, for example). To solve this muliplexing problem,

* tracepoints contain lists of probes to activate and probes contain lists

* of tracepoints to enable. If a probe is activated, it adds its ID to

* existing tracepoints or creates new ones as necessary.

*

* Most probes are activated _before_ the instruction is executed, but return

* probes are activated _after_ the effects of the last instruction of the

* function are visible. Return probes must be fired _after_ we have

* single-stepped the instruction whereas all other probes are fired

* beforehand.

*

*

* Lock Ordering

* -------------

*

* The lock ordering below -- both internally and with respect to the DTrace

* framework -- is a little tricky and bears some explanation. Each provider

* has a lock (ftp_mtx) that protects its members including reference counts

* for enabled probes (ftp_rcount), consumers actively creating probes

* (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider

* from being freed. A provider is looked up by taking the bucket lock for the

* provider hash table, and is returned with its lock held. The provider lock

* may be taken in functions invoked by the DTrace framework, but may not be

* held while calling functions in the DTrace framework.

*

* To ensure consistency over multiple calls to the DTrace framework, the

* creation lock (ftp_cmtx) should be held. Naturally, the creation lock may

* not be taken when holding the provider lock as that would create a cyclic

* lock ordering. In situations where one would naturally take the provider

* lock and then the creation lock, we instead up a reference count to prevent

* the provider from disappearing, drop the provider lock, and acquire the

* creation lock.

*

* Briefly:

* bucket lock before provider lock

* DTrace before provider lock

* creation lock before DTrace

* never hold the provider lock and creation lock simultaneously

*/

# if defined(sun)

static dev_info_t *fasttrap_devi;

# endif

static dtrace_meta_provider_id_t fasttrap_meta_id;

static timeout_id_t fasttrap_timeout;

static MUTEX_DEFINE(fasttrap_cleanup_mtx);

static uint_t fasttrap_cleanup_work;

/*

* Generation count on modifications to the global tracepoint lookup table.

*/

static volatile uint64_t fasttrap_mod_gen;

/*

* When the fasttrap provider is loaded, fasttrap_max is set to either

* FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the

* fasttrap.conf file. Each time a probe is created, fasttrap_total is

* incremented by the number of tracepoints that may be associated with that

* probe; fasttrap_total is capped at fasttrap_max.

*/

#define FASTTRAP_MAX_DEFAULT 250000

# if defined(sun)

static uint32_t fasttrap_max;

# else

uint32_t fasttrap_max;

module_param(fasttrap_max, int, 0);

# endif

static uint32_t fasttrap_total;

#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000

#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100

#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100

#define FASTTRAP_PID_NAME "pid"

fasttrap_hash_t fasttrap_tpoints;

static fasttrap_hash_t fasttrap_provs;

static fasttrap_hash_t fasttrap_procs;

static uint64_t fasttrap_pid_count; /* pid ref count */

static MUTEX_DEFINE(fasttrap_count_mtx); /* lock on ref count */

#define FASTTRAP_ENABLE_FAIL 1

#define FASTTRAP_ENABLE_PARTIAL 2

static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);

static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);

static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,

const dtrace_pattr_t *);

static void fasttrap_provider_retire(pid_t, const char *, int);

static void fasttrap_provider_free(fasttrap_provider_t *);

static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);

static void fasttrap_proc_release(fasttrap_proc_t *);

#define FASTTRAP_PROVS_INDEX(pid, name) \

((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)

#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)

static int

fasttrap_highbit(ulong_t i)

{

int h = 1;

if (i == 0)

return (0);

#ifdef _LP64

if (i & 0xffffffff00000000ul) {

h += 32; i >>= 32;

}

#endif

if (i & 0xffff0000) {

h += 16; i >>= 16;

}

if (i & 0xff00) {

h += 8; i >>= 8;

}

if (i & 0xf0) {

h += 4; i >>= 4;

}

if (i & 0xc) {

h += 2; i >>= 2;

}

if (i & 0x2) {

h += 1;

}

return (h);

}

static uint_t

fasttrap_hash_str(const char *p)

{

unsigned int g;

uint_t hval = 0;

while (*p) {

hval = (hval << 4) + *p++;

if ((g = (hval & 0xf0000000)) != 0)

hval ^= g >> 24;

hval &= ~g;

}

return (hval);

}

void

fasttrap_sigtrap(proc_t *p, proc_t *t, uintptr_t pc)

{

# if defined(sun)

sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);

HERE();

sqp->sq_info.si_signo = SIGTRAP;

sqp->sq_info.si_code = TRAP_DTRACE;

sqp->sq_info.si_addr = (caddr_t)pc;

dmutex_enter(&p->p_lock);

sigaddqa(p, t, sqp);

dmutex_exit(&p->p_lock);

if (t != NULL)

aston(t);

# else

TODO();

send_sig(SIGTRAP, p->p_task, 0);

# endif

}

/*

* This function ensures that no threads are actively using the memory

* associated with probes that were formerly live.

*/

static void

fasttrap_mod_barrier(uint64_t gen)

{

int i;

if (gen < fasttrap_mod_gen)

return;

fasttrap_mod_gen++;

/***********************************************/

/* Need to fix this - we need to ensure */

/* this is really a memory barrier. */

/***********************************************/

HERE();

/***********************************************/

/* Be careful if we have new cpus coming */

/* online after we got loaded since the */

/* cpu_core[] array wont have the right */

/* number of entries and we may panic the */

/* kernel. */

/***********************************************/

for (i = 0; i < num_online_cpus(); i++) {

dmutex_enter(&cpu_core[i].cpuc_pid_lock);

dmutex_exit(&cpu_core[i].cpuc_pid_lock);

HERE();

}

}

/*

* This is the timeout's callback for cleaning up the providers and their

* probes.

*/

/*ARGSUSED*/

static void

fasttrap_pid_cleanup_cb(void *data)

{

fasttrap_provider_t **fpp, *fp;

fasttrap_bucket_t *bucket;

dtrace_provider_id_t provid;

int i, later = 0;

static volatile int in = 0;

ASSERT(in == 0);

in = 1;

HERE();

dmutex_enter(&fasttrap_cleanup_mtx);

while (fasttrap_cleanup_work) {

fasttrap_cleanup_work = 0;

dmutex_exit(&fasttrap_cleanup_mtx);

later = 0;

/*

* Iterate over all the providers trying to remove the marked

* ones. If a provider is marked but not retired, we just

* have to take a crack at removing it -- it's no big deal if

* we can't.

*/

for (i = 0; i < fasttrap_provs.fth_nent; i++) {

bucket = &fasttrap_provs.fth_table[i];

dmutex_enter(&bucket->ftb_mtx);

fpp = (fasttrap_provider_t **)&bucket->ftb_data;

while ((fp = *fpp) != NULL) {

if (!fp->ftp_marked) {

fpp = &fp->ftp_next;

continue;

}

dmutex_enter(&fp->ftp_mtx);

/*

* If this provider has consumers actively

* creating probes (ftp_ccount) or is a USDT

* provider (ftp_mcount), we can't unregister

* or even condense.

*/

if (fp->ftp_ccount != 0 ||

fp->ftp_mcount != 0) {

dmutex_exit(&fp->ftp_mtx);

fp->ftp_marked = 0;

continue;

}

if (!fp->ftp_retired || fp->ftp_rcount != 0)

fp->ftp_marked = 0;

dmutex_exit(&fp->ftp_mtx);

/*

* If we successfully unregister this

* provider we can remove it from the hash

* chain and free the memory. If our attempt

* to unregister fails and this is a retired

* provider, increment our flag to try again

* pretty soon. If we've consumed more than

* half of our total permitted number of

* probes call dtrace_condense() to try to

* clean out the unenabled probes.

*/

provid = fp->ftp_provid;

if (dtrace_unregister(provid) != 0) {

if (fasttrap_total > fasttrap_max / 2)

(void) dtrace_condense(provid);

later += fp->ftp_marked;

fpp = &fp->ftp_next;

} else {

*fpp = fp->ftp_next;

fasttrap_provider_free(fp);

}

}

dmutex_exit(&bucket->ftb_mtx);

}

dmutex_enter(&fasttrap_cleanup_mtx);

}

ASSERT(fasttrap_timeout != 0);

/*

* If we were unable to remove a retired provider, try again after

* a second. This situation can occur in certain circumstances where

* providers cannot be unregistered even though they have no probes

* enabled because of an execution of dtrace -l or something similar.

* If the timeout has been disabled (set to 1 because we're trying

* to detach), we set fasttrap_cleanup_work to ensure that we'll

* get a chance to do that work if and when the timeout is reenabled

* (if detach fails).

*/

if (later > 0 && fasttrap_timeout != (timeout_id_t)1)

fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz);

else if (later > 0)

fasttrap_cleanup_work = 1;

else

fasttrap_timeout = 0;

dmutex_exit(&fasttrap_cleanup_mtx);

in = 0;

}

/*

* Activates the asynchronous cleanup mechanism.

*/

static void

fasttrap_pid_cleanup(void)

{

HERE();

dmutex_enter(&fasttrap_cleanup_mtx);

fasttrap_cleanup_work = 1;

if (fasttrap_timeout == 0)

fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);

dmutex_exit(&fasttrap_cleanup_mtx);

}

/*

* This is called from cfork() via dtrace_fasttrap_fork(). The child

* process's address space is (roughly) a copy of the parent process's so

* we have to remove all the instrumentation we had previously enabled in the

* parent.

*/

static void

fasttrap_fork(proc_t *p, proc_t *cp)

{

pid_t ppid = p->p_pid;

int i;

printk("in fasttrap_fork\n");

ASSERT(curproc == p);

ASSERT(p->p_proc_flag & P_PR_LOCK);

ASSERT(p->p_dtrace_count > 0);

ASSERT(cp->p_dtrace_count == 0);

/*

* This would be simpler and faster if we maintained per-process

* hash tables of enabled tracepoints. It could, however, potentially

* slow down execution of a tracepoint since we'd need to go

* through two levels of indirection. In the future, we should

* consider either maintaining per-process ancillary lists of

* enabled tracepoints or hanging a pointer to a per-process hash

* table of enabled tracepoints off the proc structure.

*/

/*

* We don't have to worry about the child process disappearing

* because we're in fork().

*/

dmutex_enter(&cp->p_lock);

sprlock_proc(cp);

dmutex_exit(&cp->p_lock);

/*

* Iterate over every tracepoint looking for ones that belong to the

* parent process, and remove each from the child process.

*/

for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {

fasttrap_tracepoint_t *tp;

fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];

dmutex_enter(&bucket->ftb_mtx);

for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {

if (tp->ftt_pid == ppid &&

tp->ftt_proc->ftpc_acount != 0) {

int ret = fasttrap_tracepoint_remove(cp, tp);

ret = ret; // avoid compiler warning

ASSERT(ret == 0);

/*

* The count of active providers can only be

* decremented (i.e. to zero) during exec,

* exit, and removal of a meta provider so it

* should be impossible to drop the count

* mid-fork.

*/

ASSERT(tp->ftt_proc->ftpc_acount != 0);

}

}

dmutex_exit(&bucket->ftb_mtx);

}

dmutex_enter(&cp->p_lock);

sprunlock(cp);

}

/*

* This is called from proc_exit() or from exec_common() if p_dtrace_probes

* is set on the proc structure to indicate that there is a pid provider

* associated with this process.

*/

static void

fasttrap_exec_exit(proc_t *p)

{

ASSERT(p == curproc);

ASSERT(MUTEX_HELD(&p->p_lock));

dmutex_exit(&p->p_lock);

//printk("fasttrap_exec_exit: pid=%d\n", p->p_pid);

/*

* We clean up the pid provider for this process here; user-land

* static probes are handled by the meta-provider remove entry point.

*/

fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);

dmutex_enter(&p->p_lock);

}

/*ARGSUSED*/

static void

fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)

{

/*

* There are no "default" pid probes.

*/

}

static int

fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)

{

fasttrap_tracepoint_t *tp, *new_tp = NULL;

fasttrap_bucket_t *bucket;

fasttrap_id_t *id;

pid_t pid;

uintptr_t pc;

ASSERT(index < probe->ftp_ntps);

HERE();

pid = probe->ftp_pid;

pc = probe->ftp_tps[index].fit_tp->ftt_pc;

id = &probe->ftp_tps[index].fit_id;

ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);

ASSERT(!(p->p_flag & SVFORK));

HERE();

/*

* Before we make any modifications, make sure we've imposed a barrier

* on the generation in which this probe was last modified.

*/

fasttrap_mod_barrier(probe->ftp_gen);

bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];

//printk("bucket=%p pid=%d pc=%lx\n", bucket, pid, pc);

HERE();

/*

* If the tracepoint has already been enabled, just add our id to the

* list of interested probes. This may be our second time through

* this path in which case we'll have constructed the tracepoint we'd

* like to install. If we can't find a match, and have an allocated

* tracepoint ready to go, enable that one now.

*

* A tracepoint whose process is defunct is also considered defunct.

* defunct.

*/

again:

dmutex_enter(&bucket->ftb_mtx);

for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {

/*

* Note that it's safe to access the active count on the

* associated proc structure because we know that at least one

* provider (this one) will still be around throughout this

* operation.

*/

if (tp->ftt_pid != pid || tp->ftt_pc != pc ||

tp->ftt_proc->ftpc_acount == 0)

continue;

/*

* Now that we've found a matching tracepoint, it would be

* a decent idea to confirm that the tracepoint is still

* enabled and the trap instruction hasn't been overwritten.

* Since this is a little hairy, we'll punt for now.

*/

/*

* This can't be the first interested probe. We don't have

* to worry about another thread being in the midst of

* deleting this tracepoint (which would be the only valid

* reason for a tracepoint to have no interested probes)

* since we're holding P_PR_LOCK for this process.

*/

ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);

HERE();

switch (id->fti_ptype) {

case DTFTP_ENTRY:

case DTFTP_OFFSETS:

case DTFTP_IS_ENABLED:

id->fti_next = tp->ftt_ids;

membar_producer();

tp->ftt_ids = id;

membar_producer();

break;

case DTFTP_RETURN:

case DTFTP_POST_OFFSETS:

id->fti_next = tp->ftt_retids;

membar_producer();

tp->ftt_retids = id;

membar_producer();

break;

default:

ASSERT(0);

}

HERE();

dmutex_exit(&bucket->ftb_mtx);

HERE();

if (new_tp != NULL) {

new_tp->ftt_ids = NULL;

new_tp->ftt_retids = NULL;

}

return (0);

}

/*

* If we have a good tracepoint ready to go, install it now while

* we have the lock held and no one can screw with us.

*/

HERE();

if (new_tp != NULL) {

int rc = 0;

new_tp->ftt_next = bucket->ftb_data;

membar_producer();

bucket->ftb_data = new_tp;

membar_producer();

dmutex_exit(&bucket->ftb_mtx);

/*

* Activate the tracepoint in the ISA-specific manner.

* If this fails, we need to report the failure, but

* indicate that this tracepoint must still be disabled

* by calling fasttrap_tracepoint_disable().

*/

HERE();

if (fasttrap_tracepoint_install(p, new_tp) != 0) {

rc = FASTTRAP_ENABLE_PARTIAL;

printk("rc=%d FASTTRAP_ENABLE_PARTIAL=%d pc=%p\n", rc, FASTTRAP_ENABLE_PARTIAL, (void *) tp->ftt_pc);

}

HERE();

/*

* Increment the count of the number of tracepoints active in

* the victim process.

*/

ASSERT(p->p_proc_flag & P_PR_LOCK);

p->p_dtrace_count++;

return (rc);

}

HERE();

dmutex_exit(&bucket->ftb_mtx);

/*

* Initialize the tracepoint that's been preallocated with the probe.

*/

new_tp = probe->ftp_tps[index].fit_tp;

HERE();

ASSERT(new_tp->ftt_pid == pid);

ASSERT(new_tp->ftt_pc == pc);

ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);

ASSERT(new_tp->ftt_ids == NULL);

ASSERT(new_tp->ftt_retids == NULL);

switch (id->fti_ptype) {

case DTFTP_ENTRY:

case DTFTP_OFFSETS:

case DTFTP_IS_ENABLED:

id->fti_next = NULL;

new_tp->ftt_ids = id;

break;

case DTFTP_RETURN:

case DTFTP_POST_OFFSETS:

id->fti_next = NULL;

new_tp->ftt_retids = id;

break;

default:

ASSERT(0);

}

/*

* If the ISA-dependent initialization goes to plan, go back to the

* beginning and try to install this freshly made tracepoint.

*/

HERE();

if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)

goto again;

HERE();

new_tp->ftt_ids = NULL;

new_tp->ftt_retids = NULL;

return (FASTTRAP_ENABLE_FAIL);

}

static void

fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)

{

fasttrap_bucket_t *bucket;

fasttrap_provider_t *provider = probe->ftp_prov;

fasttrap_tracepoint_t **pp, *tp;

fasttrap_id_t *id, **idp = NULL;

pid_t pid;

uintptr_t pc;

ASSERT(index < probe->ftp_ntps);

pid = probe->ftp_pid;

pc = probe->ftp_tps[index].fit_tp->ftt_pc;

id = &probe->ftp_tps[index].fit_id;

ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);

/*

* Find the tracepoint and make sure that our id is one of the

* ones registered with it.

*/

bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];

dmutex_enter(&bucket->ftb_mtx);

for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {

if (tp->ftt_pid == pid && tp->ftt_pc == pc &&

tp->ftt_proc == provider->ftp_proc)

break;

}

/*

* If we somehow lost this tracepoint, we're in a world of hurt.

*/

ASSERT(tp != NULL);

switch (id->fti_ptype) {

case DTFTP_ENTRY:

case DTFTP_OFFSETS:

case DTFTP_IS_ENABLED:

ASSERT(tp->ftt_ids != NULL);

idp = &tp->ftt_ids;

break;

case DTFTP_RETURN:

case DTFTP_POST_OFFSETS:

ASSERT(tp->ftt_retids != NULL);

idp = &tp->ftt_retids;

break;

default:

ASSERT(0);

}

while ((*idp)->fti_probe != probe) {

idp = &(*idp)->fti_next;

ASSERT(*idp != NULL);

}

id = *idp;

*idp = id->fti_next;

membar_producer();

ASSERT(id->fti_probe == probe);

/*

* If there are other registered enablings of this tracepoint, we're

* all done, but if this was the last probe assocated with this

* this tracepoint, we need to remove and free it.

*/

if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {

/*

* If the current probe's tracepoint is in use, swap it

* for an unused tracepoint.

*/

if (tp == probe->ftp_tps[index].fit_tp) {

fasttrap_probe_t *tmp_probe;

fasttrap_tracepoint_t **tmp_tp;

uint_t tmp_index;

if (tp->ftt_ids != NULL) {

tmp_probe = tp->ftt_ids->fti_probe;

/* LINTED - alignment */

tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);

tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;

} else {

tmp_probe = tp->ftt_retids->fti_probe;

/* LINTED - alignment */

tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);

tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;

}

ASSERT(*tmp_tp != NULL);

ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);

ASSERT((*tmp_tp)->ftt_ids == NULL);

ASSERT((*tmp_tp)->ftt_retids == NULL);

probe->ftp_tps[index].fit_tp = *tmp_tp;

*tmp_tp = tp;

}

dmutex_exit(&bucket->ftb_mtx);

/*

* Tag the modified probe with the generation in which it was

* changed.

*/

probe->ftp_gen = fasttrap_mod_gen;

return;

}

dmutex_exit(&bucket->ftb_mtx);

/*

* We can't safely remove the tracepoint from the set of active

* tracepoints until we've actually removed the fasttrap instruction

* from the process's text. We can, however, operate on this

* tracepoint secure in the knowledge that no other thread is going to

* be looking at it since we hold P_PR_LOCK on the process if it's

* live or we hold the provider lock on the process if it's dead and

* gone.

*/

/*

* We only need to remove the actual instruction if we're looking

* at an existing process

*/

if (p != NULL) {

/*

* If we fail to restore the instruction we need to kill

* this process since it's in a completely unrecoverable

* state.

*/

if (fasttrap_tracepoint_remove(p, tp) != 0)

fasttrap_sigtrap(p, NULL, pc);

/*

* Decrement the count of the number of tracepoints active

* in the victim process.

*/

ASSERT(p->p_proc_flag & P_PR_LOCK);

p->p_dtrace_count--;

}

/*

* Remove the probe from the hash table of active tracepoints.

*/

dmutex_enter(&bucket->ftb_mtx);

pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;

ASSERT(*pp != NULL);

while (*pp != tp) {

pp = &(*pp)->ftt_next;

ASSERT(*pp != NULL);

}

*pp = tp->ftt_next;

membar_producer();

dmutex_exit(&bucket->ftb_mtx);

/*

* Tag the modified probe with the generation in which it was changed.

*/

probe->ftp_gen = fasttrap_mod_gen;

}

static void

fasttrap_enable_callbacks(void)

{

/*

* We don't have to play the rw lock game here because we're

* providing something rather than taking something away --

* we can be sure that no threads have tried to follow this

* function pointer yet.

*/

HERE();

//printk("fasttrap_pid_count=%ld\n", fasttrap_pid_count);

dmutex_enter(&fasttrap_count_mtx);

if (fasttrap_pid_count == 0) {

ASSERT(dtrace_pid_probe_ptr == NULL);

ASSERT(dtrace_return_probe_ptr == NULL);

dtrace_pid_probe_ptr = &fasttrap_pid_probe;

dtrace_return_probe_ptr = &fasttrap_return_probe;

}

ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);

ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);

fasttrap_pid_count++;

dmutex_exit(&fasttrap_count_mtx);

}

static void

fasttrap_disable_callbacks(void)

{

ASSERT(MUTEX_HELD(&cpu_lock));

HERE();

dmutex_enter(&fasttrap_count_mtx);

ASSERT(fasttrap_pid_count > 0);

fasttrap_pid_count--;

if (fasttrap_pid_count == 0) {