/*

* CDDL HEADER START

*

* The contents of this file are subject to the terms of the

* Common Development and Distribution License (the "License").

* You may not use this file except in compliance with the License.

*

* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

* or http://www.opensolaris.org/os/licensing.

* See the License for the specific language governing permissions

* and limitations under the License.

*

* When distributing Covered Code, include this CDDL HEADER in each

* file and include the License file at usr/src/OPENSOLARIS.LICENSE.

* If applicable, add the following below this CDDL HEADER, with the

* fields enclosed by brackets "[]" replaced with your own identifying

* information: Portions Copyright [yyyy] [name of copyright owner]

*

* CDDL HEADER END

*/

/*

* Copyright 2008 Sun Microsystems, Inc. All rights reserved.

* Use is subject to license terms.

*/

//#pragma ident "@(#)dtrace.c 1.64 08/05/26 SMI"

/*

* DTrace - Dynamic Tracing for Solaris

*

* This is the implementation of the Solaris Dynamic Tracing framework

* (DTrace). The user-visible interface to DTrace is described at length in

* the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace

* library, the in-kernel DTrace framework, and the DTrace providers are

* described in the block comments in the <sys/dtrace.h> header file. The

* internal architecture of DTrace is described in the block comments in the

* <sys/dtrace_impl.h> header file. The comments contained within the DTrace

* implementation very much assume mastery of all of these sources; if one has

* an unanswered question about the implementation, one should consult them

* first.

*

* The functions here are ordered roughly as follows:

*

* - Probe context functions

* - Probe hashing functions

* - Non-probe context utility functions

* - Matching functions

* - Provider-to-Framework API functions

* - Probe management functions

* - DIF object functions

* - Format functions

* - Predicate functions

* - ECB functions

* - Buffer functions

* - Enabling functions

* - DOF functions

* - Anonymous enabling functions

* - Consumer state functions

* - Helper functions

* - Hook functions

* - Driver cookbook functions

*

* Each group of functions begins with a block comment labelled the "DTrace

* [Group] Functions", allowing one to find each block by searching forward

* on capital-f functions.

*/

# if linux

#include "dtrace_linux.h"

#include <sys/dtrace_impl.h>

#include "dtrace_proto.h"

#if 0

/* Comment this out - it causes build failure on 3.6.9 kernel. Not sure why it

was added in the first place. */

#include <linux/swap.h> /* want totalram_pages */

#endif

#include <linux/delay.h>

#include <linux/slab.h>

#if defined(HAVE_LINUX_FDTABLE_H)

# include <linux/fdtable.h>

#endif

# undef ASSERT

# define ASSERT(x) { \

if (!(x)) { \

static int once = TRUE; \

dtrace_linux_panic("%s:%s:%d: assertion failure %s\n", __FILE__, __func__, __LINE__, #x); \

if (once) { \

once = FALSE; \

dump_stack(); \

} \

}}

# define KERNELBASE 0

# endif

# if defined(sun)

#include <sys/errno.h>

#include <sys/stat.h>

#include <sys/modctl.h>

#include <sys/conf.h>

#include <sys/systm.h>

#include <sys/ddi.h>

#include <sys/sunddi.h>

#include <sys/cpuvar.h>

#include <sys/kmem.h>

#include <sys/strsubr.h>

#include <sys/sysmacros.h>

# endif

#include <sys/atomic.h>

#include <sys/cmn_err.h>

#include <sys/mutex_impl.h>

#include <sys/rwlock_impl.h>

#include <sys/ctf_api.h>

# if defined(sun)

#include <sys/panic.h>

#include <sys/priv_impl.h>

#include <sys/policy.h>

#include <sys/cred_impl.h>

#include <sys/procfs_isa.h>

#include <sys/taskq.h>

#include <sys/mkdev.h>

#include <sys/kdi.h>

#include <sys/zone.h>

#include <sys/socket.h>

# endif

#include <netinet/in.h>

#include <linux/inet.h>

#include <sys/privregs.h>

# if linux

#include <taskq.h>

# undef NULL

# define NULL 0

# endif

# if 0

# define RETURN(err) do { printk("%s(%d): RETURN ERROR %s\n", __func__, __LINE__, #err); return err;} while (0)

# else

# define RETURN(err) return err

# endif

/*

* DTrace Tunable Variables

*

* The following variables may be tuned by adding a line to /etc/system that

* includes both the name of the DTrace module ("dtrace") and the name of the

* variable. For example:

*

* set dtrace:dtrace_destructive_disallow = 1

*

* In general, the only variables that one should be tuning this way are those

* that affect system-wide DTrace behavior, and for which the default behavior

* is undesirable. Most of these variables are tunable on a per-consumer

* basis using DTrace options, and need not be tuned on a system-wide basis.

* When tuning these variables, avoid pathological values; while some attempt

* is made to verify the integrity of these variables, they are not considered

* part of the supported interface to DTrace, and they are therefore not

* checked comprehensively. Further, these variables should not be tuned

* dynamically via "mdb -kw" or other means; they should only be tuned via

* /etc/system.

*/

int dtrace_destructive_disallow = 0;

dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);

size_t dtrace_difo_maxsize = (256 * 1024);

dtrace_optval_t dtrace_dof_maxsize = (256 * 1024);

size_t dtrace_global_maxsize = (16 * 1024);

size_t dtrace_actions_max = (16 * 1024);

size_t dtrace_retain_max = 1024;

dtrace_optval_t dtrace_helper_actions_max = 32;

dtrace_optval_t dtrace_helper_providers_max = 32;

dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);

size_t dtrace_strsize_default = 256;

dtrace_optval_t dtrace_cleanrate_default = 9900990; /* 101 hz */

dtrace_optval_t dtrace_cleanrate_min = 200000; /* 5000 hz */

dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC; /* 1/minute */

dtrace_optval_t dtrace_aggrate_default = NANOSEC; /* 1 hz */

dtrace_optval_t dtrace_statusrate_default = NANOSEC; /* 1 hz */

dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC; /* 6/minute */

dtrace_optval_t dtrace_switchrate_default = NANOSEC; /* 1 hz */

dtrace_optval_t dtrace_nspec_default = 1;

dtrace_optval_t dtrace_specsize_default = 32 * 1024;

dtrace_optval_t dtrace_stackframes_default = 20;

dtrace_optval_t dtrace_ustackframes_default = 20;

dtrace_optval_t dtrace_jstackframes_default = 50;

dtrace_optval_t dtrace_jstackstrsize_default = 512;

int dtrace_msgdsize_max = 128;

hrtime_t dtrace_chill_max = 500 * (NANOSEC / MILLISEC); /* 500 ms */

hrtime_t dtrace_chill_interval = NANOSEC; /* 1000 ms */

int dtrace_devdepth_max = 32;

int dtrace_err_verbose;

hrtime_t dtrace_deadman_interval = NANOSEC;

hrtime_t dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;

hrtime_t dtrace_deadman_user = (hrtime_t)30 * NANOSEC;

hrtime_t dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;

# if linux

static void dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val);

static int dtrace_module_loaded(struct notifier_block *nb, unsigned long val, void *data);

static void dtrace_module_unloaded(struct modctl *modctl);

static struct notifier_block n_module_load = {

.notifier_call = dtrace_module_loaded,

};

/**********************************************************************/

/* Hold on to the ecbs during a large teardown, so we can delete */

/* in one go. */

/**********************************************************************/

# define FAST_PROBE_TEARDOWN 0

static unsigned long long cnt_free1;

static volatile int lock_teardown = -1;

# endif

/*

* DTrace External Variables

*

* As dtrace(7D) is a kernel module, any DTrace variables are obviously

* available to DTrace consumers via the backtick (`) syntax. One of these,

* dtrace_zero, is made deliberately so: it is provided as a source of

* well-known, zero-filled memory. While this variable is not documented,

* it is used by some translators as an implementation detail.

*/

const char dtrace_zero[256] = { 0 }; /* zero-filled memory */

/*

* DTrace Internal Variables

*/

static dev_info_t *dtrace_devi; /* device info */

static vmem_t *dtrace_arena; /* probe ID arena */

static vmem_t *dtrace_minor; /* minor number arena */

static taskq_t *dtrace_taskq; /* task queue */

static dtrace_probe_t **dtrace_probes; /* array of all probes */

static int dtrace_nprobes; /* number of probes */

static dtrace_provider_t *dtrace_provider; /* provider list */

static dtrace_meta_t *dtrace_meta_pid; /* user-land meta provider */

static int dtrace_opens; /* number of opens */

static int dtrace_helpers; /* number of helpers */

# if defined(sun)

static void *dtrace_softstate; /* softstate pointer */

# endif

static dtrace_hash_t *dtrace_bymod; /* probes hashed by module */

static dtrace_hash_t *dtrace_byfunc; /* probes hashed by function */

static dtrace_hash_t *dtrace_byname; /* probes hashed by name */

static dtrace_toxrange_t *dtrace_toxrange; /* toxic range array */

static int dtrace_toxranges; /* number of toxic ranges */

static int dtrace_toxranges_max; /* size of toxic range array */

static dtrace_anon_t dtrace_anon; /* anonymous enabling */

static kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */

static uint64_t dtrace_vtime_references; /* number of vtimestamp refs */

static kthread_t *dtrace_panicked; /* panicking thread */

static dtrace_ecb_t *dtrace_ecb_create_cache; /* cached created ECB */

static dtrace_genid_t dtrace_probegen; /* current probe generation */

static dtrace_helpers_t *dtrace_deferred_pid; /* deferred helper list */

static dtrace_enabling_t *dtrace_retained; /* list of retained enablings */

static dtrace_genid_t dtrace_retained_gen; /* current retained enab gen */

static dtrace_dynvar_t dtrace_dynhash_sink; /* end of dynamic hash chains */

static int dtrace_dynvar_failclean; /* dynvars failed to clean */

/**********************************************************************/

/* We wrap dtrace_probe(), so we have an extra frame to discount. */

/**********************************************************************/

#if linux

#define PROBE_FRAMES 3

#else

#define PROBE_FRAMES 2

#endif

/*

* DTrace Locking

* DTrace is protected by three (relatively coarse-grained) locks:

*

* (1) dtrace_lock is required to manipulate essentially any DTrace state,

* including enabling state, probes, ECBs, consumer state, helper state,

* etc. Importantly, dtrace_lock is _not_ required when in probe context;

* probe context is lock-free -- synchronization is handled via the

* dtrace_sync() cross call mechanism.

*

* (2) dtrace_provider_lock is required when manipulating provider state, or

* when provider state must be held constant.

*

* (3) dtrace_meta_lock is required when manipulating meta provider state, or

* when meta provider state must be held constant.

*

* The lock ordering between these three locks is dtrace_meta_lock before

* dtrace_provider_lock before dtrace_lock. (In particular, there are

* several places where dtrace_provider_lock is held by the framework as it

* calls into the providers -- which then call back into the framework,

* grabbing dtrace_lock.)

*

* There are two other locks in the mix: mod_lock and cpu_lock. With respect

* to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical

* role as a coarse-grained lock; it is acquired before both of these locks.

* With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must

* be acquired _between_ dtrace_meta_lock and any other DTrace locks.

* mod_lock is similar with respect to dtrace_provider_lock in that it must be

* acquired _between_ dtrace_provider_lock and dtrace_lock.

*/

MUTEX_DEFINE(dtrace_lock); /* probe state lock */

MUTEX_DEFINE(dtrace_provider_lock); /* provider state lock */

MUTEX_DEFINE(dtrace_meta_lock); /* meta-provider state lock */

/*

* DTrace Provider Variables

*

* These are the variables relating to DTrace as a provider (that is, the

* provider of the BEGIN, END, and ERROR probes).

*/

static dtrace_pattr_t dtrace_provider_attr = {

{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },

{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },

{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },

{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },

{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },

};

static int

dtrace_enable_nullop(void)

{

return 0;

}

static void

dtrace_nullop(void)

{}

static dtrace_pops_t dtrace_provider_ops = {

(void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,

(void (*)(void *, struct modctl *))dtrace_nullop,

(int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop,

(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,

(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,

(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,

NULL,

NULL,

NULL,

(void (*)(void *, dtrace_id_t, void *))dtrace_nullop

};

static dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */

static dtrace_id_t dtrace_probeid_end; /* special END probe */

dtrace_id_t dtrace_probeid_error; /* special ERROR probe */

/*

* DTrace Helper Tracing Variables

*/

uint32_t dtrace_helptrace_next = 0;

uint32_t dtrace_helptrace_nlocals;

char *dtrace_helptrace_buffer;

int dtrace_helptrace_bufsize = 512 * 1024;

#ifdef DEBUG

int dtrace_helptrace_enabled = 1;

#else

int dtrace_helptrace_enabled = 0;

#endif

/*

* DTrace Error Hashing

*

* On DEBUG kernels, DTrace will track the errors that has seen in a hash

* table. This is very useful for checking coverage of tests that are

* expected to induce DIF or DOF processing errors, and may be useful for

* debugging problems in the DIF code generator or in DOF generation . The

* error hash may be examined with the ::dtrace_errhash MDB dcmd.

*/

#ifdef DEBUG

static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];

static const char *dtrace_errlast;

static kthread_t *dtrace_errthread;

static MUTEX_DEFINE(dtrace_errlock);

#endif

/*

* DTrace Macros and Constants

*

* These are various macros that are useful in various spots in the

* implementation, along with a few random constants that have no meaning

* outside of the implementation. There is no real structure to this cpp

* mishmash -- but is there ever?

*/

#define DTRACE_HASHSTR(hash, probe) \

dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))

#define DTRACE_HASHNEXT(hash, probe) \

(dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)

#define DTRACE_HASHPREV(hash, probe) \

(dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)

#define DTRACE_HASHEQ(hash, lhs, rhs) \

(strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \

*((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)

#define DTRACE_AGGHASHSIZE_SLEW 17

#define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3)

/*

* The key for a thread-local variable consists of the lower 61 bits of the

* t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.

* We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never

* equal to a variable identifier. This is necessary (but not sufficient) to

* assure that global associative arrays never collide with thread-local

* variables. To guarantee that they cannot collide, we must also define the

* order for keying dynamic variables. That order is:

*

* [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]

*

* Because the variable-key and the tls-key are in orthogonal spaces, there is

* no way for a global variable key signature to match a thread-local key

* signature.

*/

#define DTRACE_TLS_THRKEY(where) { \

uint_t intr = 0; \

uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \

for (; actv; actv >>= 1) \

intr++; \

ASSERT(intr < (1 << 3)); \

(where) = ((lx_get_curthread_id() + DIF_VARIABLE_MAX) & \

(((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \

}

#define DT_BSWAP_8(x) ((x) & 0xff)

#define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))

#define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))

#define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))

#define DT_MASK_LO 0x00000000FFFFFFFFULL

#define DTRACE_STORE(type, tomax, offset, what) \

*((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);

#ifndef __i386

#define DTRACE_ALIGNCHECK(addr, size, flags) \

if (addr & (size - 1)) { \

*flags |= CPU_DTRACE_BADALIGN; \

cpu_core[cpu_get_id()].cpuc_dtrace_illval = addr; \

return (0); \

}

#else

#define DTRACE_ALIGNCHECK(addr, size, flags)

#endif

/*

* Test whether a range of memory starting at testaddr of size testsz falls

* within the range of memory described by addr, sz. We take care to avoid

* problems with overflow and underflow of the unsigned quantities, and

* disallow all negative sizes. Ranges of size 0 are allowed.

*/

#define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \

((testaddr) - (baseaddr) < (basesz) && \

(testaddr) + (testsz) - (baseaddr) <= (basesz) && \

(testaddr) + (testsz) >= (testaddr))

/*

* Test whether alloc_sz bytes will fit in the scratch region. We isolate

* alloc_sz on the righthand side of the comparison in order to avoid overflow

* or underflow in the comparison with it. This is simpler than the INRANGE

* check above, because we know that the dtms_scratch_ptr is valid in the

* range. Allocations of size zero are allowed.

*/

#define DTRACE_INSCRATCH(mstate, alloc_sz) \

((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \

(mstate)->dtms_scratch_ptr >= (alloc_sz))

#if defined(linux)

/**********************************************************************/

/* Allow us to debug where we are generating bad address events. */

/**********************************************************************/

/*#define TRACE_BADADDR(x) printk("%s(%d): BADADDR %p\n", __FILE__, __LINE__, x)*/

#define TRACE_BADADDR(x)

#define DTRACE_LOADFUNC(bits) \

/*CSTYLED*/ \

uint##bits##_t \

dtrace_load##bits(uintptr_t addr) \

{ \

/*CSTYLED*/ \

uint##bits##_t rval = 0; \

volatile uint16_t *flags = (volatile uint16_t *) \

&cpu_core[cpu_get_id()].cpuc_dtrace_flags; \

\

*flags |= CPU_DTRACE_NOFAULT; \

rval = *((volatile uint##bits##_t *)addr); \

*flags &= ~CPU_DTRACE_NOFAULT; \

\

return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \

}

#else

#define DTRACE_LOADFUNC(bits) \

/*CSTYLED*/ \

uint##bits##_t \

dtrace_load##bits(uintptr_t addr) \

{ \

size_t size = bits / NBBY; \

/*CSTYLED*/ \

uint##bits##_t rval; \

int i; \

volatile uint16_t *flags = (volatile uint16_t *) \

&cpu_core[cpu_get_id()].cpuc_dtrace_flags; \

\

DTRACE_ALIGNCHECK(addr, size, flags); \

\

for (i = 0; i < dtrace_toxranges; i++) { \

if (addr >= dtrace_toxrange[i].dtt_limit) \

continue; \

\

if (addr + size <= dtrace_toxrange[i].dtt_base) \

continue; \

\

/* \

* This address falls within a toxic region; return 0. \

*/ \

*flags |= CPU_DTRACE_BADADDR; \

TRACE_BADADDR(addr); \

cpu_core[cpu_get_id()].cpuc_dtrace_illval = addr; \

return (0); \

} \

\

*flags |= CPU_DTRACE_NOFAULT; \

/*CSTYLED*/ \

rval = *((volatile uint##bits##_t *)addr); \

*flags &= ~CPU_DTRACE_NOFAULT; \

\

return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \

}

#endif

#ifdef _LP64

#define dtrace_loadptr dtrace_load64

#else

#define dtrace_loadptr dtrace_load32

#endif

#define DTRACE_DYNHASH_FREE 0

#define DTRACE_DYNHASH_SINK 1

#define DTRACE_DYNHASH_VALID 2

#define DTRACE_MATCH_FAIL -1

#define DTRACE_MATCH_NEXT 0

#define DTRACE_MATCH_DONE 1

#define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0')

#define DTRACE_STATE_ALIGN 64

#define DTRACE_FLAGS2FLT(flags) \

(((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \

((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP : \

((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \

((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV : \

((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV : \

((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \

((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \

((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \

((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \

DTRACEFLT_UNKNOWN)

#define DTRACEACT_ISSTRING(act) \

((act)->dta_kind == DTRACEACT_DIFEXPR && \

(act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)

void (*dtrace_cpu_init)(processorid_t);

void (*dtrace_modload)(struct modctl *);

void (*dtrace_modunload)(struct modctl *);

void (*dtrace_helpers_cleanup)(void);

void (*dtrace_helpers_fork)(proc_t *, proc_t *);

void (*dtrace_cpustart_init)(void);

void (*dtrace_cpustart_fini)(void);

void (*dtrace_debugger_init)(void);

void (*dtrace_debugger_fini)(void);

static size_t dtrace_strlen(const char *, size_t);

static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);

static void dtrace_enabling_provide(dtrace_provider_t *);

static int dtrace_enabling_match(dtrace_enabling_t *, int *);

static void dtrace_enabling_matchall(void);

static dtrace_state_t *dtrace_anon_grab(void);

static uint64_t dtrace_helper(int, dtrace_mstate_t *,

dtrace_state_t *, uint64_t, uint64_t);

static dtrace_helpers_t *dtrace_helpers_create(proc_t *);

static void dtrace_buffer_drop(dtrace_buffer_t *);

static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,

dtrace_state_t *, dtrace_mstate_t *);

static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,

dtrace_optval_t);

static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);

static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);

static void dtrace_enabling_reap(void);

/*

* DTrace Probe Context Functions

*

* These functions are called from probe context. Because probe context is

* any context in which C may be called, arbitrarily locks may be held,

* interrupts may be disabled, we may be in arbitrary dispatched state, etc.

* As a result, functions called from probe context may only call other DTrace

* support functions -- they may not interact at all with the system at large.

* (Note that the ASSERT macro is made probe-context safe by redefining it in

* terms of dtrace_assfail(), a probe-context safe function.) If arbitrary

* loads are to be performed from probe context, they _must_ be in terms of

* the safe dtrace_load*() variants.

*

* Some functions in this block are not actually called from probe context;

* for these functions, there will be a comment above the function reading

* "Note: not called from probe context."

*/

# if 0

void dtrace_vpanic(const char *format, void *);

void

dtrace_panic(const char *format, ...)

{

va_list alist;

va_start(alist, format);

dtrace_vpanic(format, alist);

va_end(alist);

}

# endif

int

dtrace_assfail(const char *a, const char *f, int l)

{

dtrace_linux_panic("assertion failed: %s, file: %s, line: %d", a, f, l);

/*

* We just need something here that even the most clever compiler

* cannot optimize away.

*/

return (a[(uintptr_t)f]);

}

/*

* Atomically increment a specified error counter from probe context.

*/

static void

dtrace_error(uint32_t *counter)

{

/*

* Most counters stored to in probe context are per-CPU counters.

* However, there are some error conditions that are sufficiently

* arcane that they don't merit per-CPU storage. If these counters

* are incremented concurrently on different CPUs, scalability will be

* adversely affected -- but we don't expect them to be white-hot in a

* correctly constructed enabling...

*/

uint32_t oval, nval;

do {

oval = *counter;

if ((nval = oval + 1) == 0) {

/*

* If the counter would wrap, set it to 1 -- assuring

* that the counter is never zero when we have seen

* errors. (The counter must be 32-bits because we

* aren't guaranteed a 64-bit compare&swap operation.)

* To save this code both the infamy of being fingered

* by a priggish news story and the indignity of being

* the target of a neo-puritan witch trial, we're

* carefully avoiding any colorful description of the

* likelihood of this condition -- but suffice it to

* say that it is only slightly more likely than the

* overflow of predicate cache IDs, as discussed in

* dtrace_predicate_create().

*/

nval = 1;

}

} while (dtrace_cas32(counter, oval, nval) != oval);

}

/*

* Use the DTRACE_LOADFUNC macro to define functions for each of loading a

* uint8_t, a uint16_t, a uint32_t and a uint64_t.

*/

DTRACE_LOADFUNC(8)

DTRACE_LOADFUNC(16)

DTRACE_LOADFUNC(32)

DTRACE_LOADFUNC(64)

static int

dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)

{

if (dest < mstate->dtms_scratch_base)

return (0);

if (dest + size < dest)

return (0);

if (dest + size > mstate->dtms_scratch_ptr)

return (0);

return (1);

}

static int

dtrace_canstore_statvar(uint64_t addr, size_t sz,

dtrace_statvar_t **svars, int nsvars)

{

int i;

//printk("nsvars=%d\n", nsvars);

for (i = 0; i < nsvars; i++) {

dtrace_statvar_t *svar = svars[i];

if (svar == NULL || svar->dtsv_size == 0)

continue;

//printk("nsvars=%d addr=%p, sz=%d %p %d\n", nsvars, addr, sz, svar->dtsv_data, svar->dtsv_size);

if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))

return (1);

}

return (0);

}

/*

* Check to see if the address is within a memory region to which a store may

* be issued. This includes the DTrace scratch areas, and any DTrace variable

* region. The caller of dtrace_canstore() is responsible for performing any

* alignment checks that are needed before stores are actually executed.

*/

static int

dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,

dtrace_vstate_t *vstate)

{

/*

* First, check to see if the address is in scratch space...

*/

HERE();

//printk("addr=%p, sz=%d %p %d\n", addr, sz, mstate->dtms_scratch_base, mstate->dtms_scratch_size);

if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,

mstate->dtms_scratch_size))

return (1);

HERE();

/*

* Now check to see if it's a dynamic variable. This check will pick

* up both thread-local variables and any global dynamically-allocated

* variables.

*/

if (DTRACE_INRANGE(addr, sz, (uintptr_t)vstate->dtvs_dynvars.dtds_base,

vstate->dtvs_dynvars.dtds_size)) {

dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;

uintptr_t base = (uintptr_t)dstate->dtds_base +

(dstate->dtds_hashsize * sizeof (dtrace_dynhash_t));

uintptr_t chunkoffs;

/*

* Before we assume that we can store here, we need to make

* sure that it isn't in our metadata -- storing to our

* dynamic variable metadata would corrupt our state. For

* the range to not include any dynamic variable metadata,

* it must:

*

* (1) Start above the hash table that is at the base of

* the dynamic variable space

*

* (2) Have a starting chunk offset that is beyond the

* dtrace_dynvar_t that is at the base of every chunk

*

* (3) Not span a chunk boundary

*

*/

HERE();

if (addr < base)

return (0);

chunkoffs = (addr - base) % dstate->dtds_chunksize;

if (chunkoffs < sizeof (dtrace_dynvar_t))

return (0);

if (chunkoffs + sz > dstate->dtds_chunksize)

return (0);

return (1);

}

/*

* Finally, check the static local and global variables. These checks

* take the longest, so we perform them last.

*/

if (dtrace_canstore_statvar(addr, sz,

vstate->dtvs_locals, vstate->dtvs_nlocals))

return (1);

HERE();

if (dtrace_canstore_statvar(addr, sz,

vstate->dtvs_globals, vstate->dtvs_nglobals))

return (1);

HERE();

return (0);

}

/*

* Convenience routine to check to see if the address is within a memory

* region in which a load may be issued given the user's privilege level;

* if not, it sets the appropriate error flags and loads 'addr' into the

* illegal value slot.

*

* DTrace subroutines (DIF_SUBR_*) should use this helper to implement

* appropriate memory access protection.

*/

static int

dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,

dtrace_vstate_t *vstate)

{

volatile uintptr_t *illval = &cpu_core[cpu_get_id()].cpuc_dtrace_illval;

/*

* If we hold the privilege to read from kernel memory, then

* everything is readable.

*/

if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)

return (1);

/*

* You can obviously read that which you can store.

*/

if (dtrace_canstore(addr, sz, mstate, vstate))

return (1);

/*

* We're allowed to read from our own string table.

*/

if (DTRACE_INRANGE(addr, sz, (uintptr_t)mstate->dtms_difo->dtdo_strtab,

mstate->dtms_difo->dtdo_strlen))

return (1);

DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV);

HERE2();

*illval = addr;

return (0);

}

/*

* Convenience routine to check to see if a given string is within a memory

* region in which a load may be issued given the user's privilege level;

* this exists so that we don't need to issue unnecessary dtrace_strlen()

* calls in the event that the user has all privileges.

*/

static int

dtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,

dtrace_vstate_t *vstate)

{

size_t strsz;

/*

* If we hold the privilege to read from kernel memory, then

* everything is readable.

*/

if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)

return (1);

strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz);

if (dtrace_canload(addr, strsz, mstate, vstate))

return (1);

return (0);

}

/*

* Convenience routine to check to see if a given variable is within a memory

* region in which a load may be issued given the user's privilege level.

*/

static int

dtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate,

dtrace_vstate_t *vstate)

{

size_t sz;

ASSERT(type->dtdt_flags & DIF_TF_BYREF);

/*

* If we hold the privilege to read from kernel memory, then

* everything is readable.

*/

HERE();

//printk("dtms_access=%x\n", mstate->dtms_access);

if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)

return (1);

//printk("src=%p\n", src);

HERE();

if (type->dtdt_kind == DIF_TYPE_STRING)

sz = dtrace_strlen(src,

vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1;

else

sz = type->dtdt_size;

HERE();

return (dtrace_canload((uintptr_t)src, sz, mstate, vstate));

}

/**********************************************************************/

/* Called by dtrace_match_string: inline to avoid probe whilst we */

/* are setting up. */

/**********************************************************************/

int

dtrace_strcmp(const char *p1, const char *p2)

{

while (1) {

int ch1 = *p1++;

int ch2 = *p2++;

int ret = ch2 - ch1;

if (ret == 0 && ch1 == 0)

return 0;

if (ret)

return ret;

}

}

/*

* Compare two strings using safe loads.

*/

static int

dtrace_strncmp(char *s1, char *s2, size_t limit)

{

uint8_t c1, c2;

volatile uint16_t *flags;

if (s1 == s2 || limit == 0)

return (0);

flags = (volatile uint16_t *)&cpu_core[cpu_get_id()].cpuc_dtrace_flags;

do {

if (s1 == NULL) {

c1 = '\0';

} else {

c1 = dtrace_load8((uintptr_t)s1++);

}

if (s2 == NULL) {

c2 = '\0';

} else {

c2 = dtrace_load8((uintptr_t)s2++);

}

if (c1 != c2)

return (c1 - c2);

} while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));

return (0);

}

/*

* Compute strlen(s) for a string using safe memory accesses. The additional

* len parameter is used to specify a maximum length to ensure completion.

*/

static size_t

dtrace_strlen(const char *s, size_t lim)

{

uint_t len;

for (len = 0; len != lim; len++) {

if (dtrace_load8((uintptr_t)s++) == '\0')

break;

}

return (len);

}

/*

* Check if an address falls within a toxic region.

*/

static int

dtrace_istoxic(uintptr_t kaddr, size_t size)

{

uintptr_t taddr, tsize;

int i;

HERE();

for (i = 0; i < dtrace_toxranges; i++) {

taddr = dtrace_toxrange[i].dtt_base;

tsize = dtrace_toxrange[i].dtt_limit - taddr;

if (kaddr - taddr < tsize) {

DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);

TRACE_BADADDR(kaddr);

HERE2();

cpu_core[cpu_get_id()].cpuc_dtrace_illval = kaddr;

return (1);

}

if (taddr - kaddr < size) {

DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);

TRACE_BADADDR(kaddr);

HERE2();

cpu_core[cpu_get_id()].cpuc_dtrace_illval = taddr;

return (1);

}

}

return (0);

}

/*