diff options
Diffstat (limited to 'sys/cddl/dev/dtrace')
41 files changed, 16817 insertions, 0 deletions
diff --git a/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S b/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S new file mode 100644 index 000000000000..710232a0d428 --- /dev/null +++ b/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S @@ -0,0 +1,174 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#define _ASM +#define _LOCORE + +#include <sys/cpuvar_defs.h> +#include <sys/dtrace.h> + +#include <machine/armreg.h> +#include <machine/asm.h> + +#include "assym.inc" + +/* +void dtrace_membar_producer(void) +*/ +ENTRY(dtrace_membar_producer) + RET +END(dtrace_membar_producer) + +/* +void dtrace_membar_consumer(void) +*/ +ENTRY(dtrace_membar_consumer) + RET +END(dtrace_membar_consumer) + +/* +dtrace_icookie_t dtrace_interrupt_disable(void) +*/ +ENTRY(dtrace_interrupt_disable) + mrs x0, daif + msr daifset, #2 + RET +END(dtrace_interrupt_disable) + +/* +void dtrace_interrupt_enable(dtrace_icookie_t cookie) +*/ +ENTRY(dtrace_interrupt_enable) + msr daif, x0 + RET +END(dtrace_interrupt_enable) +/* +uint8_t +dtrace_fuword8_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword8_nocheck) + ldrb w0, [x0] + RET +END(dtrace_fuword8_nocheck) + +/* +uint16_t +dtrace_fuword16_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword16_nocheck) + ldrh w0, [x0] + RET +END(dtrace_fuword16_nocheck) + +/* +uint32_t +dtrace_fuword32_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword32_nocheck) + ldr w0, [x0] + RET +END(dtrace_fuword32_nocheck) + +/* +uint64_t +dtrace_fuword64_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword64_nocheck) + ldr x0, [x0] + RET +END(dtrace_fuword64_nocheck) + +/* +void +dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) +*/ +ENTRY(dtrace_copy) + cbz x2, 2f /* If len == 0 then skip loop */ +1: + ldrb w4, [x0], #1 /* Load from uaddr */ + strb w4, [x1], #1 /* Store in kaddr */ + sub x2, x2, #1 /* len-- */ + cbnz x2, 1b +2: + RET +END(dtrace_copy) + +/* +void +dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +XXX: Check for flags? +*/ +ENTRY(dtrace_copystr) + cbz x2, 2f /* If len == 0 then skip loop */ + +1: ldrb w4, [x0], #1 /* Load from uaddr */ + strb w4, [x1], #1 /* Store in kaddr */ + cbz w4, 2f /* If == 0 then break */ + sub x2, x2, #1 /* len-- */ + cbnz x2, 1b +2: + RET +END(dtrace_copystr) + +/* +uintptr_t +dtrace_caller(int aframes) +*/ +ENTRY(dtrace_caller) + mov x0, #-1 + RET +END(dtrace_caller) + +/* +uint32_t +dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) +*/ +ENTRY(dtrace_cas32) +1: ldxr w3, [x0] /* Load target */ + cmp w3, w1 /* Check if *target == cmp */ + bne 2f /* No, return */ + stxr w12, w2, [x0] /* Store new to target */ + cbnz w12, 1b /* Try again if store not succeed */ +2: mov w0, w3 /* Return the value loaded from target */ + RET +END(dtrace_cas32) + +/* +void * +dtrace_casptr(volatile void *target, volatile void *cmp, volatile void *new) +*/ +ENTRY(dtrace_casptr) +1: ldxr x3, [x0] /* Load target */ + cmp x3, x1 /* Check if *target == cmp */ + bne 2f /* No, return */ + stxr w12, x2, [x0] /* Store new to target */ + cbnz w12, 1b /* Try again if store not succeed */ +2: mov x0, x3 /* Return the value loaded from target */ + RET +END(dtrace_casptr) diff --git a/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c b/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c new file mode 100644 index 000000000000..082bd1904279 --- /dev/null +++ b/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c @@ -0,0 +1,409 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#include <sys/cdefs.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/stack.h> +#include <sys/pcpu.h> + +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/reg.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> + +#include <machine/atomic.h> +#include <machine/db_machdep.h> +#include <machine/md_var.h> +#include <machine/stack.h> +#include <ddb/db_sym.h> +#include <ddb/ddb.h> +#include <sys/kdb.h> + +#include "regset.h" + +/* + * Wee need some reasonable default to prevent backtrace code + * from wandering too far + */ +#define MAX_FUNCTION_SIZE 0x10000 +#define MAX_PROLOGUE_SIZE 0x100 +#define MAX_USTACK_DEPTH 2048 + +uint8_t dtrace_fuword8_nocheck(void *); +uint16_t dtrace_fuword16_nocheck(void *); +uint32_t dtrace_fuword32_nocheck(void *); +uint64_t dtrace_fuword64_nocheck(void *); + +void +dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, + uint32_t *intrpc) +{ + struct unwind_state state; + int scp_offset; + register_t sp, fp; + int depth; + + depth = 0; + + if (intrpc != 0) { + pcstack[depth++] = (pc_t) intrpc; + } + + aframes++; + + __asm __volatile("mov %0, sp" : "=&r" (sp)); + + state.fp = (uint64_t)__builtin_frame_address(0); + state.sp = sp; + state.pc = (uint64_t)dtrace_getpcstack; + + while (depth < pcstack_limit) { + if (!INKERNEL(state.pc) || !INKERNEL(state.fp)) + break; + + fp = state.fp; + state.sp = fp + 0x10; + /* FP to previous frame (X29) */ + state.fp = *(register_t *)(fp); + /* LR (X30) */ + state.pc = *(register_t *)(fp + 8) - 4; + + /* + * NB: Unlike some other architectures, we don't need to + * explicitly insert cpu_dtrace_caller as it appears in the + * normal kernel stack trace rather than a special trap frame. + */ + if (aframes > 0) { + aframes--; + } else { + pcstack[depth++] = state.pc; + } + + } + + for (; depth < pcstack_limit; depth++) { + pcstack[depth] = 0; + } +} + +static int +dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc, + uintptr_t fp) +{ + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + int ret = 0; + uintptr_t oldfp = fp; + + ASSERT(pcstack == NULL || pcstack_limit > 0); + + while (pc != 0) { + /* + * We limit the number of times we can go around this + * loop to account for a circular stack. + */ + if (ret++ >= MAX_USTACK_DEPTH) { + *flags |= CPU_DTRACE_BADSTACK; + cpu_core[curcpu].cpuc_dtrace_illval = fp; + break; + } + + if (pcstack != NULL) { + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (fp == 0) + break; + + pc = dtrace_fuword64((void *)(fp + + offsetof(struct arm64_frame, f_retaddr))); + fp = dtrace_fuword64((void *)fp); + + if (fp == oldfp) { + *flags |= CPU_DTRACE_BADSTACK; + cpu_core[curcpu].cpuc_dtrace_illval = fp; + break; + } + + /* + * ARM64TODO: + * This workaround might not be necessary. It needs to be + * revised and removed from all architectures if found + * unwanted. Leaving the original x86 comment for reference. + * + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } + + oldfp = fp; + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, sp, fp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + int n; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = tf->tf_elr; + sp = tf->tf_sp; + fp = tf->tf_x[29]; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = tf->tf_lr; + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp); + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +int +dtrace_getustackdepth(void) +{ + + printf("IMPLEMENT ME: %s\n", __func__); + + return (0); +} + +void +dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) +{ + + printf("IMPLEMENT ME: %s\n", __func__); +} + +/*ARGSUSED*/ +uint64_t +dtrace_getarg(int arg, int aframes) +{ + + printf("IMPLEMENT ME: %s\n", __func__); + + return (0); +} + +int +dtrace_getstackdepth(int aframes) +{ + struct unwind_state state; + int scp_offset; + register_t sp; + int depth; + int done; + + depth = 1; + done = 0; + + __asm __volatile("mov %0, sp" : "=&r" (sp)); + + state.fp = (uint64_t)__builtin_frame_address(0); + state.sp = sp; + state.pc = (uint64_t)dtrace_getstackdepth; + + do { + done = unwind_frame(&state); + if (!INKERNEL(state.pc) || !INKERNEL(state.fp)) + break; + depth++; + } while (!done); + + if (depth < aframes) + return (0); + else + return (depth - aframes); +} + +ulong_t +dtrace_getreg(struct trapframe *rp, uint_t reg) +{ + + printf("IMPLEMENT ME: %s\n", __func__); + + return (0); +} + +static int +dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) +{ + + if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = uaddr; + return (0); + } + + return (1); +} + +void +dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(uaddr, kaddr, size); +} + +void +dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(kaddr, uaddr, size); +} + +void +dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(uaddr, kaddr, size, flags); +} + +void +dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(kaddr, uaddr, size, flags); +} + +uint8_t +dtrace_fuword8(void *uaddr) +{ + + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + + return (dtrace_fuword8_nocheck(uaddr)); +} + +uint16_t +dtrace_fuword16(void *uaddr) +{ + + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + + return (dtrace_fuword16_nocheck(uaddr)); +} + +uint32_t +dtrace_fuword32(void *uaddr) +{ + + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + + return (dtrace_fuword32_nocheck(uaddr)); +} + +uint64_t +dtrace_fuword64(void *uaddr) +{ + + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + + return (dtrace_fuword64_nocheck(uaddr)); +} diff --git a/sys/cddl/dev/dtrace/aarch64/dtrace_subr.c b/sys/cddl/dev/dtrace/aarch64/dtrace_subr.c new file mode 100644 index 000000000000..1921052b0480 --- /dev/null +++ b/sys/cddl/dev/dtrace/aarch64/dtrace_subr.c @@ -0,0 +1,310 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/kmem.h> +#include <sys/smp.h> +#include <sys/dtrace_impl.h> +#include <sys/dtrace_bsd.h> +#include <machine/armreg.h> +#include <machine/clock.h> +#include <machine/frame.h> +#include <machine/trap.h> +#include <machine/vmparam.h> +#include <vm/pmap.h> + +extern dtrace_id_t dtrace_probeid_error; +extern int (*dtrace_invop_jump_addr)(struct trapframe *); +extern void dtrace_getnanotime(struct timespec *tsp); + +int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t); +void dtrace_invop_init(void); +void dtrace_invop_uninit(void); + +typedef struct dtrace_invop_hdlr { + int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t); + struct dtrace_invop_hdlr *dtih_next; +} dtrace_invop_hdlr_t; + +dtrace_invop_hdlr_t *dtrace_invop_hdlr; + +int +dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) +{ + dtrace_invop_hdlr_t *hdlr; + int rval; + + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) + if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) + return (rval); + + return (0); +} + + +void +dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr; + + hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); + hdlr->dtih_func = func; + hdlr->dtih_next = dtrace_invop_hdlr; + dtrace_invop_hdlr = hdlr; +} + +void +dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr, *prev; + + hdlr = dtrace_invop_hdlr; + prev = NULL; + + for (;;) { + if (hdlr == NULL) + panic("attempt to remove non-existent invop handler"); + + if (hdlr->dtih_func == func) + break; + + prev = hdlr; + hdlr = hdlr->dtih_next; + } + + if (prev == NULL) { + ASSERT(dtrace_invop_hdlr == hdlr); + dtrace_invop_hdlr = hdlr->dtih_next; + } else { + ASSERT(dtrace_invop_hdlr != hdlr); + prev->dtih_next = hdlr->dtih_next; + } + + kmem_free(hdlr, 0); +} + +/*ARGSUSED*/ +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + + (*func)(0, (uintptr_t)VM_MIN_KERNEL_ADDRESS); +} + +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) +{ + cpuset_t cpus; + + if (cpu == DTRACE_CPUALL) + cpus = all_cpus; + else + CPU_SETOF(cpu, &cpus); + + smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func, + smp_no_rendezvous_barrier, arg); +} + +static void +dtrace_sync_func(void) +{ + +} + +void +dtrace_sync(void) +{ + + dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); +} + +/* + * DTrace needs a high resolution time function which can + * be called from a probe context and guaranteed not to have + * instrumented with probes itself. + * + * Returns nanoseconds since boot. + */ +uint64_t +dtrace_gethrtime() +{ + struct timespec curtime; + + nanouptime(&curtime); + + return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec); + +} + +uint64_t +dtrace_gethrestime(void) +{ + struct timespec current_time; + + dtrace_getnanotime(¤t_time); + + return (current_time.tv_sec * 1000000000UL + current_time.tv_nsec); +} + +/* Function to handle DTrace traps during probes. See arm64/arm64/trap.c */ +int +dtrace_trap(struct trapframe *frame, u_int type) +{ + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in its per-cpu flags to indicate that it doesn't + * want to fault. On returning from the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * Check if DTrace has enabled 'no-fault' mode: + * + */ + + if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) { + /* + * There are only a couple of trap types that are expected. + * All the rest will be handled in the usual way. + */ + switch (type) { + case EXCP_DATA_ABORT: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = 0; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->tf_elr += 4; + return (1); + default: + /* Handle all other traps in the usual way. */ + break; + } + } + + /* Handle the trap in the usual way. */ + return (0); +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fault, int fltoffs, uintptr_t illval) +{ + + dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state, + (uintptr_t)epid, + (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs); +} + +static int +dtrace_invop_start(struct trapframe *frame) +{ + int data, invop, reg, update_sp; + register_t arg1, arg2; + register_t *sp; + int offs; + int tmp; + int i; + + invop = dtrace_invop(frame->tf_elr, frame, frame->tf_elr); + + tmp = (invop & LDP_STP_MASK); + if (tmp == STP_64 || tmp == LDP_64) { + sp = (register_t *)frame->tf_sp; + data = invop; + arg1 = (data >> ARG1_SHIFT) & ARG1_MASK; + arg2 = (data >> ARG2_SHIFT) & ARG2_MASK; + + offs = (data >> OFFSET_SHIFT) & OFFSET_MASK; + + switch (tmp) { + case STP_64: + if (offs >> (OFFSET_SIZE - 1)) + sp -= (~offs & OFFSET_MASK) + 1; + else + sp += (offs); + *(sp + 0) = frame->tf_x[arg1]; + *(sp + 1) = frame->tf_x[arg2]; + break; + case LDP_64: + frame->tf_x[arg1] = *(sp + 0); + frame->tf_x[arg2] = *(sp + 1); + if (offs >> (OFFSET_SIZE - 1)) + sp -= (~offs & OFFSET_MASK) + 1; + else + sp += (offs); + break; + default: + break; + } + + /* Update the stack pointer and program counter to continue */ + frame->tf_sp = (register_t)sp; + frame->tf_elr += INSN_SIZE; + return (0); + } + + if ((invop & B_MASK) == B_INSTR) { + data = (invop & B_DATA_MASK); + /* The data is the number of 4-byte words to change the pc */ + data *= 4; + frame->tf_elr += data; + return (0); + } + + if (invop == RET_INSTR) { + frame->tf_elr = frame->tf_lr; + return (0); + } + + return (-1); +} + +void +dtrace_invop_init(void) +{ + + dtrace_invop_jump_addr = dtrace_invop_start; +} + +void +dtrace_invop_uninit(void) +{ + + dtrace_invop_jump_addr = 0; +} diff --git a/sys/cddl/dev/dtrace/aarch64/regset.h b/sys/cddl/dev/dtrace/aarch64/regset.h new file mode 100644 index 000000000000..f99b48f8354f --- /dev/null +++ b/sys/cddl/dev/dtrace/aarch64/regset.h @@ -0,0 +1,51 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +#ifndef _REGSET_H +#define _REGSET_H + +/* + * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Place here */ + +#ifdef __cplusplus +} +#endif + +#endif /* _REGSET_H */ diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_asm.S b/sys/cddl/dev/dtrace/amd64/dtrace_asm.S new file mode 100644 index 000000000000..22e1d6964fb8 --- /dev/null +++ b/sys/cddl/dev/dtrace/amd64/dtrace_asm.S @@ -0,0 +1,439 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Portions Copyright 2008 John Birrell <jb@freebsd.org> + * + * $FreeBSD$ + * + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#define _ASM + +#include <machine/asmacros.h> +#include <sys/cpuvar_defs.h> +#include <sys/dtrace.h> + +#include "assym.inc" + +#define INTR_POP \ + MEXITCOUNT; \ + movq TF_RDI(%rsp),%rdi; \ + movq TF_RSI(%rsp),%rsi; \ + movq TF_RDX(%rsp),%rdx; \ + movq TF_RCX(%rsp),%rcx; \ + movq TF_R8(%rsp),%r8; \ + movq TF_R9(%rsp),%r9; \ + movq TF_RAX(%rsp),%rax; \ + movq TF_RBX(%rsp),%rbx; \ + movq TF_RBP(%rsp),%rbp; \ + movq TF_R10(%rsp),%r10; \ + movq TF_R11(%rsp),%r11; \ + movq TF_R12(%rsp),%r12; \ + movq TF_R13(%rsp),%r13; \ + movq TF_R14(%rsp),%r14; \ + movq TF_R15(%rsp),%r15; \ + testb $SEL_RPL_MASK,TF_CS(%rsp); \ + jz 1f; \ + cli; \ + swapgs; \ +1: addq $TF_RIP,%rsp; + + + ENTRY(dtrace_invop_start) + + /* + * #BP traps with %rip set to the next address. We need to decrement + * the value to indicate the address of the int3 (0xcc) instruction + * that we substituted. + */ + movq TF_RIP(%rsp), %rdi + decq %rdi + movq %rsp, %rsi + movq TF_RAX(%rsp), %rdx + call dtrace_invop + ALTENTRY(dtrace_invop_callsite) + cmpl $DTRACE_INVOP_PUSHL_EBP, %eax + je bp_push + cmpl $DTRACE_INVOP_LEAVE, %eax + je bp_leave + cmpl $DTRACE_INVOP_NOP, %eax + je bp_nop + cmpl $DTRACE_INVOP_RET, %eax + je bp_ret + + /* When all else fails handle the trap in the usual way. */ + jmpq *dtrace_invop_calltrap_addr + +bp_push: + /* + * We must emulate a "pushq %rbp". To do this, we pull the stack + * down 8 bytes, and then store the base pointer. + */ + INTR_POP + subq $16, %rsp /* make room for %rbp */ + pushq %rax /* push temp */ + movq 24(%rsp), %rax /* load calling RIP */ + movq %rax, 8(%rsp) /* store calling RIP */ + movq 32(%rsp), %rax /* load calling CS */ + movq %rax, 16(%rsp) /* store calling CS */ + movq 40(%rsp), %rax /* load calling RFLAGS */ + movq %rax, 24(%rsp) /* store calling RFLAGS */ + movq 48(%rsp), %rax /* load calling RSP */ + subq $8, %rax /* make room for %rbp */ + movq %rax, 32(%rsp) /* store calling RSP */ + movq 56(%rsp), %rax /* load calling SS */ + movq %rax, 40(%rsp) /* store calling SS */ + movq 32(%rsp), %rax /* reload calling RSP */ + movq %rbp, (%rax) /* store %rbp there */ + popq %rax /* pop off temp */ + iretq /* return from interrupt */ + /*NOTREACHED*/ + +bp_leave: + /* + * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" + * followed by a "popq %rbp". This is quite a bit simpler on amd64 + * than it is on i386 -- we can exploit the fact that the %rsp is + * explicitly saved to effect the pop without having to reshuffle + * the other data pushed for the trap. + */ + INTR_POP + pushq %rax /* push temp */ + movq 8(%rsp), %rax /* load calling RIP */ + movq %rax, 8(%rsp) /* store calling RIP */ + movq (%rbp), %rax /* get new %rbp */ + addq $8, %rbp /* adjust new %rsp */ + movq %rbp, 32(%rsp) /* store new %rsp */ + movq %rax, %rbp /* set new %rbp */ + popq %rax /* pop off temp */ + iretq /* return from interrupt */ + /*NOTREACHED*/ + +bp_nop: + /* We must emulate a "nop". */ + INTR_POP + iretq + /*NOTREACHED*/ + +bp_ret: + INTR_POP + pushq %rax /* push temp */ + movq 32(%rsp), %rax /* load %rsp */ + movq (%rax), %rax /* load calling RIP */ + movq %rax, 8(%rsp) /* store calling RIP */ + addq $8, 32(%rsp) /* adjust new %rsp */ + popq %rax /* pop off temp */ + iretq /* return from interrupt */ + /*NOTREACHED*/ + + END(dtrace_invop_start) + +/* +void dtrace_invop_init(void) +*/ + ENTRY(dtrace_invop_init) + movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip) + ret + END(dtrace_invop_init) + +/* +void dtrace_invop_uninit(void) +*/ + ENTRY(dtrace_invop_uninit) + movq $0, dtrace_invop_jump_addr(%rip) + ret + END(dtrace_invop_uninit) + +/* +greg_t dtrace_getfp(void) +*/ + ENTRY(dtrace_getfp) + movq %rbp, %rax + ret + END(dtrace_getfp) + +/* +uint32_t +dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) +*/ + ENTRY(dtrace_cas32) + movl %esi, %eax + lock + cmpxchgl %edx, (%rdi) + ret + END(dtrace_cas32) + +/* +void * +dtrace_casptr(void *target, void *cmp, void *new) +*/ + ENTRY(dtrace_casptr) + movq %rsi, %rax + lock + cmpxchgq %rdx, (%rdi) + ret + END(dtrace_casptr) + +/* +uintptr_t +dtrace_caller(int aframes) +*/ + ENTRY(dtrace_caller) + movq $-1, %rax + ret + END(dtrace_caller) + +/* +void +dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) +*/ + ENTRY(dtrace_copy_nosmap) + pushq %rbp + movq %rsp, %rbp + + xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ + movq %rdx, %rcx /* load count */ + repz /* repeat for count ... */ + smovb /* move from %ds:rsi to %ed:rdi */ + leave + ret + END(dtrace_copy_nosmap) + + ENTRY(dtrace_copy_smap) + pushq %rbp + movq %rsp, %rbp + + xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ + movq %rdx, %rcx /* load count */ + stac + repz /* repeat for count ... */ + smovb /* move from %ds:rsi to %ed:rdi */ + clac + leave + ret + END(dtrace_copy_smap) + +/* +void +dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +*/ + ENTRY(dtrace_copystr_nosmap) + pushq %rbp + movq %rsp, %rbp + +0: + movb (%rdi), %al /* load from source */ + movb %al, (%rsi) /* store to destination */ + addq $1, %rdi /* increment source pointer */ + addq $1, %rsi /* increment destination pointer */ + subq $1, %rdx /* decrement remaining count */ + cmpb $0, %al + je 2f + testq $0xfff, %rdx /* test if count is 4k-aligned */ + jnz 1f /* if not, continue with copying */ + testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ + jnz 2f +1: + cmpq $0, %rdx + jne 0b +2: + leave + ret + + END(dtrace_copystr_nosmap) + + ENTRY(dtrace_copystr_smap) + pushq %rbp + movq %rsp, %rbp + + stac +0: + movb (%rdi), %al /* load from source */ + movb %al, (%rsi) /* store to destination */ + addq $1, %rdi /* increment source pointer */ + addq $1, %rsi /* increment destination pointer */ + subq $1, %rdx /* decrement remaining count */ + cmpb $0, %al + je 2f + testq $0xfff, %rdx /* test if count is 4k-aligned */ + jnz 1f /* if not, continue with copying */ + testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ + jnz 2f +1: + cmpq $0, %rdx + jne 0b +2: + clac + leave + ret + + END(dtrace_copystr_smap) + +/* +uintptr_t +dtrace_fulword(void *addr) +*/ + ENTRY(dtrace_fulword_nosmap) + movq (%rdi), %rax + ret + END(dtrace_fulword_nosmap) + + ENTRY(dtrace_fulword_smap) + stac + movq (%rdi), %rax + clac + ret + END(dtrace_fulword_smap) + +/* +uint8_t +dtrace_fuword8_nocheck(void *addr) +*/ + ENTRY(dtrace_fuword8_nocheck_nosmap) + xorq %rax, %rax + movb (%rdi), %al + ret + END(dtrace_fuword8_nocheck_nosmap) + + ENTRY(dtrace_fuword8_nocheck_smap) + stac + xorq %rax, %rax + movb (%rdi), %al + clac + ret + END(dtrace_fuword8_nocheck_smap) + +/* +uint16_t +dtrace_fuword16_nocheck(void *addr) +*/ + ENTRY(dtrace_fuword16_nocheck_nosmap) + xorq %rax, %rax + movw (%rdi), %ax + ret + END(dtrace_fuword16_nocheck_nosmap) + + ENTRY(dtrace_fuword16_nocheck_smap) + stac + xorq %rax, %rax + movw (%rdi), %ax + clac + ret + END(dtrace_fuword16_nocheck_smap) + +/* +uint32_t +dtrace_fuword32_nocheck(void *addr) +*/ + ENTRY(dtrace_fuword32_nocheck_nosmap) + xorq %rax, %rax + movl (%rdi), %eax + ret + END(dtrace_fuword32_nocheck_nosmap) + + ENTRY(dtrace_fuword32_nocheck_smap) + stac + xorq %rax, %rax + movl (%rdi), %eax + clac + ret + END(dtrace_fuword32_nocheck_smap) + +/* +uint64_t +dtrace_fuword64_nocheck(void *addr) +*/ + ENTRY(dtrace_fuword64_nocheck_nosmap) + movq (%rdi), %rax + ret + END(dtrace_fuword64_nocheck_nosmap) + + ENTRY(dtrace_fuword64_nocheck_smap) + stac + movq (%rdi), %rax + clac + ret + END(dtrace_fuword64_nocheck_smap) + +/* +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fault, int fltoffs, uintptr_t illval) +*/ + ENTRY(dtrace_probe_error) + pushq %rbp + movq %rsp, %rbp + subq $0x8, %rsp + movq %r9, (%rsp) + movq %r8, %r9 + movq %rcx, %r8 + movq %rdx, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movl dtrace_probeid_error(%rip), %edi + call dtrace_probe + addq $0x8, %rsp + leave + ret + END(dtrace_probe_error) + +/* +void +dtrace_membar_producer(void) +*/ + ENTRY(dtrace_membar_producer) + rep; ret /* use 2 byte return instruction when branch target */ + /* AMD Software Optimization Guide - Section 6.2 */ + END(dtrace_membar_producer) + +/* +void +dtrace_membar_consumer(void) +*/ + ENTRY(dtrace_membar_consumer) + rep; ret /* use 2 byte return instruction when branch target */ + /* AMD Software Optimization Guide - Section 6.2 */ + END(dtrace_membar_consumer) + +/* +dtrace_icookie_t +dtrace_interrupt_disable(void) +*/ + ENTRY(dtrace_interrupt_disable) + pushfq + popq %rax + cli + ret + END(dtrace_interrupt_disable) + +/* +void +dtrace_interrupt_enable(dtrace_icookie_t cookie) +*/ + ENTRY(dtrace_interrupt_enable) + pushq %rdi + popfq + ret + END(dtrace_interrupt_enable) diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_isa.c b/sys/cddl/dev/dtrace/amd64/dtrace_isa.c new file mode 100644 index 000000000000..e5c6e226318e --- /dev/null +++ b/sys/cddl/dev/dtrace/amd64/dtrace_isa.c @@ -0,0 +1,734 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#include <sys/cdefs.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/stack.h> +#include <sys/pcpu.h> + +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/reg.h> +#include <machine/stack.h> +#include <x86/ifunc.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> + +#include "regset.h" + +uint8_t dtrace_fuword8_nocheck(void *); +uint16_t dtrace_fuword16_nocheck(void *); +uint32_t dtrace_fuword32_nocheck(void *); +uint64_t dtrace_fuword64_nocheck(void *); + +int dtrace_ustackdepth_max = 2048; + +void +dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, + uint32_t *intrpc) +{ + int depth = 0; + register_t rbp; + struct amd64_frame *frame; + vm_offset_t callpc; + pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller; + + if (intrpc != 0) + pcstack[depth++] = (pc_t) intrpc; + + aframes++; + + __asm __volatile("movq %%rbp,%0" : "=r" (rbp)); + + frame = (struct amd64_frame *)rbp; + while (depth < pcstack_limit) { + if (!INKERNEL((long) frame)) + break; + + callpc = frame->f_retaddr; + + if (!INKERNEL(callpc)) + break; + + if (aframes > 0) { + aframes--; + if ((aframes == 0) && (caller != 0)) { + pcstack[depth++] = caller; + } + } + else { + pcstack[depth++] = callpc; + } + + if (frame->f_frame <= frame || + (vm_offset_t)frame->f_frame >= curthread->td_kstack + + curthread->td_kstack_pages * PAGE_SIZE) + break; + frame = frame->f_frame; + } + + for (; depth < pcstack_limit; depth++) { + pcstack[depth] = 0; + } +} + +static int +dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc, + uintptr_t sp) +{ + uintptr_t oldsp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + int ret = 0; + + ASSERT(pcstack == NULL || pcstack_limit > 0); + ASSERT(dtrace_ustackdepth_max > 0); + + while (pc != 0) { + /* + * We limit the number of times we can go around this + * loop to account for a circular stack. + */ + if (ret++ >= dtrace_ustackdepth_max) { + *flags |= CPU_DTRACE_BADSTACK; + cpu_core[curcpu].cpuc_dtrace_illval = sp; + break; + } + + if (pcstack != NULL) { + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (sp == 0) + break; + + oldsp = sp; + + pc = dtrace_fuword64((void *)(sp + + offsetof(struct amd64_frame, f_retaddr))); + sp = dtrace_fuword64((void *)sp); + + if (sp == oldsp) { + *flags |= CPU_DTRACE_BADSTACK; + cpu_core[curcpu].cpuc_dtrace_illval = sp; + break; + } + + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, sp, fp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + int n; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = tf->tf_rip; + fp = tf->tf_rbp; + sp = tf->tf_rsp; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = dtrace_fuword64((void *) sp); + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp); + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +int +dtrace_getustackdepth(void) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, fp, sp; + int n = 0; + + if (p == NULL || (tf = curthread->td_frame) == NULL) + return (0); + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) + return (-1); + + pc = tf->tf_rip; + fp = tf->tf_rbp; + sp = tf->tf_rsp; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + + pc = dtrace_fuword64((void *) sp); + n++; + } + + n += dtrace_getustack_common(NULL, 0, pc, fp); + + return (n); +} + +void +dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, sp, fp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; +#ifdef notyet /* XXX signal stack */ + uintptr_t oldcontext; + size_t s1, s2; +#endif + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = tf->tf_rip; + sp = tf->tf_rsp; + fp = tf->tf_rbp; + +#ifdef notyet /* XXX signal stack */ + oldcontext = lwp->lwp_oldcontext; + s1 = sizeof (struct xframe) + 2 * sizeof (long); + s2 = s1 + sizeof (siginfo_t); +#endif + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t)pc; + *fpstack++ = 0; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = dtrace_fuword64((void *)sp); + } + + while (pc != 0) { + *pcstack++ = (uint64_t)pc; + *fpstack++ = fp; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + + if (fp == 0) + break; + +#ifdef notyet /* XXX signal stack */ + if (oldcontext == sp + s1 || oldcontext == sp + s2) { + ucontext_t *ucp = (ucontext_t *)oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fulword(&gregs[REG_FP]); + pc = dtrace_fulword(&gregs[REG_PC]); + + oldcontext = dtrace_fulword(&ucp->uc_link); + } else +#endif /* XXX */ + { + pc = dtrace_fuword64((void *)(fp + + offsetof(struct amd64_frame, f_retaddr))); + fp = dtrace_fuword64((void *)fp); + } + + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } + } + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +/*ARGSUSED*/ +uint64_t +dtrace_getarg(int arg, int aframes) +{ + uintptr_t val; + struct amd64_frame *fp = (struct amd64_frame *)dtrace_getfp(); + uintptr_t *stack; + int i; + + /* + * A total of 6 arguments are passed via registers; any argument with + * index of 5 or lower is therefore in a register. + */ + int inreg = 5; + + for (i = 1; i <= aframes; i++) { + fp = fp->f_frame; + + if (P2ROUNDUP(fp->f_retaddr, 16) == + (long)dtrace_invop_callsite) { + /* + * In the case of amd64, we will use the pointer to the + * regs structure that was pushed when we took the + * trap. To get this structure, we must increment + * beyond the frame structure, and then again beyond + * the calling RIP stored in dtrace_invop(). If the + * argument that we're seeking is passed on the stack, + * we'll pull the true stack pointer out of the saved + * registers and decrement our argument by the number + * of arguments passed in registers; if the argument + * we're seeking is passed in registers, we can just + * load it directly. + */ + struct trapframe *tf = (struct trapframe *)&fp[1]; + + if (arg <= inreg) { + switch (arg) { + case 0: + stack = (uintptr_t *)&tf->tf_rdi; + break; + case 1: + stack = (uintptr_t *)&tf->tf_rsi; + break; + case 2: + stack = (uintptr_t *)&tf->tf_rdx; + break; + case 3: + stack = (uintptr_t *)&tf->tf_rcx; + break; + case 4: + stack = (uintptr_t *)&tf->tf_r8; + break; + case 5: + stack = (uintptr_t *)&tf->tf_r9; + break; + } + arg = 0; + } else { + stack = (uintptr_t *)(tf->tf_rsp); + arg -= inreg; + } + goto load; + } + + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- the provider simply called dtrace_probe() + * directly. As this is the case, we need to shift the argument + * that we're looking for: the probe ID is the first argument to + * dtrace_probe(), so the argument n will actually be found where + * one would expect to find argument (n + 1). + */ + arg++; + + if (arg <= inreg) { + /* + * This shouldn't happen. If the argument is passed in a + * register then it should have been, well, passed in a + * register... + */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + arg -= (inreg + 1); + stack = (uintptr_t *)&fp[1]; + +load: + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + val = stack[arg]; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + + return (val); +} + +int +dtrace_getstackdepth(int aframes) +{ + int depth = 0; + struct amd64_frame *frame; + vm_offset_t rbp; + + aframes++; + rbp = dtrace_getfp(); + frame = (struct amd64_frame *)rbp; + depth++; + for(;;) { + if (!INKERNEL((long) frame)) + break; + if (!INKERNEL((long) frame->f_frame)) + break; + depth++; + if (frame->f_frame <= frame || + (vm_offset_t)frame->f_frame >= curthread->td_kstack + + curthread->td_kstack_pages * PAGE_SIZE) + break; + frame = frame->f_frame; + } + if (depth < aframes) + return 0; + else + return depth - aframes; +} + +ulong_t +dtrace_getreg(struct trapframe *rp, uint_t reg) +{ + /* This table is dependent on reg.d. */ + int regmap[] = { + REG_GS, /* 0 GS */ + REG_FS, /* 1 FS */ + REG_ES, /* 2 ES */ + REG_DS, /* 3 DS */ + REG_RDI, /* 4 EDI */ + REG_RSI, /* 5 ESI */ + REG_RBP, /* 6 EBP, REG_FP */ + REG_RSP, /* 7 ESP */ + REG_RBX, /* 8 EBX, REG_R1 */ + REG_RDX, /* 9 EDX */ + REG_RCX, /* 10 ECX */ + REG_RAX, /* 11 EAX, REG_R0 */ + REG_TRAPNO, /* 12 TRAPNO */ + REG_ERR, /* 13 ERR */ + REG_RIP, /* 14 EIP, REG_PC */ + REG_CS, /* 15 CS */ + REG_RFL, /* 16 EFL, REG_PS */ + REG_RSP, /* 17 UESP, REG_SP */ + REG_SS /* 18 SS */ + }; + + if (reg <= SS) { + if (reg >= sizeof (regmap) / sizeof (int)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + reg = regmap[reg]; + } else { + /* This is dependent on reg.d. */ + reg -= SS + 1; + } + + switch (reg) { + case REG_RDI: + return (rp->tf_rdi); + case REG_RSI: + return (rp->tf_rsi); + case REG_RDX: + return (rp->tf_rdx); + case REG_RCX: + return (rp->tf_rcx); + case REG_R8: + return (rp->tf_r8); + case REG_R9: + return (rp->tf_r9); + case REG_RAX: + return (rp->tf_rax); + case REG_RBX: + return (rp->tf_rbx); + case REG_RBP: + return (rp->tf_rbp); + case REG_R10: + return (rp->tf_r10); + case REG_R11: + return (rp->tf_r11); + case REG_R12: + return (rp->tf_r12); + case REG_R13: + return (rp->tf_r13); + case REG_R14: + return (rp->tf_r14); + case REG_R15: + return (rp->tf_r15); + case REG_DS: + return (rp->tf_ds); + case REG_ES: + return (rp->tf_es); + case REG_FS: + return (rp->tf_fs); + case REG_GS: + return (rp->tf_gs); + case REG_TRAPNO: + return (rp->tf_trapno); + case REG_ERR: + return (rp->tf_err); + case REG_RIP: + return (rp->tf_rip); + case REG_CS: + return (rp->tf_cs); + case REG_SS: + return (rp->tf_ss); + case REG_RFL: + return (rp->tf_rflags); + case REG_RSP: + return (rp->tf_rsp); + default: + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } +} + +static int +dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) +{ + ASSERT(INKERNEL(kaddr) && kaddr + size >= kaddr); + + if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = uaddr; + return (0); + } + + return (1); +} + +void +dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(uaddr, kaddr, size); +} + +void +dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(kaddr, uaddr, size); +} + +void +dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(uaddr, kaddr, size, flags); +} + +void +dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(kaddr, uaddr, size, flags); +} + +uint8_t +dtrace_fuword8(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword8_nocheck(uaddr)); +} + +uint16_t +dtrace_fuword16(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword16_nocheck(uaddr)); +} + +uint32_t +dtrace_fuword32(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword32_nocheck(uaddr)); +} + +uint64_t +dtrace_fuword64(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword64_nocheck(uaddr)); +} + +/* + * ifunc resolvers for SMAP support + */ +void dtrace_copy_nosmap(uintptr_t, uintptr_t, size_t); +void dtrace_copy_smap(uintptr_t, uintptr_t, size_t); +DEFINE_IFUNC(, void, dtrace_copy, (uintptr_t, uintptr_t, size_t)) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + dtrace_copy_smap : dtrace_copy_nosmap); +} + +void dtrace_copystr_nosmap(uintptr_t, uintptr_t, size_t, volatile uint16_t *); +void dtrace_copystr_smap(uintptr_t, uintptr_t, size_t, volatile uint16_t *); +DEFINE_IFUNC(, void, dtrace_copystr, (uintptr_t, uintptr_t, size_t, + volatile uint16_t *)) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + dtrace_copystr_smap : dtrace_copystr_nosmap); +} + +uintptr_t dtrace_fulword_nosmap(void *); +uintptr_t dtrace_fulword_smap(void *); +DEFINE_IFUNC(, uintptr_t, dtrace_fulword, (void *)) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + dtrace_fulword_smap : dtrace_fulword_nosmap); +} + +uint8_t dtrace_fuword8_nocheck_nosmap(void *); +uint8_t dtrace_fuword8_nocheck_smap(void *); +DEFINE_IFUNC(, uint8_t, dtrace_fuword8_nocheck, (void *)) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + dtrace_fuword8_nocheck_smap : dtrace_fuword8_nocheck_nosmap); +} + +uint16_t dtrace_fuword16_nocheck_nosmap(void *); +uint16_t dtrace_fuword16_nocheck_smap(void *); +DEFINE_IFUNC(, uint16_t, dtrace_fuword16_nocheck, (void *)) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + dtrace_fuword16_nocheck_smap : dtrace_fuword16_nocheck_nosmap); +} + +uint32_t dtrace_fuword32_nocheck_nosmap(void *); +uint32_t dtrace_fuword32_nocheck_smap(void *); +DEFINE_IFUNC(, uint32_t, dtrace_fuword32_nocheck, (void *)) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + dtrace_fuword32_nocheck_smap : dtrace_fuword32_nocheck_nosmap); +} + +uint64_t dtrace_fuword64_nocheck_nosmap(void *); +uint64_t dtrace_fuword64_nocheck_smap(void *); +DEFINE_IFUNC(, uint64_t, dtrace_fuword64_nocheck, (void *)) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + dtrace_fuword64_nocheck_smap : dtrace_fuword64_nocheck_nosmap); +} diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_subr.c b/sys/cddl/dev/dtrace/amd64/dtrace_subr.c new file mode 100644 index 000000000000..82d20ebba936 --- /dev/null +++ b/sys/cddl/dev/dtrace/amd64/dtrace_subr.c @@ -0,0 +1,449 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * Copyright (c) 2011, Joyent, Inc. All rights reserved. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/kmem.h> +#include <sys/smp.h> +#include <sys/dtrace_impl.h> +#include <sys/dtrace_bsd.h> +#include <machine/clock.h> +#include <machine/cpufunc.h> +#include <machine/frame.h> +#include <machine/psl.h> +#include <machine/trap.h> +#include <vm/pmap.h> + +extern void dtrace_getnanotime(struct timespec *tsp); + +int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t); + +typedef struct dtrace_invop_hdlr { + int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t); + struct dtrace_invop_hdlr *dtih_next; +} dtrace_invop_hdlr_t; + +dtrace_invop_hdlr_t *dtrace_invop_hdlr; + +int +dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) +{ + dtrace_invop_hdlr_t *hdlr; + int rval; + + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) + if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) + return (rval); + + return (0); +} + +void +dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr; + + hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); + hdlr->dtih_func = func; + hdlr->dtih_next = dtrace_invop_hdlr; + dtrace_invop_hdlr = hdlr; +} + +void +dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL; + + for (;;) { + if (hdlr == NULL) + panic("attempt to remove non-existent invop handler"); + + if (hdlr->dtih_func == func) + break; + + prev = hdlr; + hdlr = hdlr->dtih_next; + } + + if (prev == NULL) { + ASSERT(dtrace_invop_hdlr == hdlr); + dtrace_invop_hdlr = hdlr->dtih_next; + } else { + ASSERT(dtrace_invop_hdlr != hdlr); + prev->dtih_next = hdlr->dtih_next; + } + + kmem_free(hdlr, 0); +} + +/*ARGSUSED*/ +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + (*func)(0, (uintptr_t) addr_PTmap); +} + +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) +{ + cpuset_t cpus; + + if (cpu == DTRACE_CPUALL) + cpus = all_cpus; + else + CPU_SETOF(cpu, &cpus); + + smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func, + smp_no_rendezvous_barrier, arg); +} + +static void +dtrace_sync_func(void) +{ +} + +void +dtrace_sync(void) +{ + dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); +} + +#ifdef notyet +void +dtrace_safe_synchronous_signal(void) +{ + kthread_t *t = curthread; + struct regs *rp = lwptoregs(ttolwp(t)); + size_t isz = t->t_dtrace_npc - t->t_dtrace_pc; + + ASSERT(t->t_dtrace_on); + + /* + * If we're not in the range of scratch addresses, we're not actually + * tracing user instructions so turn off the flags. If the instruction + * we copied out caused a synchonous trap, reset the pc back to its + * original value and turn off the flags. + */ + if (rp->r_pc < t->t_dtrace_scrpc || + rp->r_pc > t->t_dtrace_astpc + isz) { + t->t_dtrace_ft = 0; + } else if (rp->r_pc == t->t_dtrace_scrpc || + rp->r_pc == t->t_dtrace_astpc) { + rp->r_pc = t->t_dtrace_pc; + t->t_dtrace_ft = 0; + } +} + +int +dtrace_safe_defer_signal(void) +{ + kthread_t *t = curthread; + struct regs *rp = lwptoregs(ttolwp(t)); + size_t isz = t->t_dtrace_npc - t->t_dtrace_pc; + + ASSERT(t->t_dtrace_on); + + /* + * If we're not in the range of scratch addresses, we're not actually + * tracing user instructions so turn off the flags. + */ + if (rp->r_pc < t->t_dtrace_scrpc || + rp->r_pc > t->t_dtrace_astpc + isz) { + t->t_dtrace_ft = 0; + return (0); + } + + /* + * If we have executed the original instruction, but we have performed + * neither the jmp back to t->t_dtrace_npc nor the clean up of any + * registers used to emulate %rip-relative instructions in 64-bit mode, + * we'll save ourselves some effort by doing that here and taking the + * signal right away. We detect this condition by seeing if the program + * counter is the range [scrpc + isz, astpc). + */ + if (rp->r_pc >= t->t_dtrace_scrpc + isz && + rp->r_pc < t->t_dtrace_astpc) { +#ifdef __amd64 + /* + * If there is a scratch register and we're on the + * instruction immediately after the modified instruction, + * restore the value of that scratch register. + */ + if (t->t_dtrace_reg != 0 && + rp->r_pc == t->t_dtrace_scrpc + isz) { + switch (t->t_dtrace_reg) { + case REG_RAX: + rp->r_rax = t->t_dtrace_regv; + break; + case REG_RCX: + rp->r_rcx = t->t_dtrace_regv; + break; + case REG_R8: + rp->r_r8 = t->t_dtrace_regv; + break; + case REG_R9: + rp->r_r9 = t->t_dtrace_regv; + break; + } + } +#endif + rp->r_pc = t->t_dtrace_npc; + t->t_dtrace_ft = 0; + return (0); + } + + /* + * Otherwise, make sure we'll return to the kernel after executing + * the copied out instruction and defer the signal. + */ + if (!t->t_dtrace_step) { + ASSERT(rp->r_pc < t->t_dtrace_astpc); + rp->r_pc += t->t_dtrace_astpc - t->t_dtrace_scrpc; + t->t_dtrace_step = 1; + } + + t->t_dtrace_ast = 1; + + return (1); +} +#endif + +static int64_t tgt_cpu_tsc; +static int64_t hst_cpu_tsc; +static int64_t tsc_skew[MAXCPU]; +static uint64_t nsec_scale; + +/* See below for the explanation of this macro. */ +#define SCALE_SHIFT 28 + +static void +dtrace_gethrtime_init_cpu(void *arg) +{ + uintptr_t cpu = (uintptr_t) arg; + + if (cpu == curcpu) + tgt_cpu_tsc = rdtsc(); + else + hst_cpu_tsc = rdtsc(); +} + +#ifdef EARLY_AP_STARTUP +static void +dtrace_gethrtime_init(void *arg) +{ + struct pcpu *pc; + uint64_t tsc_f; + cpuset_t map; + int i; +#else +/* + * Get the frequency and scale factor as early as possible so that they can be + * used for boot-time tracing. + */ +static void +dtrace_gethrtime_init_early(void *arg) +{ + uint64_t tsc_f; +#endif + + /* + * Get TSC frequency known at this moment. + * This should be constant if TSC is invariant. + * Otherwise tick->time conversion will be inaccurate, but + * will preserve monotonic property of TSC. + */ + tsc_f = atomic_load_acq_64(&tsc_freq); + + /* + * The following line checks that nsec_scale calculated below + * doesn't overflow 32-bit unsigned integer, so that it can multiply + * another 32-bit integer without overflowing 64-bit. + * Thus minimum supported TSC frequency is 62.5MHz. + */ + KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)), + ("TSC frequency is too low")); + + /* + * We scale up NANOSEC/tsc_f ratio to preserve as much precision + * as possible. + * 2^28 factor was chosen quite arbitrarily from practical + * considerations: + * - it supports TSC frequencies as low as 62.5MHz (see above); + * - it provides quite good precision (e < 0.01%) up to THz + * (terahertz) values; + */ + nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f; +#ifndef EARLY_AP_STARTUP +} +SYSINIT(dtrace_gethrtime_init_early, SI_SUB_CPU, SI_ORDER_ANY, + dtrace_gethrtime_init_early, NULL); + +static void +dtrace_gethrtime_init(void *arg) +{ + struct pcpu *pc; + cpuset_t map; + int i; +#endif + + if (vm_guest != VM_GUEST_NO) + return; + + /* The current CPU is the reference one. */ + sched_pin(); + tsc_skew[curcpu] = 0; + CPU_FOREACH(i) { + if (i == curcpu) + continue; + + pc = pcpu_find(i); + CPU_SETOF(PCPU_GET(cpuid), &map); + CPU_SET(pc->pc_cpuid, &map); + + smp_rendezvous_cpus(map, NULL, + dtrace_gethrtime_init_cpu, + smp_no_rendezvous_barrier, (void *)(uintptr_t) i); + + tsc_skew[i] = tgt_cpu_tsc - hst_cpu_tsc; + } + sched_unpin(); +} +#ifdef EARLY_AP_STARTUP +SYSINIT(dtrace_gethrtime_init, SI_SUB_DTRACE, SI_ORDER_ANY, + dtrace_gethrtime_init, NULL); +#else +SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, + NULL); +#endif + +/* + * DTrace needs a high resolution time function which can + * be called from a probe context and guaranteed not to have + * instrumented with probes itself. + * + * Returns nanoseconds since boot. + */ +uint64_t +dtrace_gethrtime(void) +{ + uint64_t tsc; + uint32_t lo, hi; + register_t rflags; + + /* + * We split TSC value into lower and higher 32-bit halves and separately + * scale them with nsec_scale, then we scale them down by 2^28 + * (see nsec_scale calculations) taking into account 32-bit shift of + * the higher half and finally add. + */ + rflags = intr_disable(); + tsc = rdtsc() - tsc_skew[curcpu]; + intr_restore(rflags); + + lo = tsc; + hi = tsc >> 32; + return (((lo * nsec_scale) >> SCALE_SHIFT) + + ((hi * nsec_scale) << (32 - SCALE_SHIFT))); +} + +uint64_t +dtrace_gethrestime(void) +{ + struct timespec current_time; + + dtrace_getnanotime(¤t_time); + + return (current_time.tv_sec * 1000000000ULL + current_time.tv_nsec); +} + +/* Function to handle DTrace traps during probes. See amd64/amd64/trap.c. */ +int +dtrace_trap(struct trapframe *frame, u_int type) +{ + uint16_t nofault; + + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in its per-cpu flags to indicate that it doesn't + * want to fault. On returning from the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * Check if DTrace has enabled 'no-fault' mode: + */ + sched_pin(); + nofault = cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT; + sched_unpin(); + if (nofault) { + KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled")); + + /* + * There are only a couple of trap types that are expected. + * All the rest will be handled in the usual way. + */ + switch (type) { + /* General protection fault. */ + case T_PROTFLT: + /* Flag an illegal operation. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->tf_rip += dtrace_instr_size((u_char *) frame->tf_rip); + return (1); + /* Page fault. */ + case T_PAGEFLT: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = frame->tf_addr; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->tf_rip += dtrace_instr_size((u_char *) frame->tf_rip); + return (1); + default: + /* Handle all other traps in the usual way. */ + break; + } + } + + /* Handle the trap in the usual way. */ + return (0); +} diff --git a/sys/cddl/dev/dtrace/arm/dtrace_asm.S b/sys/cddl/dev/dtrace/arm/dtrace_asm.S new file mode 100644 index 000000000000..1b9996bb133a --- /dev/null +++ b/sys/cddl/dev/dtrace/arm/dtrace_asm.S @@ -0,0 +1,198 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#define _ASM +#define _LOCORE + +#include <sys/cpuvar_defs.h> +#include <sys/dtrace.h> + +#include <machine/asm.h> +#include <machine/armreg.h> + +#include "assym.inc" + +/* +void dtrace_membar_producer(void) +*/ +ENTRY(dtrace_membar_producer) + RET +END(dtrace_membar_producer) + +/* +void dtrace_membar_consumer(void) +*/ +ENTRY(dtrace_membar_consumer) + RET +END(dtrace_membar_consumer) + +/* +dtrace_icookie_t dtrace_interrupt_disable(void) +*/ +ENTRY(dtrace_interrupt_disable) + mrs r0, cpsr + mov r1, r0 + orr r1, r1, #(PSR_I | PSR_F) + msr cpsr_c, r1 + RET +END(dtrace_interrupt_disable) + +/* +void dtrace_interrupt_enable(dtrace_icookie_t cookie) +*/ +ENTRY(dtrace_interrupt_enable) + and r0, r0, #(PSR_I | PSR_F) + mrs r1, cpsr + bic r1, r1, #(PSR_I | PSR_F) + orr r1, r1, r0 + msr cpsr_c, r1 + RET +END(dtrace_interrupt_enable) +/* +uint8_t +dtrace_fuword8_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword8_nocheck) + ldrb r3, [r0] + mov r0, r3 + RET +END(dtrace_fuword8_nocheck) + +/* +uint16_t +dtrace_fuword16_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword16_nocheck) + ldrh r3, [r0] + mov r0, r3 + RET +END(dtrace_fuword16_nocheck) + +/* +uint32_t +dtrace_fuword32_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword32_nocheck) + ldr r3, [r0] + mov r0, r3 + RET +END(dtrace_fuword32_nocheck) + +/* +uint64_t +dtrace_fuword64_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword64_nocheck) + ldm r0, {r2, r3} + + mov r0, r2 + mov r1, r3 +#if defined(__BIG_ENDIAN__) +/* big endian */ + mov r0, r3 + mov r1, r2 +#else +/* little endian */ + mov r0, r2 + mov r1, r3 + +#endif + RET +END(dtrace_fuword64_nocheck) + +/* +void +dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) +*/ +ENTRY(dtrace_copy) + stmfd sp!, {r4-r5} /* stack is 8 byte aligned */ + teq r2, #0x00000000 + mov r5, #0x00000000 + beq 2f + +1: ldrb r4, [r0], #0x0001 + add r5, r5, #0x00000001 + strb r4, [r1], #0x0001 + teqne r5, r2 + bne 1b + +2: ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ + RET +END(dtrace_copy) + +/* +void +dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +XXX: Check for flags? +*/ +ENTRY(dtrace_copystr) + stmfd sp!, {r4-r5} /* stack is 8 byte aligned */ + teq r2, #0x00000000 + mov r5, #0x00000000 + beq 2f + +1: ldrb r4, [r0], #0x0001 + add r5, r5, #0x00000001 + teq r4, #0x00000000 + strb r4, [r1], #0x0001 + teqne r5, r2 + bne 1b + +2: ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ + RET +END(dtrace_copystr) + +/* +uintptr_t +dtrace_caller(int aframes) +*/ +ENTRY(dtrace_caller) + mov r0, #-1 + RET +END(dtrace_caller) + +/* +uint32_t +dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) + +void * +dtrace_casptr(volatile void *target, volatile void *cmp, volatile void *new) +*/ +ENTRY(dtrace_cas32) +EENTRY(dtrace_casptr) +1: ldrex r3, [r0] /* Load target */ + cmp r3, r1 /* Check if *target == cmp */ + bne 2f /* No, return */ + strex ip, r2, [r0] /* Store new to target */ + cmp ip, #0 /* Did the store succeed? */ + bne 1b /* No, try again */ +2: mov r0, r3 /* Return the value loaded from target */ + RET +EEND(dtrace_casptr) +END(dtrace_cas32) diff --git a/sys/cddl/dev/dtrace/arm/dtrace_isa.c b/sys/cddl/dev/dtrace/arm/dtrace_isa.c new file mode 100644 index 000000000000..9bac37669a84 --- /dev/null +++ b/sys/cddl/dev/dtrace/arm/dtrace_isa.c @@ -0,0 +1,263 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#include <sys/cdefs.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/stack.h> +#include <sys/pcpu.h> + +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/reg.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> + +#include <machine/atomic.h> +#include <machine/db_machdep.h> +#include <machine/md_var.h> +#include <machine/stack.h> +#include <ddb/db_sym.h> +#include <ddb/ddb.h> +#include <sys/kdb.h> + +#include "regset.h" + +/* + * Wee need some reasonable default to prevent backtrace code + * from wandering too far + */ +#define MAX_FUNCTION_SIZE 0x10000 +#define MAX_PROLOGUE_SIZE 0x100 + + +uint8_t dtrace_fuword8_nocheck(void *); +uint16_t dtrace_fuword16_nocheck(void *); +uint32_t dtrace_fuword32_nocheck(void *); +uint64_t dtrace_fuword64_nocheck(void *); + +void +dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, + uint32_t *intrpc) +{ + struct unwind_state state; + register_t sp; + int scp_offset; + int depth = 0; + + if (intrpc != 0) + pcstack[depth++] = (pc_t) intrpc; + + aframes++; + + __asm __volatile("mov %0, sp" : "=&r" (sp)); + + state.registers[FP] = (uint32_t)__builtin_frame_address(0); + state.registers[SP] = sp; + state.registers[LR] = (uint32_t)__builtin_return_address(0); + state.registers[PC] = (uint32_t)dtrace_getpcstack; + + while (depth < pcstack_limit) { + int done; + + done = unwind_stack_one(&state, 1); + + /* + * NB: Unlike some other architectures, we don't need to + * explicitly insert cpu_dtrace_caller as it appears in the + * normal kernel stack trace rather than a special trap frame. + */ + if (aframes > 0) { + aframes--; + } else { + pcstack[depth++] = state.registers[PC]; + } + + if (done) + break; + } + + for (; depth < pcstack_limit; depth++) { + pcstack[depth] = 0; + } +} + +void +dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) +{ + printf("IMPLEMENT ME: %s\n", __func__); +} + +int +dtrace_getustackdepth(void) +{ + printf("IMPLEMENT ME: %s\n", __func__); + return (0); +} + +void +dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) +{ + printf("IMPLEMENT ME: %s\n", __func__); +} + +/*ARGSUSED*/ +uint64_t +dtrace_getarg(int arg, int aframes) +{ +/* struct arm_frame *fp = (struct arm_frame *)dtrace_getfp();*/ + + return (0); +} + +int +dtrace_getstackdepth(int aframes) +{ + struct unwind_state state; + register_t sp; + int scp_offset; + int done = 0; + int depth = 1; + + __asm __volatile("mov %0, sp" : "=&r" (sp)); + + state.registers[FP] = (uint32_t)__builtin_frame_address(0); + state.registers[SP] = sp; + state.registers[LR] = (uint32_t)__builtin_return_address(0); + state.registers[PC] = (uint32_t)dtrace_getstackdepth; + + do { + done = unwind_stack_one(&state, 1); + depth++; + } while (!done); + + if (depth < aframes) + return 0; + else + return depth - aframes; +} + +ulong_t +dtrace_getreg(struct trapframe *rp, uint_t reg) +{ + printf("IMPLEMENT ME: %s\n", __func__); + + return (0); +} + +static int +dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) +{ + + if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = uaddr; + return (0); + } + + return (1); +} + +void +dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(uaddr, kaddr, size); +} + +void +dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(kaddr, uaddr, size); +} + +void +dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(uaddr, kaddr, size, flags); +} + +void +dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(kaddr, uaddr, size, flags); +} + +uint8_t +dtrace_fuword8(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword8_nocheck(uaddr)); +} + +uint16_t +dtrace_fuword16(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword16_nocheck(uaddr)); +} + +uint32_t +dtrace_fuword32(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword32_nocheck(uaddr)); +} + +uint64_t +dtrace_fuword64(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword64_nocheck(uaddr)); +} diff --git a/sys/cddl/dev/dtrace/arm/dtrace_subr.c b/sys/cddl/dev/dtrace/arm/dtrace_subr.c new file mode 100644 index 000000000000..e2833131fcb8 --- /dev/null +++ b/sys/cddl/dev/dtrace/arm/dtrace_subr.c @@ -0,0 +1,345 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/kmem.h> +#include <sys/smp.h> +#include <sys/dtrace_impl.h> +#include <sys/dtrace_bsd.h> +#include <machine/armreg.h> +#include <machine/clock.h> +#include <machine/frame.h> +#include <machine/trap.h> +#include <vm/pmap.h> + +#define DELAYBRANCH(x) ((int)(x) < 0) + +#define BIT_PC 15 +#define BIT_LR 14 +#define BIT_SP 13 + +extern dtrace_id_t dtrace_probeid_error; +extern int (*dtrace_invop_jump_addr)(struct trapframe *); +extern void dtrace_getnanotime(struct timespec *tsp); + +int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t); +void dtrace_invop_init(void); +void dtrace_invop_uninit(void); + +typedef struct dtrace_invop_hdlr { + int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t); + struct dtrace_invop_hdlr *dtih_next; +} dtrace_invop_hdlr_t; + +dtrace_invop_hdlr_t *dtrace_invop_hdlr; + +int +dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) +{ + dtrace_invop_hdlr_t *hdlr; + int rval; + + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) + if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) + return (rval); + + return (0); +} + + +void +dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr; + + hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); + hdlr->dtih_func = func; + hdlr->dtih_next = dtrace_invop_hdlr; + dtrace_invop_hdlr = hdlr; +} + +void +dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL; + + for (;;) { + if (hdlr == NULL) + panic("attempt to remove non-existent invop handler"); + + if (hdlr->dtih_func == func) + break; + + prev = hdlr; + hdlr = hdlr->dtih_next; + } + + if (prev == NULL) { + ASSERT(dtrace_invop_hdlr == hdlr); + dtrace_invop_hdlr = hdlr->dtih_next; + } else { + ASSERT(dtrace_invop_hdlr != hdlr); + prev->dtih_next = hdlr->dtih_next; + } + + kmem_free(hdlr, 0); +} + + +/*ARGSUSED*/ +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + printf("IMPLEMENT ME: dtrace_toxic_ranges\n"); +} + +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) +{ + cpuset_t cpus; + + if (cpu == DTRACE_CPUALL) + cpus = all_cpus; + else + CPU_SETOF(cpu, &cpus); + + smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func, + smp_no_rendezvous_barrier, arg); +} + +static void +dtrace_sync_func(void) +{ +} + +void +dtrace_sync(void) +{ + dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); +} + +/* + * DTrace needs a high resolution time function which can + * be called from a probe context and guaranteed not to have + * instrumented with probes itself. + * + * Returns nanoseconds since boot. + */ +uint64_t +dtrace_gethrtime() +{ + struct timespec curtime; + + nanouptime(&curtime); + + return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec); + +} + +uint64_t +dtrace_gethrestime(void) +{ + struct timespec current_time; + + dtrace_getnanotime(¤t_time); + + return (current_time.tv_sec * 1000000000UL + current_time.tv_nsec); +} + +/* Function to handle DTrace traps during probes. See amd64/amd64/trap.c */ +int +dtrace_trap(struct trapframe *frame, u_int type) +{ + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in its per-cpu flags to indicate that it doesn't + * want to fault. On returning from the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * Check if DTrace has enabled 'no-fault' mode: + * + */ + if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) { + /* + * There are only a couple of trap types that are expected. + * All the rest will be handled in the usual way. + */ + switch (type) { + /* Page fault. */ + case FAULT_ALIGN: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = 0; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->tf_pc += sizeof(int); + return (1); + default: + /* Handle all other traps in the usual way. */ + break; + } + } + + /* Handle the trap in the usual way. */ + return (0); +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fault, int fltoffs, uintptr_t illval) +{ + + dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state, + (uintptr_t)epid, + (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs); +} + +static int +dtrace_invop_start(struct trapframe *frame) +{ + register_t *r0, *sp; + int data, invop, reg, update_sp; + + invop = dtrace_invop(frame->tf_pc, frame, frame->tf_pc); + switch (invop & DTRACE_INVOP_MASK) { + case DTRACE_INVOP_PUSHM: + sp = (register_t *)frame->tf_svc_sp; + r0 = &frame->tf_r0; + data = DTRACE_INVOP_DATA(invop); + + /* + * Store the pc, lr, and sp. These have their own + * entries in the struct. + */ + if (data & (1 << BIT_PC)) { + sp--; + *sp = frame->tf_pc; + } + if (data & (1 << BIT_LR)) { + sp--; + *sp = frame->tf_svc_lr; + } + if (data & (1 << BIT_SP)) { + sp--; + *sp = frame->tf_svc_sp; + } + + /* Store the general registers */ + for (reg = 12; reg >= 0; reg--) { + if (data & (1 << reg)) { + sp--; + *sp = r0[reg]; + } + } + + /* Update the stack pointer and program counter to continue */ + frame->tf_svc_sp = (register_t)sp; + frame->tf_pc += 4; + break; + case DTRACE_INVOP_POPM: + sp = (register_t *)frame->tf_svc_sp; + r0 = &frame->tf_r0; + data = DTRACE_INVOP_DATA(invop); + + /* Read the general registers */ + for (reg = 0; reg <= 12; reg++) { + if (data & (1 << reg)) { + r0[reg] = *sp; + sp++; + } + } + + /* + * Set the stack pointer. If we don't update it here we will + * need to update it at the end as the instruction would do + */ + update_sp = 1; + if (data & (1 << BIT_SP)) { + frame->tf_svc_sp = *sp; + *sp++; + update_sp = 0; + } + + /* Update the link register, we need to use the correct copy */ + if (data & (1 << BIT_LR)) { + frame->tf_svc_lr = *sp; + *sp++; + } + /* + * And the program counter. If it's not in the list skip over + * it when we return so to not hit this again. + */ + if (data & (1 << BIT_PC)) { + frame->tf_pc = *sp; + *sp++; + } else + frame->tf_pc += 4; + + /* Update the stack pointer if we haven't already done so */ + if (update_sp) + frame->tf_svc_sp = (register_t)sp; + break; + case DTRACE_INVOP_B: + data = DTRACE_INVOP_DATA(invop) & 0x00ffffff; + /* Sign extend the data */ + if ((data & (1 << 23)) != 0) + data |= 0xff000000; + /* The data is the number of 4-byte words to change the pc */ + data *= 4; + data += 8; + frame->tf_pc += data; + break; + default: + return (-1); + break; + } + + return (0); +} + +void dtrace_invop_init(void) +{ + dtrace_invop_jump_addr = dtrace_invop_start; +} + +void dtrace_invop_uninit(void) +{ + dtrace_invop_jump_addr = 0; +} diff --git a/sys/cddl/dev/dtrace/arm/regset.h b/sys/cddl/dev/dtrace/arm/regset.h new file mode 100644 index 000000000000..ce9e97ea7a09 --- /dev/null +++ b/sys/cddl/dev/dtrace/arm/regset.h @@ -0,0 +1,57 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +#ifndef _REGSET_H +#define _REGSET_H + +/* + * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#if 0 +#define REG_LINK R14 +#define REG_SP R12 +#define REG_PS R0 +#define REG_R0 R0 +#define REG_R1 R1 +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _REGSET_H */ diff --git a/sys/cddl/dev/dtrace/dtrace_anon.c b/sys/cddl/dev/dtrace/dtrace_anon.c new file mode 100644 index 000000000000..b81ec5be3f11 --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_anon.c @@ -0,0 +1,84 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ + +/* + * DTrace Anonymous Enabling Functions + */ +static void +dtrace_anon_init(void *dummy) +{ + dtrace_state_t *state = NULL; + dtrace_enabling_t *enab; + + mutex_enter(&cpu_lock); + mutex_enter(&dtrace_provider_lock); + mutex_enter(&dtrace_lock); + + dtrace_anon_property(); + + mutex_exit(&cpu_lock); + + /* + * If there are already providers, we must ask them to provide their + * probes, and then match any anonymous enabling against them. Note + * that there should be no other retained enablings at this time: + * the only retained enablings at this time should be the anonymous + * enabling. + */ + if (dtrace_anon.dta_enabling != NULL) { + ASSERT(dtrace_retained == dtrace_anon.dta_enabling); + + dtrace_enabling_provide(NULL); + state = dtrace_anon.dta_state; + + /* + * We couldn't hold cpu_lock across the above call to + * dtrace_enabling_provide(), but we must hold it to actually + * enable the probes. We have to drop all of our locks, pick + * up cpu_lock, and regain our locks before matching the + * retained anonymous enabling. + */ + mutex_exit(&dtrace_lock); + mutex_exit(&dtrace_provider_lock); + + mutex_enter(&cpu_lock); + mutex_enter(&dtrace_provider_lock); + mutex_enter(&dtrace_lock); + + if ((enab = dtrace_anon.dta_enabling) != NULL) + (void) dtrace_enabling_match(enab, NULL); + + mutex_exit(&cpu_lock); + } + + mutex_exit(&dtrace_provider_lock); + mutex_exit(&dtrace_lock); + + if (state != NULL) { + /* + * If we created any anonymous state, set it going now. + */ + (void) dtrace_state_go(state, &dtrace_anon.dta_beganon); + } +} diff --git a/sys/cddl/dev/dtrace/dtrace_cddl.h b/sys/cddl/dev/dtrace/dtrace_cddl.h new file mode 100644 index 000000000000..344fe562bff9 --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_cddl.h @@ -0,0 +1,174 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ + +#ifndef _DTRACE_CDDL_H_ +#define _DTRACE_CDDL_H_ + +#include <sys/proc.h> + +#define LOCK_LEVEL 10 + +/* + * Kernel DTrace extension to 'struct proc' for FreeBSD. + */ +typedef struct kdtrace_proc { + int p_dtrace_probes; /* Are there probes for this proc? */ + u_int64_t p_dtrace_count; /* Number of DTrace tracepoints */ + void *p_dtrace_helpers; /* DTrace helpers, if any */ + int p_dtrace_model; + uint64_t p_fasttrap_tp_gen; /* Tracepoint hash table gen */ +} kdtrace_proc_t; + +/* + * Kernel DTrace extension to 'struct thread' for FreeBSD. + */ +typedef struct kdtrace_thread { + u_int8_t td_dtrace_stop; /* Indicates a DTrace-desired stop */ + u_int8_t td_dtrace_sig; /* Signal sent via DTrace's raise() */ + u_int8_t td_dtrace_inprobe; /* Are we in a probe? */ + u_int td_predcache; /* DTrace predicate cache */ + u_int64_t td_dtrace_vtime; /* DTrace virtual time */ + u_int64_t td_dtrace_start; /* DTrace slice start time */ + + union __tdu { + struct __tds { + u_int8_t _td_dtrace_on; + /* Hit a fasttrap tracepoint. */ + u_int8_t _td_dtrace_step; + /* About to return to kernel. */ + u_int8_t _td_dtrace_ret; + /* Handling a return probe. */ + u_int8_t _td_dtrace_ast; + /* Saved ast flag. */ +#ifdef __amd64__ + u_int8_t _td_dtrace_reg; +#endif + } _tds; + u_long _td_dtrace_ft; /* Bitwise or of these flags. */ + } _tdu; +#define td_dtrace_ft _tdu._td_dtrace_ft +#define td_dtrace_on _tdu._tds._td_dtrace_on +#define td_dtrace_step _tdu._tds._td_dtrace_step +#define td_dtrace_ret _tdu._tds._td_dtrace_ret +#define td_dtrace_ast _tdu._tds._td_dtrace_ast +#define td_dtrace_reg _tdu._tds._td_dtrace_reg + + uintptr_t td_dtrace_pc; /* DTrace saved pc from fasttrap. */ + uintptr_t td_dtrace_npc; /* DTrace next pc from fasttrap. */ + uintptr_t td_dtrace_scrpc; + /* DTrace per-thread scratch location. */ + uintptr_t td_dtrace_astpc; + /* DTrace return sequence location. */ +#ifdef __amd64__ + uintptr_t td_dtrace_regv; +#endif + u_int64_t td_hrtime; /* Last time on cpu. */ + void *td_dtrace_sscr; /* Saved scratch space location. */ + void *td_systrace_args; /* syscall probe arguments. */ + uint64_t td_fasttrap_tp_gen; /* Tracepoint hash table gen. */ +} kdtrace_thread_t; + +/* + * Definitions to reference fields in the FreeBSD DTrace structures defined + * above using the names of fields in similar structures in Solaris. Note + * that the separation on FreeBSD is a licensing constraint designed to + * keep the GENERIC kernel BSD licensed. + */ +#define t_dtrace_vtime td_dtrace->td_dtrace_vtime +#define t_dtrace_start td_dtrace->td_dtrace_start +#define t_dtrace_stop td_dtrace->td_dtrace_stop +#define t_dtrace_sig td_dtrace->td_dtrace_sig +#define t_dtrace_inprobe td_dtrace->td_dtrace_inprobe +#define t_predcache td_dtrace->td_predcache +#define t_dtrace_ft td_dtrace->td_dtrace_ft +#define t_dtrace_on td_dtrace->td_dtrace_on +#define t_dtrace_step td_dtrace->td_dtrace_step +#define t_dtrace_ret td_dtrace->td_dtrace_ret +#define t_dtrace_ast td_dtrace->td_dtrace_ast +#define t_dtrace_reg td_dtrace->td_dtrace_reg +#define t_dtrace_pc td_dtrace->td_dtrace_pc +#define t_dtrace_npc td_dtrace->td_dtrace_npc +#define t_dtrace_scrpc td_dtrace->td_dtrace_scrpc +#define t_dtrace_astpc td_dtrace->td_dtrace_astpc +#define t_dtrace_regv td_dtrace->td_dtrace_regv +#define t_dtrace_sscr td_dtrace->td_dtrace_sscr +#define t_dtrace_systrace_args td_dtrace->td_systrace_args +#define t_fasttrap_tp_gen td_dtrace->td_fasttrap_tp_gen +#define p_dtrace_helpers p_dtrace->p_dtrace_helpers +#define p_dtrace_count p_dtrace->p_dtrace_count +#define p_dtrace_probes p_dtrace->p_dtrace_probes +#define p_model p_dtrace->p_dtrace_model +#define p_fasttrap_tp_gen p_dtrace->p_fasttrap_tp_gen + +#define DATAMODEL_NATIVE 0 +#ifdef __amd64__ +#define DATAMODEL_LP64 0 +#define DATAMODEL_ILP32 1 +#else +#define DATAMODEL_LP64 1 +#define DATAMODEL_ILP32 0 +#endif + +/* + * Definitions for fields in struct proc which are named differently in FreeBSD. + */ +#define p_cred p_ucred +#define p_parent p_pptr + +/* + * Definitions for fields in struct thread which are named differently in FreeBSD. + */ +#define t_procp td_proc +#define t_tid td_tid +#define t_did td_tid +#define t_cred td_ucred + + +int priv_policy(const cred_t *, int, boolean_t, int, const char *); +boolean_t priv_policy_only(const cred_t *, int, boolean_t); +boolean_t priv_policy_choice(const cred_t *, int, boolean_t); + +/* + * Test privilege. Audit success or failure, allow privilege debugging. + * Returns 0 for success, err for failure. + */ +#define PRIV_POLICY(cred, priv, all, err, reason) \ + priv_policy((cred), (priv), (all), (err), (reason)) + +/* + * Test privilege. Audit success only, no privilege debugging. + * Returns 1 for success, and 0 for failure. + */ +#define PRIV_POLICY_CHOICE(cred, priv, all) \ + priv_policy_choice((cred), (priv), (all)) + +/* + * Test privilege. No priv_debugging, no auditing. + * Returns 1 for success, and 0 for failure. + */ + +#define PRIV_POLICY_ONLY(cred, priv, all) \ + priv_policy_only((cred), (priv), (all)) + +#endif /* !_DTRACE_CDDL_H_ */ diff --git a/sys/cddl/dev/dtrace/dtrace_debug.c b/sys/cddl/dev/dtrace/dtrace_debug.c new file mode 100644 index 000000000000..5edd2fd603e1 --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_debug.c @@ -0,0 +1,560 @@ +/*- + * Copyright (C) 2008 John Birrell <jb@freebsd.org>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice(s), this list of conditions and the following disclaimer as + * the first lines of this file unmodified other than the possible + * addition of one or more copyright notices. + * 2. Redistributions in binary form must reproduce the above copyright + * notice(s), this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifdef DEBUG + +#include <machine/atomic.h> + +#define DTRACE_DEBUG_BUFR_SIZE (32 * 1024) + +struct dtrace_debug_data { + uintptr_t lock __aligned(CACHE_LINE_SIZE); + char bufr[DTRACE_DEBUG_BUFR_SIZE]; + char *first; + char *last; + char *next; +} dtrace_debug_data[MAXCPU]; + +static char dtrace_debug_bufr[DTRACE_DEBUG_BUFR_SIZE]; + +static void +dtrace_debug_lock(int cpu) +{ + uintptr_t tid; + + tid = (uintptr_t)curthread; + spinlock_enter(); + while (atomic_cmpset_acq_ptr(&dtrace_debug_data[cpu].lock, 0, tid) == 0) /* Loop until the lock is obtained. */ + ; +} + +static void +dtrace_debug_unlock(int cpu) +{ + atomic_store_rel_ptr(&dtrace_debug_data[cpu].lock, 0); + spinlock_exit(); +} + +static void +dtrace_debug_init(void *dummy) +{ + int i; + struct dtrace_debug_data *d; + + CPU_FOREACH(i) { + d = &dtrace_debug_data[i]; + + if (d->first == NULL) { + d->first = d->bufr; + d->next = d->bufr; + d->last = d->bufr + DTRACE_DEBUG_BUFR_SIZE - 1; + *(d->last) = '\0'; + } + } +} + +SYSINIT(dtrace_debug_init, SI_SUB_KDTRACE, SI_ORDER_ANY, dtrace_debug_init, NULL); +SYSINIT(dtrace_debug_smpinit, SI_SUB_SMP, SI_ORDER_ANY, dtrace_debug_init, NULL); + +static void +dtrace_debug_output(void) +{ + char *p; + int i; + struct dtrace_debug_data *d; + uintptr_t count; + + CPU_FOREACH(i) { + dtrace_debug_lock(i); + + d = &dtrace_debug_data[i]; + + count = 0; + + if (d->first < d->next) { + char *p1 = dtrace_debug_bufr; + + count = (uintptr_t) d->next - (uintptr_t) d->first; + + for (p = d->first; p < d->next; p++) + *p1++ = *p; + } else if (d->first > d->next) { + char *p1 = dtrace_debug_bufr; + + count = (uintptr_t) d->last - (uintptr_t) d->first; + + for (p = d->first; p < d->last; p++) + *p1++ = *p; + + count += (uintptr_t) d->next - (uintptr_t) d->bufr; + + for (p = d->bufr; p < d->next; p++) + *p1++ = *p; + } + + d->first = d->bufr; + d->next = d->bufr; + + dtrace_debug_unlock(i); + + if (count > 0) { + char *last = dtrace_debug_bufr + count; + + p = dtrace_debug_bufr; + + while (p < last) { + if (*p == '\0') { + p++; + continue; + } + + printf("%s", p); + + p += strlen(p); + } + } + } +} + +/* + * Functions below here are called from the probe context, so they can't call + * _any_ functions outside the dtrace module without running foul of the function + * boundary trace provider (fbt). The purpose of these functions is limited to + * buffering debug strings for output when the probe completes on the current CPU. + */ + +static __inline void +dtrace_debug__putc(int cpu, char c) +{ + struct dtrace_debug_data *d; + + d = &dtrace_debug_data[cpu]; + *d->next++ = c; + + if (d->next == d->last) + d->next = d->bufr; + + *(d->next) = '\0'; + + if (d->next == d->first) + d->first++; + + if (d->first == d->last) + d->first = d->bufr; +} + +static void __used +dtrace_debug_putc(char c) +{ + int cpu; + + cpu = curcpu; + dtrace_debug_lock(cpu); + + dtrace_debug__putc(cpu, c); + + dtrace_debug_unlock(cpu); +} + +static void __used +dtrace_debug_puts(const char *s) +{ + int cpu; + + cpu = curcpu; + dtrace_debug_lock(cpu); + + while (*s != '\0') + dtrace_debug__putc(cpu, *s++); + + dtrace_debug__putc(cpu, '\0'); + + dtrace_debug_unlock(cpu); +} + +/* + * Snaffled from sys/kern/subr_prf.c + * + * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse + * order; return an optional length and a pointer to the last character + * written in the buffer (i.e., the first character of the string). + * The buffer pointed to by `nbuf' must have length >= MAXNBUF. + */ +static char * +dtrace_debug_ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper) +{ + char *p, c; + + p = nbuf; + *p = '\0'; + do { + c = hex2ascii(num % base); + *++p = upper ? toupper(c) : c; + } while (num /= base); + if (lenp) + *lenp = p - nbuf; + return (p); +} + +#define MAXNBUF (sizeof(intmax_t) * NBBY + 1) + +static void +dtrace_debug_vprintf(int cpu, const char *fmt, va_list ap) +{ + char nbuf[MAXNBUF]; + const char *p, *percent, *q; + u_char *up; + int ch, n; + uintmax_t num; + int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot; + int cflag, hflag, jflag, tflag, zflag; + int dwidth, upper; + int radix = 10; + char padc; + int stop = 0, retval = 0; + + num = 0; + + if (fmt == NULL) + fmt = "(fmt null)\n"; + + for (;;) { + padc = ' '; + width = 0; + while ((ch = (u_char)*fmt++) != '%' || stop) { + if (ch == '\0') { + dtrace_debug__putc(cpu, '\0'); + return; + } + dtrace_debug__putc(cpu, ch); + } + percent = fmt - 1; + qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0; + sign = 0; dot = 0; dwidth = 0; upper = 0; + cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0; +reswitch: switch (ch = (u_char)*fmt++) { + case '.': + dot = 1; + goto reswitch; + case '#': + sharpflag = 1; + goto reswitch; + case '+': + sign = 1; + goto reswitch; + case '-': + ladjust = 1; + goto reswitch; + case '%': + dtrace_debug__putc(cpu, ch); + break; + case '*': + if (!dot) { + width = va_arg(ap, int); + if (width < 0) { + ladjust = !ladjust; + width = -width; + } + } else { + dwidth = va_arg(ap, int); + } + goto reswitch; + case '0': + if (!dot) { + padc = '0'; + goto reswitch; + } + case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + for (n = 0;; ++fmt) { + n = n * 10 + ch - '0'; + ch = *fmt; + if (ch < '0' || ch > '9') + break; + } + if (dot) + dwidth = n; + else + width = n; + goto reswitch; + case 'b': + num = (u_int)va_arg(ap, int); + p = va_arg(ap, char *); + for (q = dtrace_debug_ksprintn(nbuf, num, *p++, NULL, 0); *q;) + dtrace_debug__putc(cpu, *q--); + + if (num == 0) + break; + + for (tmp = 0; *p;) { + n = *p++; + if (num & (1 << (n - 1))) { + dtrace_debug__putc(cpu, tmp ? ',' : '<'); + for (; (n = *p) > ' '; ++p) + dtrace_debug__putc(cpu, n); + tmp = 1; + } else + for (; *p > ' '; ++p) + continue; + } + if (tmp) + dtrace_debug__putc(cpu, '>'); + break; + case 'c': + dtrace_debug__putc(cpu, va_arg(ap, int)); + break; + case 'D': + up = va_arg(ap, u_char *); + p = va_arg(ap, char *); + if (!width) + width = 16; + while(width--) { + dtrace_debug__putc(cpu, hex2ascii(*up >> 4)); + dtrace_debug__putc(cpu, hex2ascii(*up & 0x0f)); + up++; + if (width) + for (q=p;*q;q++) + dtrace_debug__putc(cpu, *q); + } + break; + case 'd': + case 'i': + base = 10; + sign = 1; + goto handle_sign; + case 'h': + if (hflag) { + hflag = 0; + cflag = 1; + } else + hflag = 1; + goto reswitch; + case 'j': + jflag = 1; + goto reswitch; + case 'l': + if (lflag) { + lflag = 0; + qflag = 1; + } else + lflag = 1; + goto reswitch; + case 'n': + if (jflag) + *(va_arg(ap, intmax_t *)) = retval; + else if (qflag) + *(va_arg(ap, quad_t *)) = retval; + else if (lflag) + *(va_arg(ap, long *)) = retval; + else if (zflag) + *(va_arg(ap, size_t *)) = retval; + else if (hflag) + *(va_arg(ap, short *)) = retval; + else if (cflag) + *(va_arg(ap, char *)) = retval; + else + *(va_arg(ap, int *)) = retval; + break; + case 'o': + base = 8; + goto handle_nosign; + case 'p': + base = 16; + sharpflag = (width == 0); + sign = 0; + num = (uintptr_t)va_arg(ap, void *); + goto number; + case 'q': + qflag = 1; + goto reswitch; + case 'r': + base = radix; + if (sign) + goto handle_sign; + goto handle_nosign; + case 's': + p = va_arg(ap, char *); + if (p == NULL) + p = "(null)"; + if (!dot) + n = strlen (p); + else + for (n = 0; n < dwidth && p[n]; n++) + continue; + + width -= n; + + if (!ladjust && width > 0) + while (width--) + dtrace_debug__putc(cpu, padc); + while (n--) + dtrace_debug__putc(cpu, *p++); + if (ladjust && width > 0) + while (width--) + dtrace_debug__putc(cpu, padc); + break; + case 't': + tflag = 1; + goto reswitch; + case 'u': + base = 10; + goto handle_nosign; + case 'X': + upper = 1; + case 'x': + base = 16; + goto handle_nosign; + case 'y': + base = 16; + sign = 1; + goto handle_sign; + case 'z': + zflag = 1; + goto reswitch; +handle_nosign: + sign = 0; + if (jflag) + num = va_arg(ap, uintmax_t); + else if (qflag) + num = va_arg(ap, u_quad_t); + else if (tflag) + num = va_arg(ap, ptrdiff_t); + else if (lflag) + num = va_arg(ap, u_long); + else if (zflag) + num = va_arg(ap, size_t); + else if (hflag) + num = (u_short)va_arg(ap, int); + else if (cflag) + num = (u_char)va_arg(ap, int); + else + num = va_arg(ap, u_int); + goto number; +handle_sign: + if (jflag) + num = va_arg(ap, intmax_t); + else if (qflag) + num = va_arg(ap, quad_t); + else if (tflag) + num = va_arg(ap, ptrdiff_t); + else if (lflag) + num = va_arg(ap, long); + else if (zflag) + num = va_arg(ap, size_t); + else if (hflag) + num = (short)va_arg(ap, int); + else if (cflag) + num = (char)va_arg(ap, int); + else + num = va_arg(ap, int); +number: + if (sign && (intmax_t)num < 0) { + neg = 1; + num = -(intmax_t)num; + } + p = dtrace_debug_ksprintn(nbuf, num, base, &tmp, upper); + if (sharpflag && num != 0) { + if (base == 8) + tmp++; + else if (base == 16) + tmp += 2; + } + if (neg) + tmp++; + + if (!ladjust && padc != '0' && width + && (width -= tmp) > 0) + while (width--) + dtrace_debug__putc(cpu, padc); + if (neg) + dtrace_debug__putc(cpu, '-'); + if (sharpflag && num != 0) { + if (base == 8) { + dtrace_debug__putc(cpu, '0'); + } else if (base == 16) { + dtrace_debug__putc(cpu, '0'); + dtrace_debug__putc(cpu, 'x'); + } + } + if (!ladjust && width && (width -= tmp) > 0) + while (width--) + dtrace_debug__putc(cpu, padc); + + while (*p) + dtrace_debug__putc(cpu, *p--); + + if (ladjust && width && (width -= tmp) > 0) + while (width--) + dtrace_debug__putc(cpu, padc); + + break; + default: + while (percent < fmt) + dtrace_debug__putc(cpu, *percent++); + /* + * Since we ignore an formatting argument it is no + * longer safe to obey the remaining formatting + * arguments as the arguments will no longer match + * the format specs. + */ + stop = 1; + break; + } + } + + dtrace_debug__putc(cpu, '\0'); +} + +void +dtrace_debug_printf(const char *fmt, ...) +{ + va_list ap; + int cpu; + + cpu = curcpu; + dtrace_debug_lock(cpu); + + va_start(ap, fmt); + + dtrace_debug_vprintf(cpu, fmt, ap); + + va_end(ap); + + dtrace_debug_unlock(cpu); +} + +#else + +#define dtrace_debug_output() +#define dtrace_debug_puts(_s) +#define dtrace_debug_printf(fmt, ...) + +#endif diff --git a/sys/cddl/dev/dtrace/dtrace_hacks.c b/sys/cddl/dev/dtrace/dtrace_hacks.c new file mode 100644 index 000000000000..3f8997382c75 --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_hacks.c @@ -0,0 +1,10 @@ +/* $FreeBSD$ */ +/* XXX Hacks.... */ + +dtrace_cacheid_t dtrace_predcache_id; + +boolean_t +priv_policy_only(const cred_t *a, int b, boolean_t c) +{ + return 0; +} diff --git a/sys/cddl/dev/dtrace/dtrace_ioctl.c b/sys/cddl/dev/dtrace/dtrace_ioctl.c new file mode 100644 index 000000000000..da36ee73897c --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_ioctl.c @@ -0,0 +1,848 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ + +static int dtrace_verbose_ioctl; +SYSCTL_INT(_debug_dtrace, OID_AUTO, verbose_ioctl, CTLFLAG_RW, + &dtrace_verbose_ioctl, 0, "log DTrace ioctls"); + +#define DTRACE_IOCTL_PRINTF(fmt, ...) if (dtrace_verbose_ioctl) printf(fmt, ## __VA_ARGS__ ) + +static int +dtrace_ioctl_helper(struct cdev *dev, u_long cmd, caddr_t addr, int flags, + struct thread *td) +{ + struct proc *p; + dof_helper_t *dhp; + dof_hdr_t *dof; + int rval; + + dhp = NULL; + dof = NULL; + rval = 0; + switch (cmd) { + case DTRACEHIOC_ADDDOF: + dhp = (dof_helper_t *)addr; + addr = (caddr_t)(uintptr_t)dhp->dofhp_dof; + p = curproc; + if (p->p_pid == dhp->dofhp_pid) { + dof = dtrace_dof_copyin((uintptr_t)addr, &rval); + } else { + p = pfind(dhp->dofhp_pid); + if (p == NULL) + return (EINVAL); + if (!P_SHOULDSTOP(p) || + (p->p_flag & (P_TRACED | P_WEXIT)) != P_TRACED || + p->p_pptr != curproc) { + PROC_UNLOCK(p); + return (EINVAL); + } + _PHOLD(p); + PROC_UNLOCK(p); + dof = dtrace_dof_copyin_proc(p, (uintptr_t)addr, &rval); + } + + if (dof == NULL) { + if (p != curproc) + PRELE(p); + break; + } + + mutex_enter(&dtrace_lock); + if ((rval = dtrace_helper_slurp(dof, dhp, p)) != -1) { + dhp->dofhp_gen = rval; + rval = 0; + } else { + rval = EINVAL; + } + mutex_exit(&dtrace_lock); + if (p != curproc) + PRELE(p); + break; + case DTRACEHIOC_REMOVE: + mutex_enter(&dtrace_lock); + rval = dtrace_helper_destroygen(NULL, *(int *)(uintptr_t)addr); + mutex_exit(&dtrace_lock); + break; + default: + rval = ENOTTY; + break; + } + return (rval); +} + +/* ARGSUSED */ +static int +dtrace_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, + int flags __unused, struct thread *td) +{ + dtrace_state_t *state; + devfs_get_cdevpriv((void **) &state); + + int error = 0; + if (state == NULL) + return (EINVAL); + + if (state->dts_anon) { + ASSERT(dtrace_anon.dta_state == NULL); + state = state->dts_anon; + } + + switch (cmd) { + case DTRACEIOC_AGGDESC: { + dtrace_aggdesc_t **paggdesc = (dtrace_aggdesc_t **) addr; + dtrace_aggdesc_t aggdesc; + dtrace_action_t *act; + dtrace_aggregation_t *agg; + int nrecs; + uint32_t offs; + dtrace_recdesc_t *lrec; + void *buf; + size_t size; + uintptr_t dest; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_AGGDESC\n",__func__,__LINE__); + + if (copyin((void *) *paggdesc, &aggdesc, sizeof (aggdesc)) != 0) + return (EFAULT); + + mutex_enter(&dtrace_lock); + + if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) { + mutex_exit(&dtrace_lock); + return (EINVAL); + } + + aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid; + + nrecs = aggdesc.dtagd_nrecs; + aggdesc.dtagd_nrecs = 0; + + offs = agg->dtag_base; + lrec = &agg->dtag_action.dta_rec; + aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs; + + for (act = agg->dtag_first; ; act = act->dta_next) { + ASSERT(act->dta_intuple || + DTRACEACT_ISAGG(act->dta_kind)); + + /* + * If this action has a record size of zero, it + * denotes an argument to the aggregating action. + * Because the presence of this record doesn't (or + * shouldn't) affect the way the data is interpreted, + * we don't copy it out to save user-level the + * confusion of dealing with a zero-length record. + */ + if (act->dta_rec.dtrd_size == 0) { + ASSERT(agg->dtag_hasarg); + continue; + } + + aggdesc.dtagd_nrecs++; + + if (act == &agg->dtag_action) + break; + } + + /* + * Now that we have the size, we need to allocate a temporary + * buffer in which to store the complete description. We need + * the temporary buffer to be able to drop dtrace_lock() + * across the copyout(), below. + */ + size = sizeof (dtrace_aggdesc_t) + + (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t)); + + buf = kmem_alloc(size, KM_SLEEP); + dest = (uintptr_t)buf; + + bcopy(&aggdesc, (void *)dest, sizeof (aggdesc)); + dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]); + + for (act = agg->dtag_first; ; act = act->dta_next) { + dtrace_recdesc_t rec = act->dta_rec; + + /* + * See the comment in the above loop for why we pass + * over zero-length records. + */ + if (rec.dtrd_size == 0) { + ASSERT(agg->dtag_hasarg); + continue; + } + + if (nrecs-- == 0) + break; + + rec.dtrd_offset -= offs; + bcopy(&rec, (void *)dest, sizeof (rec)); + dest += sizeof (dtrace_recdesc_t); + + if (act == &agg->dtag_action) + break; + } + + mutex_exit(&dtrace_lock); + + if (copyout(buf, (void *) *paggdesc, dest - (uintptr_t)buf) != 0) { + kmem_free(buf, size); + return (EFAULT); + } + + kmem_free(buf, size); + return (0); + } + case DTRACEIOC_AGGSNAP: + case DTRACEIOC_BUFSNAP: { + dtrace_bufdesc_t **pdesc = (dtrace_bufdesc_t **) addr; + dtrace_bufdesc_t desc; + caddr_t cached; + dtrace_buffer_t *buf; + + dtrace_debug_output(); + + if (copyin((void *) *pdesc, &desc, sizeof (desc)) != 0) + return (EFAULT); + + DTRACE_IOCTL_PRINTF("%s(%d): %s curcpu %d cpu %d\n", + __func__,__LINE__, + cmd == DTRACEIOC_AGGSNAP ? + "DTRACEIOC_AGGSNAP":"DTRACEIOC_BUFSNAP", + curcpu, desc.dtbd_cpu); + + if (desc.dtbd_cpu >= MAXCPU || CPU_ABSENT(desc.dtbd_cpu)) + return (ENOENT); + + mutex_enter(&dtrace_lock); + + if (cmd == DTRACEIOC_BUFSNAP) { + buf = &state->dts_buffer[desc.dtbd_cpu]; + } else { + buf = &state->dts_aggbuffer[desc.dtbd_cpu]; + } + + if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) { + size_t sz = buf->dtb_offset; + + if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) { + mutex_exit(&dtrace_lock); + return (EBUSY); + } + + /* + * If this buffer has already been consumed, we're + * going to indicate that there's nothing left here + * to consume. + */ + if (buf->dtb_flags & DTRACEBUF_CONSUMED) { + mutex_exit(&dtrace_lock); + + desc.dtbd_size = 0; + desc.dtbd_drops = 0; + desc.dtbd_errors = 0; + desc.dtbd_oldest = 0; + sz = sizeof (desc); + + if (copyout(&desc, (void *) *pdesc, sz) != 0) + return (EFAULT); + + return (0); + } + + /* + * If this is a ring buffer that has wrapped, we want + * to copy the whole thing out. + */ + if (buf->dtb_flags & DTRACEBUF_WRAPPED) { + dtrace_buffer_polish(buf); + sz = buf->dtb_size; + } + + if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) { + mutex_exit(&dtrace_lock); + return (EFAULT); + } + + desc.dtbd_size = sz; + desc.dtbd_drops = buf->dtb_drops; + desc.dtbd_errors = buf->dtb_errors; + desc.dtbd_oldest = buf->dtb_xamot_offset; + desc.dtbd_timestamp = dtrace_gethrtime(); + + mutex_exit(&dtrace_lock); + + if (copyout(&desc, (void *) *pdesc, sizeof (desc)) != 0) + return (EFAULT); + + buf->dtb_flags |= DTRACEBUF_CONSUMED; + + return (0); + } + + if (buf->dtb_tomax == NULL) { + ASSERT(buf->dtb_xamot == NULL); + mutex_exit(&dtrace_lock); + return (ENOENT); + } + + cached = buf->dtb_tomax; + ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); + + dtrace_xcall(desc.dtbd_cpu, + (dtrace_xcall_t)dtrace_buffer_switch, buf); + + state->dts_errors += buf->dtb_xamot_errors; + + /* + * If the buffers did not actually switch, then the cross call + * did not take place -- presumably because the given CPU is + * not in the ready set. If this is the case, we'll return + * ENOENT. + */ + if (buf->dtb_tomax == cached) { + ASSERT(buf->dtb_xamot != cached); + mutex_exit(&dtrace_lock); + return (ENOENT); + } + + ASSERT(cached == buf->dtb_xamot); + + DTRACE_IOCTL_PRINTF("%s(%d): copyout the buffer snapshot\n",__func__,__LINE__); + + /* + * We have our snapshot; now copy it out. + */ + if (copyout(buf->dtb_xamot, desc.dtbd_data, + buf->dtb_xamot_offset) != 0) { + mutex_exit(&dtrace_lock); + return (EFAULT); + } + + desc.dtbd_size = buf->dtb_xamot_offset; + desc.dtbd_drops = buf->dtb_xamot_drops; + desc.dtbd_errors = buf->dtb_xamot_errors; + desc.dtbd_oldest = 0; + desc.dtbd_timestamp = buf->dtb_switched; + + mutex_exit(&dtrace_lock); + + DTRACE_IOCTL_PRINTF("%s(%d): copyout buffer desc: size %zd drops %lu errors %lu\n",__func__,__LINE__,(size_t) desc.dtbd_size,(u_long) desc.dtbd_drops,(u_long) desc.dtbd_errors); + + /* + * Finally, copy out the buffer description. + */ + if (copyout(&desc, (void *) *pdesc, sizeof (desc)) != 0) + return (EFAULT); + + return (0); + } + case DTRACEIOC_CONF: { + dtrace_conf_t conf; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_CONF\n",__func__,__LINE__); + + bzero(&conf, sizeof (conf)); + conf.dtc_difversion = DIF_VERSION; + conf.dtc_difintregs = DIF_DIR_NREGS; + conf.dtc_diftupregs = DIF_DTR_NREGS; + conf.dtc_ctfmodel = CTF_MODEL_NATIVE; + + *((dtrace_conf_t *) addr) = conf; + + return (0); + } + case DTRACEIOC_DOFGET: { + dof_hdr_t **pdof = (dof_hdr_t **) addr; + dof_hdr_t hdr, *dof = *pdof; + int rval; + uint64_t len; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_DOFGET\n",__func__,__LINE__); + + if (copyin((void *)dof, &hdr, sizeof (hdr)) != 0) + return (EFAULT); + + mutex_enter(&dtrace_lock); + dof = dtrace_dof_create(state); + mutex_exit(&dtrace_lock); + + len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz); + rval = copyout(dof, (void *) *pdof, len); + dtrace_dof_destroy(dof); + + return (rval == 0 ? 0 : EFAULT); + } + case DTRACEIOC_ENABLE: { + dof_hdr_t *dof = NULL; + dtrace_enabling_t *enab = NULL; + dtrace_vstate_t *vstate; + int err = 0; + int rval; + dtrace_enable_io_t *p = (dtrace_enable_io_t *) addr; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_ENABLE\n",__func__,__LINE__); + + /* + * If a NULL argument has been passed, we take this as our + * cue to reevaluate our enablings. + */ + if (p->dof == NULL) { + dtrace_enabling_matchall(); + + return (0); + } + + if ((dof = dtrace_dof_copyin((uintptr_t) p->dof, &rval)) == NULL) + return (EINVAL); + + mutex_enter(&cpu_lock); + mutex_enter(&dtrace_lock); + vstate = &state->dts_vstate; + + if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) { + mutex_exit(&dtrace_lock); + mutex_exit(&cpu_lock); + dtrace_dof_destroy(dof); + return (EBUSY); + } + + if (dtrace_dof_slurp(dof, vstate, td->td_ucred, &enab, 0, 0, + B_TRUE) != 0) { + mutex_exit(&dtrace_lock); + mutex_exit(&cpu_lock); + dtrace_dof_destroy(dof); + return (EINVAL); + } + + if ((rval = dtrace_dof_options(dof, state)) != 0) { + dtrace_enabling_destroy(enab); + mutex_exit(&dtrace_lock); + mutex_exit(&cpu_lock); + dtrace_dof_destroy(dof); + return (rval); + } + + if ((err = dtrace_enabling_match(enab, &p->n_matched)) == 0) { + err = dtrace_enabling_retain(enab); + } else { + dtrace_enabling_destroy(enab); + } + + mutex_exit(&cpu_lock); + mutex_exit(&dtrace_lock); + dtrace_dof_destroy(dof); + + return (err); + } + case DTRACEIOC_EPROBE: { + dtrace_eprobedesc_t **pepdesc = (dtrace_eprobedesc_t **) addr; + dtrace_eprobedesc_t epdesc; + dtrace_ecb_t *ecb; + dtrace_action_t *act; + void *buf; + size_t size; + uintptr_t dest; + int nrecs; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_EPROBE\n",__func__,__LINE__); + + if (copyin((void *)*pepdesc, &epdesc, sizeof (epdesc)) != 0) + return (EFAULT); + + mutex_enter(&dtrace_lock); + + if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) { + mutex_exit(&dtrace_lock); + return (EINVAL); + } + + if (ecb->dte_probe == NULL) { + mutex_exit(&dtrace_lock); + return (EINVAL); + } + + epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id; + epdesc.dtepd_uarg = ecb->dte_uarg; + epdesc.dtepd_size = ecb->dte_size; + + nrecs = epdesc.dtepd_nrecs; + epdesc.dtepd_nrecs = 0; + for (act = ecb->dte_action; act != NULL; act = act->dta_next) { + if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple) + continue; + + epdesc.dtepd_nrecs++; + } + + /* + * Now that we have the size, we need to allocate a temporary + * buffer in which to store the complete description. We need + * the temporary buffer to be able to drop dtrace_lock() + * across the copyout(), below. + */ + size = sizeof (dtrace_eprobedesc_t) + + (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t)); + + buf = kmem_alloc(size, KM_SLEEP); + dest = (uintptr_t)buf; + + bcopy(&epdesc, (void *)dest, sizeof (epdesc)); + dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]); + + for (act = ecb->dte_action; act != NULL; act = act->dta_next) { + if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple) + continue; + + if (nrecs-- == 0) + break; + + bcopy(&act->dta_rec, (void *)dest, + sizeof (dtrace_recdesc_t)); + dest += sizeof (dtrace_recdesc_t); + } + + mutex_exit(&dtrace_lock); + + if (copyout(buf, (void *) *pepdesc, dest - (uintptr_t)buf) != 0) { + kmem_free(buf, size); + return (EFAULT); + } + + kmem_free(buf, size); + return (0); + } + case DTRACEIOC_FORMAT: { + dtrace_fmtdesc_t *fmt = (dtrace_fmtdesc_t *) addr; + char *str; + int len; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_FORMAT\n",__func__,__LINE__); + + mutex_enter(&dtrace_lock); + + if (fmt->dtfd_format == 0 || + fmt->dtfd_format > state->dts_nformats) { + mutex_exit(&dtrace_lock); + return (EINVAL); + } + + /* + * Format strings are allocated contiguously and they are + * never freed; if a format index is less than the number + * of formats, we can assert that the format map is non-NULL + * and that the format for the specified index is non-NULL. + */ + ASSERT(state->dts_formats != NULL); + str = state->dts_formats[fmt->dtfd_format - 1]; + ASSERT(str != NULL); + + len = strlen(str) + 1; + + if (len > fmt->dtfd_length) { + fmt->dtfd_length = len; + } else { + if (copyout(str, fmt->dtfd_string, len) != 0) { + mutex_exit(&dtrace_lock); + return (EINVAL); + } + } + + mutex_exit(&dtrace_lock); + return (0); + } + case DTRACEIOC_GO: { + int rval; + processorid_t *cpuid = (processorid_t *) addr; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_GO\n",__func__,__LINE__); + + rval = dtrace_state_go(state, cpuid); + + return (rval); + } + case DTRACEIOC_PROBEARG: { + dtrace_argdesc_t *desc = (dtrace_argdesc_t *) addr; + dtrace_probe_t *probe; + dtrace_provider_t *prov; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_PROBEARG\n",__func__,__LINE__); + + if (desc->dtargd_id == DTRACE_IDNONE) + return (EINVAL); + + if (desc->dtargd_ndx == DTRACE_ARGNONE) + return (EINVAL); + + mutex_enter(&dtrace_provider_lock); +#ifdef illumos + mutex_enter(&mod_lock); +#endif + mutex_enter(&dtrace_lock); + + if (desc->dtargd_id > dtrace_nprobes) { + mutex_exit(&dtrace_lock); +#ifdef illumos + mutex_exit(&mod_lock); +#endif + mutex_exit(&dtrace_provider_lock); + return (EINVAL); + } + + if ((probe = dtrace_probes[desc->dtargd_id - 1]) == NULL) { + mutex_exit(&dtrace_lock); +#ifdef illumos + mutex_exit(&mod_lock); +#endif + mutex_exit(&dtrace_provider_lock); + return (EINVAL); + } + + mutex_exit(&dtrace_lock); + + prov = probe->dtpr_provider; + + if (prov->dtpv_pops.dtps_getargdesc == NULL) { + /* + * There isn't any typed information for this probe. + * Set the argument number to DTRACE_ARGNONE. + */ + desc->dtargd_ndx = DTRACE_ARGNONE; + } else { + desc->dtargd_native[0] = '\0'; + desc->dtargd_xlate[0] = '\0'; + desc->dtargd_mapping = desc->dtargd_ndx; + + prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg, + probe->dtpr_id, probe->dtpr_arg, desc); + } + +#ifdef illumos + mutex_exit(&mod_lock); +#endif + mutex_exit(&dtrace_provider_lock); + + return (0); + } + case DTRACEIOC_PROBEMATCH: + case DTRACEIOC_PROBES: { + dtrace_probedesc_t *p_desc = (dtrace_probedesc_t *) addr; + dtrace_probe_t *probe = NULL; + dtrace_probekey_t pkey; + dtrace_id_t i; + int m = 0; + uint32_t priv = 0; + uid_t uid = 0; + zoneid_t zoneid = 0; + + DTRACE_IOCTL_PRINTF("%s(%d): %s\n",__func__,__LINE__, + cmd == DTRACEIOC_PROBEMATCH ? + "DTRACEIOC_PROBEMATCH":"DTRACEIOC_PROBES"); + + p_desc->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; + p_desc->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; + p_desc->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; + p_desc->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; + + /* + * Before we attempt to match this probe, we want to give + * all providers the opportunity to provide it. + */ + if (p_desc->dtpd_id == DTRACE_IDNONE) { + mutex_enter(&dtrace_provider_lock); + dtrace_probe_provide(p_desc, NULL); + mutex_exit(&dtrace_provider_lock); + p_desc->dtpd_id++; + } + + if (cmd == DTRACEIOC_PROBEMATCH) { + dtrace_probekey(p_desc, &pkey); + pkey.dtpk_id = DTRACE_IDNONE; + } + + dtrace_cred2priv(td->td_ucred, &priv, &uid, &zoneid); + + mutex_enter(&dtrace_lock); + + if (cmd == DTRACEIOC_PROBEMATCH) { + for (i = p_desc->dtpd_id; i <= dtrace_nprobes; i++) { + if ((probe = dtrace_probes[i - 1]) != NULL && + (m = dtrace_match_probe(probe, &pkey, + priv, uid, zoneid)) != 0) + break; + } + + if (m < 0) { + mutex_exit(&dtrace_lock); + return (EINVAL); + } + + } else { + for (i = p_desc->dtpd_id; i <= dtrace_nprobes; i++) { + if ((probe = dtrace_probes[i - 1]) != NULL && + dtrace_match_priv(probe, priv, uid, zoneid)) + break; + } + } + + if (probe == NULL) { + mutex_exit(&dtrace_lock); + return (ESRCH); + } + + dtrace_probe_description(probe, p_desc); + mutex_exit(&dtrace_lock); + + return (0); + } + case DTRACEIOC_PROVIDER: { + dtrace_providerdesc_t *pvd = (dtrace_providerdesc_t *) addr; + dtrace_provider_t *pvp; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_PROVIDER\n",__func__,__LINE__); + + pvd->dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0'; + mutex_enter(&dtrace_provider_lock); + + for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) { + if (strcmp(pvp->dtpv_name, pvd->dtvd_name) == 0) + break; + } + + mutex_exit(&dtrace_provider_lock); + + if (pvp == NULL) + return (ESRCH); + + bcopy(&pvp->dtpv_priv, &pvd->dtvd_priv, sizeof (dtrace_ppriv_t)); + bcopy(&pvp->dtpv_attr, &pvd->dtvd_attr, sizeof (dtrace_pattr_t)); + + return (0); + } + case DTRACEIOC_REPLICATE: { + dtrace_repldesc_t *desc = (dtrace_repldesc_t *) addr; + dtrace_probedesc_t *match = &desc->dtrpd_match; + dtrace_probedesc_t *create = &desc->dtrpd_create; + int err; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_REPLICATE\n",__func__,__LINE__); + + match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; + match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; + match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; + match->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; + + create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; + create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; + create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; + create->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; + + mutex_enter(&dtrace_lock); + err = dtrace_enabling_replicate(state, match, create); + mutex_exit(&dtrace_lock); + + return (err); + } + case DTRACEIOC_STATUS: { + dtrace_status_t *stat = (dtrace_status_t *) addr; + dtrace_dstate_t *dstate; + int i, j; + uint64_t nerrs; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_STATUS\n",__func__,__LINE__); + + /* + * See the comment in dtrace_state_deadman() for the reason + * for setting dts_laststatus to INT64_MAX before setting + * it to the correct value. + */ + state->dts_laststatus = INT64_MAX; + dtrace_membar_producer(); + state->dts_laststatus = dtrace_gethrtime(); + + bzero(stat, sizeof (*stat)); + + mutex_enter(&dtrace_lock); + + if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) { + mutex_exit(&dtrace_lock); + return (ENOENT); + } + + if (state->dts_activity == DTRACE_ACTIVITY_DRAINING) + stat->dtst_exiting = 1; + + nerrs = state->dts_errors; + dstate = &state->dts_vstate.dtvs_dynvars; + + CPU_FOREACH(i) { + dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i]; + + stat->dtst_dyndrops += dcpu->dtdsc_drops; + stat->dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops; + stat->dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops; + + if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL) + stat->dtst_filled++; + + nerrs += state->dts_buffer[i].dtb_errors; + + for (j = 0; j < state->dts_nspeculations; j++) { + dtrace_speculation_t *spec; + dtrace_buffer_t *buf; + + spec = &state->dts_speculations[j]; + buf = &spec->dtsp_buffer[i]; + stat->dtst_specdrops += buf->dtb_xamot_drops; + } + } + + stat->dtst_specdrops_busy = state->dts_speculations_busy; + stat->dtst_specdrops_unavail = state->dts_speculations_unavail; + stat->dtst_stkstroverflows = state->dts_stkstroverflows; + stat->dtst_dblerrors = state->dts_dblerrors; + stat->dtst_killed = + (state->dts_activity == DTRACE_ACTIVITY_KILLED); + stat->dtst_errors = nerrs; + + mutex_exit(&dtrace_lock); + + return (0); + } + case DTRACEIOC_STOP: { + int rval; + processorid_t *cpuid = (processorid_t *) addr; + + DTRACE_IOCTL_PRINTF("%s(%d): DTRACEIOC_STOP\n",__func__,__LINE__); + + mutex_enter(&dtrace_lock); + rval = dtrace_state_stop(state, cpuid); + mutex_exit(&dtrace_lock); + + return (rval); + } + default: + error = ENOTTY; + } + return (error); +} diff --git a/sys/cddl/dev/dtrace/dtrace_load.c b/sys/cddl/dev/dtrace/dtrace_load.c new file mode 100644 index 000000000000..029af60d1cd8 --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_load.c @@ -0,0 +1,170 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ + +#ifndef EARLY_AP_STARTUP +static void +dtrace_ap_start(void *dummy) +{ + int i; + + mutex_enter(&cpu_lock); + + /* Setup the rest of the CPUs. */ + CPU_FOREACH(i) { + if (i == 0) + continue; + + (void) dtrace_cpu_setup(CPU_CONFIG, i); + } + + mutex_exit(&cpu_lock); +} + +SYSINIT(dtrace_ap_start, SI_SUB_SMP, SI_ORDER_ANY, dtrace_ap_start, NULL); +#endif + +static void +dtrace_load(void *dummy) +{ + dtrace_provider_id_t id; +#ifdef EARLY_AP_STARTUP + int i; +#endif + +#ifndef illumos + /* + * DTrace uses negative logic for the destructive mode switch, so it + * is required to translate from the sysctl which uses positive logic. + */ + if (dtrace_allow_destructive) + dtrace_destructive_disallow = 0; + else + dtrace_destructive_disallow = 1; +#endif + + /* Hook into the trap handler. */ + dtrace_trap_func = dtrace_trap; + + /* Hang our hook for thread switches. */ + dtrace_vtime_switch_func = dtrace_vtime_switch; + + /* Hang our hook for exceptions. */ + dtrace_invop_init(); + + dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri, 0, 0, 0); + + dtrace_arena = new_unrhdr(1, INT_MAX, &dtrace_unr_mtx); + + /* Register callbacks for linker file load and unload events. */ + dtrace_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, + dtrace_kld_load, NULL, EVENTHANDLER_PRI_ANY); + dtrace_kld_unload_try_tag = EVENTHANDLER_REGISTER(kld_unload_try, + dtrace_kld_unload_try, NULL, EVENTHANDLER_PRI_ANY); + + /* + * Initialise the mutexes without 'witness' because the dtrace + * code is mostly written to wait for memory. To have the + * witness code change a malloc() from M_WAITOK to M_NOWAIT + * because a lock is held would surely create a panic in a + * low memory situation. And that low memory situation might be + * the very problem we are trying to trace. + */ + mutex_init(&dtrace_lock,"dtrace probe state", MUTEX_DEFAULT, NULL); + mutex_init(&dtrace_provider_lock,"dtrace provider state", MUTEX_DEFAULT, NULL); + mutex_init(&dtrace_meta_lock,"dtrace meta-provider state", MUTEX_DEFAULT, NULL); +#ifdef DEBUG + mutex_init(&dtrace_errlock,"dtrace error lock", MUTEX_DEFAULT, NULL); +#endif + + mutex_enter(&cpu_lock); + mutex_enter(&dtrace_provider_lock); + mutex_enter(&dtrace_lock); + + dtrace_state_cache = kmem_cache_create("dtrace_state_cache", + sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN, + NULL, NULL, NULL, NULL, NULL, 0); + + ASSERT(MUTEX_HELD(&cpu_lock)); + dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod), + offsetof(dtrace_probe_t, dtpr_nextmod), + offsetof(dtrace_probe_t, dtpr_prevmod)); + + dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func), + offsetof(dtrace_probe_t, dtpr_nextfunc), + offsetof(dtrace_probe_t, dtpr_prevfunc)); + + dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name), + offsetof(dtrace_probe_t, dtpr_nextname), + offsetof(dtrace_probe_t, dtpr_prevname)); + + if (dtrace_retain_max < 1) { + cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; " + "setting to 1", dtrace_retain_max); + dtrace_retain_max = 1; + } + + /* + * Now discover our toxic ranges. + */ + dtrace_toxic_ranges(dtrace_toxrange_add); + + /* + * Before we register ourselves as a provider to our own framework, + * we would like to assert that dtrace_provider is NULL -- but that's + * not true if we were loaded as a dependency of a DTrace provider. + * Once we've registered, we can assert that dtrace_provider is our + * pseudo provider. + */ + (void) dtrace_register("dtrace", &dtrace_provider_attr, + DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id); + + ASSERT(dtrace_provider != NULL); + ASSERT((dtrace_provider_id_t)dtrace_provider == id); + + dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "BEGIN", 0, NULL); + dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "END", 0, NULL); + dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "ERROR", 1, NULL); + + mutex_exit(&dtrace_lock); + mutex_exit(&dtrace_provider_lock); + +#ifdef EARLY_AP_STARTUP + CPU_FOREACH(i) { + (void) dtrace_cpu_setup(CPU_CONFIG, i); + } +#else + /* Setup the boot CPU */ + (void) dtrace_cpu_setup(CPU_CONFIG, 0); +#endif + + mutex_exit(&cpu_lock); + + dtrace_dev = make_dev(&dtrace_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, + "dtrace/dtrace"); + helper_dev = make_dev(&helper_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660, + "dtrace/helper"); +} diff --git a/sys/cddl/dev/dtrace/dtrace_modevent.c b/sys/cddl/dev/dtrace/dtrace_modevent.c new file mode 100644 index 000000000000..8d318532dee9 --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_modevent.c @@ -0,0 +1,47 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ + +/* ARGSUSED */ +static int +dtrace_modevent(module_t mod __unused, int type, void *data __unused) +{ + int error = 0; + + switch (type) { + case MOD_LOAD: + break; + + case MOD_UNLOAD: + break; + + case MOD_SHUTDOWN: + break; + + default: + error = EOPNOTSUPP; + break; + + } + return (error); +} diff --git a/sys/cddl/dev/dtrace/dtrace_sysctl.c b/sys/cddl/dev/dtrace/dtrace_sysctl.c new file mode 100644 index 000000000000..87fa705e2d96 --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_sysctl.c @@ -0,0 +1,97 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ + +/* Report registered DTrace providers. */ +static int +sysctl_dtrace_providers(SYSCTL_HANDLER_ARGS) +{ + char *p_name = NULL; + dtrace_provider_t + *prov = dtrace_provider; + int error = 0; + size_t len = 0; + + mutex_enter(&dtrace_provider_lock); + mutex_enter(&dtrace_lock); + + /* Compute the length of the space-separated provider name string. */ + while (prov != NULL) { + len += strlen(prov->dtpv_name) + 1; + prov = prov->dtpv_next; + } + + if ((p_name = kmem_alloc(len, KM_SLEEP)) == NULL) + error = ENOMEM; + else { + /* Start with an empty string. */ + *p_name = '\0'; + + /* Point to the first provider again. */ + prov = dtrace_provider; + + /* Loop through the providers, appending the names. */ + while (prov != NULL) { + if (prov != dtrace_provider) + (void) strlcat(p_name, " ", len); + + (void) strlcat(p_name, prov->dtpv_name, len); + + prov = prov->dtpv_next; + } + } + + mutex_exit(&dtrace_lock); + mutex_exit(&dtrace_provider_lock); + + if (p_name != NULL) { + error = sysctl_handle_string(oidp, p_name, len, req); + + kmem_free(p_name, 0); + } + + return (error); +} + +SYSCTL_NODE(_debug, OID_AUTO, dtrace, CTLFLAG_RD, 0, "DTrace debug parameters"); + +SYSCTL_PROC(_debug_dtrace, OID_AUTO, providers, CTLTYPE_STRING | CTLFLAG_RD, + 0, 0, sysctl_dtrace_providers, "A", "available DTrace providers"); + +SYSCTL_NODE(_kern, OID_AUTO, dtrace, CTLFLAG_RD, 0, "DTrace parameters"); + +SYSCTL_INT(_kern_dtrace, OID_AUTO, err_verbose, CTLFLAG_RW, + &dtrace_err_verbose, 0, + "print DIF and DOF validation errors to the message buffer"); + +SYSCTL_INT(_kern_dtrace, OID_AUTO, memstr_max, CTLFLAG_RW, &dtrace_memstr_max, + 0, "largest allowed argument to memstr(), 0 indicates no limit"); + +SYSCTL_QUAD(_kern_dtrace, OID_AUTO, dof_maxsize, CTLFLAG_RW, + &dtrace_dof_maxsize, 0, "largest allowed DOF table"); + +SYSCTL_QUAD(_kern_dtrace, OID_AUTO, helper_actions_max, CTLFLAG_RW, + &dtrace_helper_actions_max, 0, "maximum number of allowed helper actions"); + +SYSCTL_INT(_security_bsd, OID_AUTO, allow_destructive_dtrace, CTLFLAG_RDTUN, + &dtrace_allow_destructive, 1, "Allow destructive mode DTrace scripts"); diff --git a/sys/cddl/dev/dtrace/dtrace_test.c b/sys/cddl/dev/dtrace/dtrace_test.c new file mode 100644 index 000000000000..9caf2f00f32c --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_test.c @@ -0,0 +1,108 @@ +/*- + * Copyright 2008 John Birrell <jb@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ +#include <sys/cdefs.h> +#include <sys/types.h> +#include <sys/param.h> +#include <sys/systm.h> + +#include <sys/conf.h> +#include <sys/kernel.h> +#include <sys/module.h> +#include <sys/sdt.h> +#include <sys/sysctl.h> +#include <sys/vnode.h> + +SDT_PROVIDER_DEFINE(test); + +SDT_PROBE_DEFINE7(test, , , sdttest, "int", "int", "int", "int", "int", + "int", "int"); + +/* + * These are variables that the DTrace test suite references in the + * Solaris kernel. We define them here so that the tests function + * unaltered. + */ +int kmem_flags; + +typedef struct vnode vnode_t; +vnode_t dummy; +vnode_t *rootvp = &dummy; + +/* + * Test SDT probes with more than 5 arguments. On amd64, such probes require + * special handling since only the first 5 arguments will be passed to + * dtrace_probe() in registers; the rest must be fetched off the stack. + */ +static int +dtrace_test_sdttest(SYSCTL_HANDLER_ARGS) +{ + int val, error; + + val = 0; + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || req->newptr == NULL) + return (error); + else if (val == 0) + return (0); + + SDT_PROBE7(test, , , sdttest, 1, 2, 3, 4, 5, 6, 7); + + return (error); +} + +static SYSCTL_NODE(_debug, OID_AUTO, dtracetest, CTLFLAG_RD, 0, ""); + +SYSCTL_PROC(_debug_dtracetest, OID_AUTO, sdttest, CTLTYPE_INT | CTLFLAG_RW, + NULL, 0, dtrace_test_sdttest, "I", "Trigger the SDT test probe"); + +static int +dtrace_test_modevent(module_t mod, int type, void *data) +{ + int error = 0; + + switch (type) { + case MOD_LOAD: + break; + + case MOD_UNLOAD: + break; + + case MOD_SHUTDOWN: + break; + + default: + error = EOPNOTSUPP; + break; + + } + return (error); +} + +DEV_MODULE(dtrace_test, dtrace_test_modevent, NULL); +MODULE_VERSION(dtrace_test, 1); +MODULE_DEPEND(dtrace_test, dtraceall, 1, 1, 1); diff --git a/sys/cddl/dev/dtrace/dtrace_unload.c b/sys/cddl/dev/dtrace/dtrace_unload.c new file mode 100644 index 000000000000..b0d22e6527f3 --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_unload.c @@ -0,0 +1,126 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ + +static int +dtrace_unload() +{ + dtrace_state_t *state; + int error = 0; + + destroy_dev(dtrace_dev); + destroy_dev(helper_dev); + + mutex_enter(&dtrace_provider_lock); + mutex_enter(&dtrace_lock); + mutex_enter(&cpu_lock); + + ASSERT(dtrace_opens == 0); + + if (dtrace_helpers > 0) { + mutex_exit(&cpu_lock); + mutex_exit(&dtrace_lock); + mutex_exit(&dtrace_provider_lock); + return (EBUSY); + } + + if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) { + mutex_exit(&cpu_lock); + mutex_exit(&dtrace_lock); + mutex_exit(&dtrace_provider_lock); + return (EBUSY); + } + + dtrace_provider = NULL; + EVENTHANDLER_DEREGISTER(kld_load, dtrace_kld_load_tag); + EVENTHANDLER_DEREGISTER(kld_unload_try, dtrace_kld_unload_try_tag); + + if ((state = dtrace_anon_grab()) != NULL) { + /* + * If there were ECBs on this state, the provider should + * have not been allowed to detach; assert that there is + * none. + */ + ASSERT(state->dts_necbs == 0); + dtrace_state_destroy(state); + } + + bzero(&dtrace_anon, sizeof (dtrace_anon_t)); + + mutex_exit(&cpu_lock); + + if (dtrace_probes != NULL) { + kmem_free(dtrace_probes, 0); + dtrace_probes = NULL; + dtrace_nprobes = 0; + } + + dtrace_hash_destroy(dtrace_bymod); + dtrace_hash_destroy(dtrace_byfunc); + dtrace_hash_destroy(dtrace_byname); + dtrace_bymod = NULL; + dtrace_byfunc = NULL; + dtrace_byname = NULL; + + kmem_cache_destroy(dtrace_state_cache); + + delete_unrhdr(dtrace_arena); + + if (dtrace_toxrange != NULL) { + kmem_free(dtrace_toxrange, 0); + dtrace_toxrange = NULL; + dtrace_toxranges = 0; + dtrace_toxranges_max = 0; + } + + ASSERT(dtrace_vtime_references == 0); + ASSERT(dtrace_opens == 0); + ASSERT(dtrace_retained == NULL); + + mutex_exit(&dtrace_lock); + mutex_exit(&dtrace_provider_lock); + + mutex_destroy(&dtrace_meta_lock); + mutex_destroy(&dtrace_provider_lock); + mutex_destroy(&dtrace_lock); +#ifdef DEBUG + mutex_destroy(&dtrace_errlock); +#endif + + taskq_destroy(dtrace_taskq); + + /* Reset our hook for exceptions. */ + dtrace_invop_uninit(); + + /* + * Reset our hook for thread switches, but ensure that vtime isn't + * active first. + */ + dtrace_vtime_active = 0; + dtrace_vtime_switch_func = NULL; + + /* Unhook from the trap handler. */ + dtrace_trap_func = NULL; + + return (error); +} diff --git a/sys/cddl/dev/dtrace/dtrace_vtime.c b/sys/cddl/dev/dtrace/dtrace_vtime.c new file mode 100644 index 000000000000..a3fa7f77287a --- /dev/null +++ b/sys/cddl/dev/dtrace/dtrace_vtime.c @@ -0,0 +1,101 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ + +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +void +dtrace_vtime_enable(void) +{ + dtrace_vtime_state_t state, nstate = 0; + + do { + state = dtrace_vtime_active; + + switch (state) { + case DTRACE_VTIME_INACTIVE: + nstate = DTRACE_VTIME_ACTIVE; + break; + + case DTRACE_VTIME_INACTIVE_TNF: + nstate = DTRACE_VTIME_ACTIVE_TNF; + break; + + case DTRACE_VTIME_ACTIVE: + case DTRACE_VTIME_ACTIVE_TNF: + panic("DTrace virtual time already enabled"); + /*NOTREACHED*/ + } + + } while (dtrace_cas32((uint32_t *)&dtrace_vtime_active, + state, nstate) != state); +} + +void +dtrace_vtime_disable(void) +{ + dtrace_vtime_state_t state, nstate = 0; + + do { + state = dtrace_vtime_active; + + switch (state) { + case DTRACE_VTIME_ACTIVE: + nstate = DTRACE_VTIME_INACTIVE; + break; + + case DTRACE_VTIME_ACTIVE_TNF: + nstate = DTRACE_VTIME_INACTIVE_TNF; + break; + + case DTRACE_VTIME_INACTIVE: + case DTRACE_VTIME_INACTIVE_TNF: + panic("DTrace virtual time already disabled"); + /*NOTREACHED*/ + } + + } while (dtrace_cas32((uint32_t *)&dtrace_vtime_active, + state, nstate) != state); +} + +void +dtrace_vtime_switch(kthread_t *next) +{ + dtrace_icookie_t cookie; + hrtime_t ts; + + cookie = dtrace_interrupt_disable(); + ts = dtrace_gethrtime(); + + if (curthread->t_dtrace_start != 0) { + curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start; + curthread->t_dtrace_start = 0; + } + + if (next != NULL) + next->t_dtrace_start = ts; + + dtrace_interrupt_enable(cookie); +} diff --git a/sys/cddl/dev/dtrace/i386/dtrace_asm.S b/sys/cddl/dev/dtrace/i386/dtrace_asm.S new file mode 100644 index 000000000000..116db1f84be1 --- /dev/null +++ b/sys/cddl/dev/dtrace/i386/dtrace_asm.S @@ -0,0 +1,355 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#define _ASM + +#include <machine/asmacros.h> +#include <sys/cpuvar_defs.h> +#include <sys/dtrace.h> + +#include "assym.inc" + + ENTRY(dtrace_invop_start) + + pushl %eax /* push %eax -- may be return value */ + pushl %esp /* push stack pointer */ + subl $8, (%esp) /* skip first arg and segment regs */ + pushl 40(%esp) /* push calling EIP */ + + /* + * Call dtrace_invop to let it check if the exception was + * a fbt one. The return value in %eax will tell us what + * dtrace_invop wants us to do. + */ + call dtrace_invop + ALTENTRY(dtrace_invop_callsite) + addl $12, %esp + cmpl $DTRACE_INVOP_PUSHL_EBP, %eax + je invop_push + cmpl $DTRACE_INVOP_POPL_EBP, %eax + je invop_pop + cmpl $DTRACE_INVOP_LEAVE, %eax + je invop_leave + cmpl $DTRACE_INVOP_NOP, %eax + je invop_nop + + /* When all else fails handle the trap in the usual way. */ + jmpl *dtrace_invop_calltrap_addr + +invop_push: + /* + * We must emulate a "pushl %ebp". To do this, we pull the stack + * down 4 bytes, and then store the base pointer. + */ + popal + subl $4, %esp /* make room for %ebp */ + pushl %eax /* push temp */ + movl 8(%esp), %eax /* load calling EIP */ + incl %eax /* increment over LOCK prefix */ + movl %eax, 4(%esp) /* store calling EIP */ + movl 12(%esp), %eax /* load calling CS */ + movl %eax, 8(%esp) /* store calling CS */ + movl 16(%esp), %eax /* load calling EFLAGS */ + movl %eax, 12(%esp) /* store calling EFLAGS */ + movl %ebp, 16(%esp) /* push %ebp */ + popl %eax /* pop off temp */ + iret /* Return from interrupt. */ +invop_pop: + /* + * We must emulate a "popl %ebp". To do this, we do the opposite of + * the above: we remove the %ebp from the stack, and squeeze up the + * saved state from the trap. + */ + popal + pushl %eax /* push temp */ + movl 16(%esp), %ebp /* pop %ebp */ + movl 12(%esp), %eax /* load calling EFLAGS */ + movl %eax, 16(%esp) /* store calling EFLAGS */ + movl 8(%esp), %eax /* load calling CS */ + movl %eax, 12(%esp) /* store calling CS */ + movl 4(%esp), %eax /* load calling EIP */ + incl %eax /* increment over LOCK prefix */ + movl %eax, 8(%esp) /* store calling EIP */ + popl %eax /* pop off temp */ + addl $4, %esp /* adjust stack pointer */ + iret /* Return from interrupt. */ +invop_leave: + /* + * We must emulate a "leave", which is the same as a "movl %ebp, %esp" + * followed by a "popl %ebp". This looks similar to the above, but + * requires two temporaries: one for the new base pointer, and one + * for the staging register. + */ + popa + pushl %eax /* push temp */ + pushl %ebx /* push temp */ + movl %ebp, %ebx /* set temp to old %ebp */ + movl (%ebx), %ebp /* pop %ebp */ + movl 16(%esp), %eax /* load calling EFLAGS */ + movl %eax, (%ebx) /* store calling EFLAGS */ + movl 12(%esp), %eax /* load calling CS */ + movl %eax, -4(%ebx) /* store calling CS */ + movl 8(%esp), %eax /* load calling EIP */ + incl %eax /* increment over LOCK prefix */ + movl %eax, -8(%ebx) /* store calling EIP */ + subl $8, %ebx /* adjust for three pushes, one pop */ + movl %ebx, 8(%esp) /* temporarily store new %esp */ + popl %ebx /* pop off temp */ + popl %eax /* pop off temp */ + movl (%esp), %esp /* set stack pointer */ + iret /* return from interrupt */ +invop_nop: + /* + * We must emulate a "nop". This is obviously not hard: we need only + * advance the %eip by one. + */ + popa + incl (%esp) + iret /* return from interrupt */ + + END(dtrace_invop_start) + +/* +void dtrace_invop_init(void) +*/ + ENTRY(dtrace_invop_init) + movl $dtrace_invop_start, dtrace_invop_jump_addr + ret + END(dtrace_invop_init) + +/* +void dtrace_invop_uninit(void) +*/ + ENTRY(dtrace_invop_uninit) + movl $0, dtrace_invop_jump_addr + ret + END(dtrace_invop_uninit) + +/* +greg_t dtrace_getfp(void) +*/ + + ENTRY(dtrace_getfp) + movl %ebp, %eax + ret + END(dtrace_getfp) + +/* +uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) +*/ + + ENTRY(dtrace_cas32) + ALTENTRY(dtrace_casptr) + movl 4(%esp), %edx + movl 8(%esp), %eax + movl 12(%esp), %ecx + lock + cmpxchgl %ecx, (%edx) + ret + END(dtrace_casptr) + END(dtrace_cas32) + +/* +uintptr_t dtrace_caller(int aframes) +*/ + + ENTRY(dtrace_caller) + movl $-1, %eax + ret + END(dtrace_caller) + +/* +void dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) +*/ + + ENTRY(dtrace_copy) + pushl %ebp + movl %esp, %ebp + pushl %esi + pushl %edi + + movl 8(%ebp), %esi /* Load source address */ + movl 12(%ebp), %edi /* Load destination address */ + movl 16(%ebp), %ecx /* Load count */ + repz /* Repeat for count... */ + smovb /* move from %ds:si to %es:di */ + + popl %edi + popl %esi + movl %ebp, %esp + popl %ebp + ret + END(dtrace_copy) + +/* +void dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size) +*/ + + ENTRY(dtrace_copystr) + + pushl %ebp /* Setup stack frame */ + movl %esp, %ebp + pushl %ebx /* Save registers */ + + movl 8(%ebp), %ebx /* Load source address */ + movl 12(%ebp), %edx /* Load destination address */ + movl 16(%ebp), %ecx /* Load count */ + +0: + movb (%ebx), %al /* Load from source */ + movb %al, (%edx) /* Store to destination */ + incl %ebx /* Increment source pointer */ + incl %edx /* Increment destination pointer */ + decl %ecx /* Decrement remaining count */ + cmpb $0, %al + je 1f + cmpl $0, %ecx + jne 0b + +1: + popl %ebx + movl %ebp, %esp + popl %ebp + ret + + END(dtrace_copystr) + +/* +uintptr_t dtrace_fulword(void *addr) +*/ + + ENTRY(dtrace_fulword) + movl 4(%esp), %ecx + xorl %eax, %eax + movl (%ecx), %eax + ret + END(dtrace_fulword) + +/* +uint8_t dtrace_fuword8_nocheck(void *addr) +*/ + + ENTRY(dtrace_fuword8_nocheck) + movl 4(%esp), %ecx + xorl %eax, %eax + movzbl (%ecx), %eax + ret + END(dtrace_fuword8_nocheck) + +/* +uint16_t dtrace_fuword16_nocheck(void *addr) +*/ + + ENTRY(dtrace_fuword16_nocheck) + movl 4(%esp), %ecx + xorl %eax, %eax + movzwl (%ecx), %eax + ret + END(dtrace_fuword16_nocheck) + +/* +uint32_t dtrace_fuword32_nocheck(void *addr) +*/ + + ENTRY(dtrace_fuword32_nocheck) + movl 4(%esp), %ecx + xorl %eax, %eax + movl (%ecx), %eax + ret + END(dtrace_fuword32_nocheck) + +/* +uint64_t dtrace_fuword64_nocheck(void *addr) +*/ + + ENTRY(dtrace_fuword64_nocheck) + movl 4(%esp), %ecx + xorl %eax, %eax + xorl %edx, %edx + movl (%ecx), %eax + movl 4(%ecx), %edx + ret + END(dtrace_fuword64_nocheck) + +/* +void dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, int fault, int fltoffs, uintptr_t illval) +*/ + + ENTRY(dtrace_probe_error) + pushl %ebp + movl %esp, %ebp + pushl 0x1c(%ebp) + pushl 0x18(%ebp) + pushl 0x14(%ebp) + pushl 0x10(%ebp) + pushl 0xc(%ebp) + pushl 0x8(%ebp) + pushl dtrace_probeid_error + call dtrace_probe + movl %ebp, %esp + popl %ebp + ret + END(dtrace_probe_error) + +/* +void dtrace_membar_producer(void) +*/ + + ENTRY(dtrace_membar_producer) + rep; ret /* use 2 byte return instruction when branch target */ + /* AMD Software Optimization Guide - Section 6.2 */ + END(dtrace_membar_producer) + +/* +void dtrace_membar_consumer(void) +*/ + + ENTRY(dtrace_membar_consumer) + rep; ret /* use 2 byte return instruction when branch target */ + /* AMD Software Optimization Guide - Section 6.2 */ + END(dtrace_membar_consumer) + +/* +dtrace_icookie_t dtrace_interrupt_disable(void) +*/ + ENTRY(dtrace_interrupt_disable) + pushfl + popl %eax + cli + ret + END(dtrace_interrupt_disable) + +/* +void dtrace_interrupt_enable(dtrace_icookie_t cookie) +*/ + ENTRY(dtrace_interrupt_enable) + movl 4(%esp), %eax + pushl %eax + popfl + ret + END(dtrace_interrupt_enable) diff --git a/sys/cddl/dev/dtrace/i386/dtrace_isa.c b/sys/cddl/dev/dtrace/i386/dtrace_isa.c new file mode 100644 index 000000000000..6da1b5088f5d --- /dev/null +++ b/sys/cddl/dev/dtrace/i386/dtrace_isa.c @@ -0,0 +1,679 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#include <sys/cdefs.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/stack.h> +#include <sys/pcpu.h> + +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/pcb.h> +#include <machine/stack.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> + +#include "regset.h" + +extern uintptr_t kernbase; +uintptr_t kernelbase = (uintptr_t) &kernbase; + +uint8_t dtrace_fuword8_nocheck(void *); +uint16_t dtrace_fuword16_nocheck(void *); +uint32_t dtrace_fuword32_nocheck(void *); +uint64_t dtrace_fuword64_nocheck(void *); + +int dtrace_ustackdepth_max = 2048; + +void +dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, + uint32_t *intrpc) +{ + int depth = 0; + register_t ebp; + struct i386_frame *frame; + vm_offset_t callpc; + pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller; + + if (intrpc != 0) + pcstack[depth++] = (pc_t) intrpc; + + aframes++; + + __asm __volatile("movl %%ebp,%0" : "=r" (ebp)); + + frame = (struct i386_frame *)ebp; + while (depth < pcstack_limit) { + if (!INKERNEL(frame)) + break; + + callpc = frame->f_retaddr; + + if (!INKERNEL(callpc)) + break; + + if (aframes > 0) { + aframes--; + if ((aframes == 0) && (caller != 0)) { + pcstack[depth++] = caller; + } + } + else { + pcstack[depth++] = callpc; + } + + if (frame->f_frame <= frame || + (vm_offset_t)frame->f_frame >= curthread->td_kstack + + curthread->td_kstack_pages * PAGE_SIZE) + break; + frame = frame->f_frame; + } + + for (; depth < pcstack_limit; depth++) { + pcstack[depth] = 0; + } +} + +static int +dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc, + uintptr_t sp) +{ +#ifdef notyet + proc_t *p = curproc; + uintptr_t oldcontext = lwp->lwp_oldcontext; /* XXX signal stack. */ + size_t s1, s2; +#endif + uintptr_t oldsp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + int ret = 0; + + ASSERT(pcstack == NULL || pcstack_limit > 0); + ASSERT(dtrace_ustackdepth_max > 0); + +#ifdef notyet /* XXX signal stack. */ + if (p->p_model == DATAMODEL_NATIVE) { + s1 = sizeof (struct frame) + 2 * sizeof (long); + s2 = s1 + sizeof (siginfo_t); + } else { + s1 = sizeof (struct frame32) + 3 * sizeof (int); + s2 = s1 + sizeof (siginfo32_t); + } +#endif + + while (pc != 0) { + /* + * We limit the number of times we can go around this + * loop to account for a circular stack. + */ + if (ret++ >= dtrace_ustackdepth_max) { + *flags |= CPU_DTRACE_BADSTACK; + cpu_core[curcpu].cpuc_dtrace_illval = sp; + break; + } + + if (pcstack != NULL) { + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (sp == 0) + break; + + oldsp = sp; + +#ifdef notyet /* XXX signal stack. */ + if (oldcontext == sp + s1 || oldcontext == sp + s2) { + if (p->p_model == DATAMODEL_NATIVE) { + ucontext_t *ucp = (ucontext_t *)oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fulword(&gregs[REG_FP]); + pc = dtrace_fulword(&gregs[REG_PC]); + + oldcontext = dtrace_fulword(&ucp->uc_link); + } else { + ucontext32_t *ucp = (ucontext32_t *)oldcontext; + greg32_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fuword32(&gregs[EBP]); + pc = dtrace_fuword32(&gregs[EIP]); + + oldcontext = dtrace_fuword32(&ucp->uc_link); + } + } else { + if (p->p_model == DATAMODEL_NATIVE) { + struct frame *fr = (struct frame *)sp; + + pc = dtrace_fulword(&fr->fr_savpc); + sp = dtrace_fulword(&fr->fr_savfp); + } else { + struct frame32 *fr = (struct frame32 *)sp; + + pc = dtrace_fuword32(&fr->fr_savpc); + sp = dtrace_fuword32(&fr->fr_savfp); + } + } +#else + pc = dtrace_fuword32((void *)(sp + + offsetof(struct i386_frame, f_retaddr))); + sp = dtrace_fuword32((void *)sp); +#endif /* ! notyet */ + + if (sp == oldsp) { + *flags |= CPU_DTRACE_BADSTACK; + cpu_core[curcpu].cpuc_dtrace_illval = sp; + break; + } + + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, sp, fp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + int n; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = tf->tf_eip; + fp = tf->tf_ebp; + sp = tf->tf_esp; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = dtrace_fuword32((void *) sp); + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, sp); + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +int +dtrace_getustackdepth(void) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, fp, sp; + int n = 0; + + if (p == NULL || (tf = curthread->td_frame) == NULL) + return (0); + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) + return (-1); + + pc = tf->tf_eip; + fp = tf->tf_ebp; + sp = tf->tf_esp; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + + pc = dtrace_fuword32((void *) sp); + n++; + } + + n += dtrace_getustack_common(NULL, 0, pc, fp); + + return (n); +} + +void +dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, sp, fp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; +#ifdef notyet /* XXX signal stack */ + uintptr_t oldcontext; + size_t s1, s2; +#endif + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = tf->tf_eip; + fp = tf->tf_ebp; + sp = tf->tf_esp; + +#ifdef notyet /* XXX signal stack */ + oldcontext = lwp->lwp_oldcontext; + + if (p->p_model == DATAMODEL_NATIVE) { + s1 = sizeof (struct frame) + 2 * sizeof (long); + s2 = s1 + sizeof (siginfo_t); + } else { + s1 = sizeof (struct frame32) + 3 * sizeof (int); + s2 = s1 + sizeof (siginfo32_t); + } +#endif + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t)pc; + *fpstack++ = 0; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = dtrace_fuword32((void *)sp); + } + + while (pc != 0) { + *pcstack++ = (uint64_t)pc; + *fpstack++ = fp; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + + if (fp == 0) + break; + +#ifdef notyet /* XXX signal stack */ + if (oldcontext == sp + s1 || oldcontext == sp + s2) { + if (p->p_model == DATAMODEL_NATIVE) { + ucontext_t *ucp = (ucontext_t *)oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fulword(&gregs[REG_FP]); + pc = dtrace_fulword(&gregs[REG_PC]); + + oldcontext = dtrace_fulword(&ucp->uc_link); + } else { + ucontext_t *ucp = (ucontext_t *)oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fuword32(&gregs[EBP]); + pc = dtrace_fuword32(&gregs[EIP]); + + oldcontext = dtrace_fuword32(&ucp->uc_link); + } + } else +#endif /* XXX */ + { + pc = dtrace_fuword32((void *)(fp + + offsetof(struct i386_frame, f_retaddr))); + fp = dtrace_fuword32((void *)fp); + } + + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } + } + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +uint64_t +dtrace_getarg(int arg, int aframes) +{ + struct trapframe *frame; + struct i386_frame *fp = (struct i386_frame *)dtrace_getfp(); + uintptr_t *stack, val; + int i; + + for (i = 1; i <= aframes; i++) { + fp = fp->f_frame; + + if (P2ROUNDUP(fp->f_retaddr, 4) == + (long)dtrace_invop_callsite) { + /* + * If we pass through the invalid op handler, we will + * use the trap frame pointer that it pushed on the + * stack as the second argument to dtrace_invop() as + * the pointer to the stack. When using this stack, we + * must skip the third argument to dtrace_invop(), + * which is included in the i386_frame. + */ + frame = (struct trapframe *)(((uintptr_t **)&fp[1])[0]); + /* + * Skip the three hardware-saved registers and the + * return address. + */ + stack = (uintptr_t *)frame->tf_isp + 4; + goto load; + } + + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- the provider simply called dtrace_probe() + * directly. As this is the case, we need to shift the argument + * that we're looking for: the probe ID is the first argument to + * dtrace_probe(), so the argument n will actually be found where + * one would expect to find argument (n + 1). + */ + arg++; + + stack = (uintptr_t *)fp + 2; + +load: + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + val = stack[arg]; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + + return (val); +} + +int +dtrace_getstackdepth(int aframes) +{ + int depth = 0; + struct i386_frame *frame; + vm_offset_t ebp; + + aframes++; + ebp = dtrace_getfp(); + frame = (struct i386_frame *)ebp; + depth++; + for(;;) { + if (!INKERNEL((long) frame)) + break; + if (!INKERNEL((long) frame->f_frame)) + break; + depth++; + if (frame->f_frame <= frame || + (vm_offset_t)frame->f_frame >= curthread->td_kstack + + curthread->td_kstack_pages * PAGE_SIZE) + break; + frame = frame->f_frame; + } + if (depth < aframes) + return 0; + else + return depth - aframes; +} + +ulong_t +dtrace_getreg(struct trapframe *rp, uint_t reg) +{ + struct pcb *pcb; + int regmap[] = { /* Order is dependent on reg.d */ + REG_GS, /* 0 GS */ + REG_FS, /* 1 FS */ + REG_ES, /* 2 ES */ + REG_DS, /* 3 DS */ + REG_RDI, /* 4 EDI */ + REG_RSI, /* 5 ESI */ + REG_RBP, /* 6 EBP, REG_FP */ + REG_RSP, /* 7 ESP */ + REG_RBX, /* 8 EBX */ + REG_RDX, /* 9 EDX, REG_R1 */ + REG_RCX, /* 10 ECX */ + REG_RAX, /* 11 EAX, REG_R0 */ + REG_TRAPNO, /* 12 TRAPNO */ + REG_ERR, /* 13 ERR */ + REG_RIP, /* 14 EIP, REG_PC */ + REG_CS, /* 15 CS */ + REG_RFL, /* 16 EFL, REG_PS */ + REG_RSP, /* 17 UESP, REG_SP */ + REG_SS /* 18 SS */ + }; + + if (reg > SS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + if (reg >= sizeof (regmap) / sizeof (int)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + reg = regmap[reg]; + + switch(reg) { + case REG_GS: + if ((pcb = curthread->td_pcb) == NULL) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + return (pcb->pcb_gs); + case REG_FS: + return (rp->tf_fs); + case REG_ES: + return (rp->tf_es); + case REG_DS: + return (rp->tf_ds); + case REG_RDI: + return (rp->tf_edi); + case REG_RSI: + return (rp->tf_esi); + case REG_RBP: + return (rp->tf_ebp); + case REG_RSP: + return (rp->tf_isp); + case REG_RBX: + return (rp->tf_ebx); + case REG_RCX: + return (rp->tf_ecx); + case REG_RAX: + return (rp->tf_eax); + case REG_TRAPNO: + return (rp->tf_trapno); + case REG_ERR: + return (rp->tf_err); + case REG_RIP: + return (rp->tf_eip); + case REG_CS: + return (rp->tf_cs); + case REG_RFL: + return (rp->tf_eflags); +#if 0 + case REG_RSP: + return (rp->tf_esp); +#endif + case REG_SS: + return (rp->tf_ss); + default: + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } +} + +static int +dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) +{ + ASSERT(kaddr >= kernelbase && kaddr + size >= kaddr); + + if (uaddr + size >= kernelbase || uaddr + size < uaddr) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = uaddr; + return (0); + } + + return (1); +} + +void +dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(uaddr, kaddr, size); +} + +void +dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(kaddr, uaddr, size); +} + +void +dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(uaddr, kaddr, size, flags); +} + +void +dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(kaddr, uaddr, size, flags); +} + +uint8_t +dtrace_fuword8(void *uaddr) +{ + if ((uintptr_t)uaddr >= kernelbase) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword8_nocheck(uaddr)); +} + +uint16_t +dtrace_fuword16(void *uaddr) +{ + if ((uintptr_t)uaddr >= kernelbase) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword16_nocheck(uaddr)); +} + +uint32_t +dtrace_fuword32(void *uaddr) +{ + if ((uintptr_t)uaddr >= kernelbase) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword32_nocheck(uaddr)); +} + +uint64_t +dtrace_fuword64(void *uaddr) +{ + if ((uintptr_t)uaddr >= kernelbase) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword64_nocheck(uaddr)); +} diff --git a/sys/cddl/dev/dtrace/i386/dtrace_subr.c b/sys/cddl/dev/dtrace/i386/dtrace_subr.c new file mode 100644 index 000000000000..4c0decaa3c87 --- /dev/null +++ b/sys/cddl/dev/dtrace/i386/dtrace_subr.c @@ -0,0 +1,451 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * Copyright (c) 2011, Joyent, Inc. All rights reserved. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/cpuset.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/kmem.h> +#include <sys/smp.h> +#include <sys/dtrace_impl.h> +#include <sys/dtrace_bsd.h> +#include <machine/clock.h> +#include <machine/cpufunc.h> +#include <machine/frame.h> +#include <machine/psl.h> +#include <machine/trap.h> +#include <vm/pmap.h> + +extern uintptr_t kernelbase; + +extern void dtrace_getnanotime(struct timespec *tsp); + +int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t); + +typedef struct dtrace_invop_hdlr { + int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t); + struct dtrace_invop_hdlr *dtih_next; +} dtrace_invop_hdlr_t; + +dtrace_invop_hdlr_t *dtrace_invop_hdlr; + +int +dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) +{ + dtrace_invop_hdlr_t *hdlr; + int rval; + + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) + if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) + return (rval); + + return (0); +} + +void +dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr; + + hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); + hdlr->dtih_func = func; + hdlr->dtih_next = dtrace_invop_hdlr; + dtrace_invop_hdlr = hdlr; +} + +void +dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL; + + for (;;) { + if (hdlr == NULL) + panic("attempt to remove non-existent invop handler"); + + if (hdlr->dtih_func == func) + break; + + prev = hdlr; + hdlr = hdlr->dtih_next; + } + + if (prev == NULL) { + ASSERT(dtrace_invop_hdlr == hdlr); + dtrace_invop_hdlr = hdlr->dtih_next; + } else { + ASSERT(dtrace_invop_hdlr != hdlr); + prev->dtih_next = hdlr->dtih_next; + } + + kmem_free(hdlr, 0); +} + +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + (*func)(0, kernelbase); +} + +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) +{ + cpuset_t cpus; + + if (cpu == DTRACE_CPUALL) + cpus = all_cpus; + else + CPU_SETOF(cpu, &cpus); + + smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func, + smp_no_rendezvous_barrier, arg); +} + +static void +dtrace_sync_func(void) +{ +} + +void +dtrace_sync(void) +{ + dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); +} + +#ifdef notyet +void +dtrace_safe_synchronous_signal(void) +{ + kthread_t *t = curthread; + struct regs *rp = lwptoregs(ttolwp(t)); + size_t isz = t->t_dtrace_npc - t->t_dtrace_pc; + + ASSERT(t->t_dtrace_on); + + /* + * If we're not in the range of scratch addresses, we're not actually + * tracing user instructions so turn off the flags. If the instruction + * we copied out caused a synchonous trap, reset the pc back to its + * original value and turn off the flags. + */ + if (rp->r_pc < t->t_dtrace_scrpc || + rp->r_pc > t->t_dtrace_astpc + isz) { + t->t_dtrace_ft = 0; + } else if (rp->r_pc == t->t_dtrace_scrpc || + rp->r_pc == t->t_dtrace_astpc) { + rp->r_pc = t->t_dtrace_pc; + t->t_dtrace_ft = 0; + } +} + +int +dtrace_safe_defer_signal(void) +{ + kthread_t *t = curthread; + struct regs *rp = lwptoregs(ttolwp(t)); + size_t isz = t->t_dtrace_npc - t->t_dtrace_pc; + + ASSERT(t->t_dtrace_on); + + /* + * If we're not in the range of scratch addresses, we're not actually + * tracing user instructions so turn off the flags. + */ + if (rp->r_pc < t->t_dtrace_scrpc || + rp->r_pc > t->t_dtrace_astpc + isz) { + t->t_dtrace_ft = 0; + return (0); + } + + /* + * If we have executed the original instruction, but we have performed + * neither the jmp back to t->t_dtrace_npc nor the clean up of any + * registers used to emulate %rip-relative instructions in 64-bit mode, + * we'll save ourselves some effort by doing that here and taking the + * signal right away. We detect this condition by seeing if the program + * counter is the range [scrpc + isz, astpc). + */ + if (rp->r_pc >= t->t_dtrace_scrpc + isz && + rp->r_pc < t->t_dtrace_astpc) { +#ifdef __amd64 + /* + * If there is a scratch register and we're on the + * instruction immediately after the modified instruction, + * restore the value of that scratch register. + */ + if (t->t_dtrace_reg != 0 && + rp->r_pc == t->t_dtrace_scrpc + isz) { + switch (t->t_dtrace_reg) { + case REG_RAX: + rp->r_rax = t->t_dtrace_regv; + break; + case REG_RCX: + rp->r_rcx = t->t_dtrace_regv; + break; + case REG_R8: + rp->r_r8 = t->t_dtrace_regv; + break; + case REG_R9: + rp->r_r9 = t->t_dtrace_regv; + break; + } + } +#endif + rp->r_pc = t->t_dtrace_npc; + t->t_dtrace_ft = 0; + return (0); + } + + /* + * Otherwise, make sure we'll return to the kernel after executing + * the copied out instruction and defer the signal. + */ + if (!t->t_dtrace_step) { + ASSERT(rp->r_pc < t->t_dtrace_astpc); + rp->r_pc += t->t_dtrace_astpc - t->t_dtrace_scrpc; + t->t_dtrace_step = 1; + } + + t->t_dtrace_ast = 1; + + return (1); +} +#endif + +static int64_t tgt_cpu_tsc; +static int64_t hst_cpu_tsc; +static int64_t tsc_skew[MAXCPU]; +static uint64_t nsec_scale; + +/* See below for the explanation of this macro. */ +#define SCALE_SHIFT 28 + +static void +dtrace_gethrtime_init_cpu(void *arg) +{ + uintptr_t cpu = (uintptr_t) arg; + + if (cpu == curcpu) + tgt_cpu_tsc = rdtsc(); + else + hst_cpu_tsc = rdtsc(); +} + +#ifdef EARLY_AP_STARTUP +static void +dtrace_gethrtime_init(void *arg) +{ + struct pcpu *pc; + uint64_t tsc_f; + cpuset_t map; + int i; +#else +/* + * Get the frequency and scale factor as early as possible so that they can be + * used for boot-time tracing. + */ +static void +dtrace_gethrtime_init_early(void *arg) +{ + uint64_t tsc_f; +#endif + + /* + * Get TSC frequency known at this moment. + * This should be constant if TSC is invariant. + * Otherwise tick->time conversion will be inaccurate, but + * will preserve monotonic property of TSC. + */ + tsc_f = atomic_load_acq_64(&tsc_freq); + + /* + * The following line checks that nsec_scale calculated below + * doesn't overflow 32-bit unsigned integer, so that it can multiply + * another 32-bit integer without overflowing 64-bit. + * Thus minimum supported TSC frequency is 62.5MHz. + */ + KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)), + ("TSC frequency is too low")); + + /* + * We scale up NANOSEC/tsc_f ratio to preserve as much precision + * as possible. + * 2^28 factor was chosen quite arbitrarily from practical + * considerations: + * - it supports TSC frequencies as low as 62.5MHz (see above); + * - it provides quite good precision (e < 0.01%) up to THz + * (terahertz) values; + */ + nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f; +#ifndef EARLY_AP_STARTUP +} +SYSINIT(dtrace_gethrtime_init_early, SI_SUB_CPU, SI_ORDER_ANY, + dtrace_gethrtime_init_early, NULL); + +static void +dtrace_gethrtime_init(void *arg) +{ + cpuset_t map; + struct pcpu *pc; + int i; +#endif + + if (vm_guest != VM_GUEST_NO) + return; + + /* The current CPU is the reference one. */ + sched_pin(); + tsc_skew[curcpu] = 0; + CPU_FOREACH(i) { + if (i == curcpu) + continue; + + pc = pcpu_find(i); + CPU_SETOF(PCPU_GET(cpuid), &map); + CPU_SET(pc->pc_cpuid, &map); + + smp_rendezvous_cpus(map, NULL, + dtrace_gethrtime_init_cpu, + smp_no_rendezvous_barrier, (void *)(uintptr_t) i); + + tsc_skew[i] = tgt_cpu_tsc - hst_cpu_tsc; + } + sched_unpin(); +} +#ifdef EARLY_AP_STARTUP +SYSINIT(dtrace_gethrtime_init, SI_SUB_DTRACE, SI_ORDER_ANY, + dtrace_gethrtime_init, NULL); +#else +SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, + NULL); +#endif + +/* + * DTrace needs a high resolution time function which can + * be called from a probe context and guaranteed not to have + * instrumented with probes itself. + * + * Returns nanoseconds since boot. + */ +uint64_t +dtrace_gethrtime(void) +{ + uint64_t tsc; + uint32_t lo, hi; + register_t eflags; + + /* + * We split TSC value into lower and higher 32-bit halves and separately + * scale them with nsec_scale, then we scale them down by 2^28 + * (see nsec_scale calculations) taking into account 32-bit shift of + * the higher half and finally add. + */ + eflags = intr_disable(); + tsc = rdtsc() - tsc_skew[curcpu]; + intr_restore(eflags); + + lo = tsc; + hi = tsc >> 32; + return (((lo * nsec_scale) >> SCALE_SHIFT) + + ((hi * nsec_scale) << (32 - SCALE_SHIFT))); +} + +uint64_t +dtrace_gethrestime(void) +{ + struct timespec current_time; + + dtrace_getnanotime(¤t_time); + + return (current_time.tv_sec * 1000000000ULL + current_time.tv_nsec); +} + +/* Function to handle DTrace traps during probes. See i386/i386/trap.c */ +int +dtrace_trap(struct trapframe *frame, u_int type) +{ + uint16_t nofault; + + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in its per-cpu flags to indicate that it doesn't + * want to fault. On returning from the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * Check if DTrace has enabled 'no-fault' mode: + */ + sched_pin(); + nofault = cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT; + sched_unpin(); + if (nofault) { + KASSERT((read_eflags() & PSL_I) == 0, ("interrupts enabled")); + + /* + * There are only a couple of trap types that are expected. + * All the rest will be handled in the usual way. + */ + switch (type) { + /* General protection fault. */ + case T_PROTFLT: + /* Flag an illegal operation. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->tf_eip += dtrace_instr_size((u_char *) frame->tf_eip); + return (1); + /* Page fault. */ + case T_PAGEFLT: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = rcr2(); + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->tf_eip += dtrace_instr_size((u_char *) frame->tf_eip); + return (1); + default: + /* Handle all other traps in the usual way. */ + break; + } + } + + /* Handle the trap in the usual way. */ + return (0); +} diff --git a/sys/cddl/dev/dtrace/mips/dtrace_asm.S b/sys/cddl/dev/dtrace/mips/dtrace_asm.S new file mode 100644 index 000000000000..b40336c0f114 --- /dev/null +++ b/sys/cddl/dev/dtrace/mips/dtrace_asm.S @@ -0,0 +1,234 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#define _ASM +#define _LOCORE + +#include <sys/cpuvar_defs.h> +#include <sys/dtrace.h> + +#include <machine/asm.h> +#include <machine/cpu.h> +#include <machine/cpuregs.h> +#include <machine/regnum.h> + + .set noreorder # Noreorder is default style! + +/* + * Primitives + */ + + .text + +/* +void dtrace_membar_producer(void) +*/ +LEAF(dtrace_membar_producer) + j ra + nop +END(dtrace_membar_producer) + +/* +void dtrace_membar_consumer(void) +*/ +LEAF(dtrace_membar_consumer) + j ra + nop +END(dtrace_membar_consumer) + +/* +dtrace_icookie_t dtrace_interrupt_disable(void) +*/ +LEAF(dtrace_interrupt_disable) + mfc0 t0, MIPS_COP_0_STATUS + move v0, t0 + and v0, v0, MIPS_SR_INT_IE + and t0, t0, ~MIPS_SR_INT_IE + mtc0 t0, MIPS_COP_0_STATUS + j ra + nop +END(dtrace_interrupt_disable) + +/* +void dtrace_interrupt_enable(dtrace_icookie_t cookie) +*/ +LEAF(dtrace_interrupt_enable) + mfc0 t0, MIPS_COP_0_STATUS + beqz a0, not_enabled + or t0, t0, MIPS_SR_INT_IE + mtc0 t0, MIPS_COP_0_STATUS +not_enabled: + j ra + nop +END(dtrace_interrupt_enable) + +/* +uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) +*/ +LEAF(dtrace_cas32) +1: + move t1, a2 + ll t0, 0(a0) + bne t0, a1, 2f + nop + sc t1, 0(a0) + beqz t1, 1b + nop +2: move v0, t0 + j ra + nop +END(dtrace_cas32) + +/* +void * +dtrace_casptr(void *target, void *cmp, void *new) +*/ +LEAF(dtrace_casptr) +1: + move t1, a2 + PTR_LL t0, 0(a0) + bne t0, a1, 2f + nop + PTR_SC t1, 0(a0) + beqz t1, 1b + nop +2: move v0, t0 + j ra + nop +END(dtrace_casptr) + + +/* +uintptr_t +dtrace_fulword(void *addr) +*/ +LEAF(dtrace_fulword) +END(dtrace_fulword) + +/* +uint8_t +dtrace_fuword8_nocheck(void *addr) +*/ +LEAF(dtrace_fuword8_nocheck) + lbu v0, 0(a0) + j ra + nop +END(dtrace_fuword8_nocheck) + +/* +uint16_t +dtrace_fuword16_nocheck(void *addr) +*/ +LEAF(dtrace_fuword16_nocheck) + lhu v0, 0(a0) + j ra + nop +END(dtrace_fuword16_nocheck) + +/* +uint32_t +dtrace_fuword32_nocheck(void *addr) +*/ +LEAF(dtrace_fuword32_nocheck) + lw v0, 0(a0) + j ra + nop +END(dtrace_fuword32_nocheck) + +/* +uint64_t +dtrace_fuword64_nocheck(void *addr) +*/ +LEAF(dtrace_fuword64_nocheck) +#if defined(__mips_n64) || defined(__mips_n32) + ld v0, 0(a0) +#else + lw v1,4(a0) + lw v0,0(a0) +#endif + j ra + nop +END(dtrace_fuword64_nocheck) + +/* +XXX: unoptimized +void +dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) +*/ +LEAF(dtrace_copy) +1: + beqz a2, 2f + nop + lbu t0, 0(a0) + sb t0, 0(a1) + PTR_ADDU a0, a0, 1 + PTR_ADDU a1, a1, 1 + INT_SUBU a2, a2, 1 + j 1b + nop +2: + j ra + nop +END(dtrace_copy) + +/* +XXX: Unoptimized. Check for flags on page boundaries only(?) +void +dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +*/ +LEAF(dtrace_copystr) +1: + lbu t0, 0(a0) + sb t0, 0(a1) + PTR_ADDU a0, a0, 1 + PTR_ADDU a1, a1, 1 + INT_SUBU a2, a2, 1 + beqz t0, 2f + nop + lhu t1, (a3) + and t1, t1, CPU_DTRACE_BADADDR + bnez t1, 2f + nop + + bnez a2, 1b + nop +2: + j ra + nop +END(dtrace_copystr) + +/* +uintptr_t +dtrace_caller(int aframes) +*/ +LEAF(dtrace_caller) + li v0, -1 + j ra + nop +END(dtrace_caller) diff --git a/sys/cddl/dev/dtrace/mips/dtrace_isa.c b/sys/cddl/dev/dtrace/mips/dtrace_isa.c new file mode 100644 index 000000000000..c09225c734b9 --- /dev/null +++ b/sys/cddl/dev/dtrace/mips/dtrace_isa.c @@ -0,0 +1,725 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#include <sys/cdefs.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/stack.h> +#include <sys/pcpu.h> + +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/reg.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> + +#include <machine/db_machdep.h> +#include <machine/md_var.h> +#include <machine/mips_opcode.h> +#include <ddb/db_sym.h> +#include <ddb/ddb.h> +#include <sys/kdb.h> + +#include "regset.h" + +#ifdef __mips_n64 +#define MIPS_IS_VALID_KERNELADDR(reg) ((((reg) & 3) == 0) && \ + ((vm_offset_t)(reg) >= MIPS_XKPHYS_START)) +#else +#define MIPS_IS_VALID_KERNELADDR(reg) ((((reg) & 3) == 0) && \ + ((vm_offset_t)(reg) >= MIPS_KSEG0_START)) +#endif + + + +/* + * Wee need some reasonable default to prevent backtrace code + * from wandering too far + */ +#define MAX_FUNCTION_SIZE 0x10000 +#define MAX_PROLOGUE_SIZE 0x100 + +uint8_t dtrace_fuword8_nocheck(void *); +uint16_t dtrace_fuword16_nocheck(void *); +uint32_t dtrace_fuword32_nocheck(void *); +uint64_t dtrace_fuword64_nocheck(void *); + +static int dtrace_next_frame(register_t *pc, register_t *sp, register_t *args, int *valid_args); +static int dtrace_next_uframe(register_t *pc, register_t *sp, register_t *ra); + +void +dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, + uint32_t *intrpc) +{ + int depth = 0; + vm_offset_t callpc; + pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller; + register_t sp, ra, pc; + + if (intrpc != 0) + pcstack[depth++] = (pc_t) intrpc; + + aframes++; + + sp = (register_t)(intptr_t)__builtin_frame_address(0); + ra = (register_t)(intptr_t)__builtin_return_address(0); + + __asm __volatile( + "jal 99f\n" + "nop\n" + "99:\n" + "move %0, $31\n" /* get ra */ + "move $31, %1\n" /* restore ra */ + : "=r" (pc) + : "r" (ra)); + + while (depth < pcstack_limit) { + + callpc = pc; + + if (aframes > 0) { + aframes--; + if ((aframes == 0) && (caller != 0)) { + pcstack[depth++] = caller; + } + } + else { + pcstack[depth++] = callpc; + } + + if (dtrace_next_frame(&pc, &sp, NULL, NULL) < 0) + break; + } + + for (; depth < pcstack_limit; depth++) { + pcstack[depth] = 0; + } +} + +void +dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) +{ + proc_t *p = curproc; + struct trapframe *tf; + register_t sp, ra, pc; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = (uint64_t)tf->pc; + sp = (uint64_t)tf->sp; + ra = (uint64_t)tf->ra; + *pcstack++ = (uint64_t)tf->pc; + + /* + * Unwind, and unwind, and unwind + */ + while (1) { + if (dtrace_next_uframe(&pc, &sp, &ra) < 0) + break; + + *pcstack++ = pc; + pcstack_limit--; + + if (pcstack_limit <= 0) + break; + } + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +int +dtrace_getustackdepth(void) +{ + int n = 0; + proc_t *p = curproc; + struct trapframe *tf; + register_t sp, ra, pc; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + + if (*flags & CPU_DTRACE_FAULT) + return (0); + + if (p == NULL || (tf = curthread->td_frame) == NULL) + return (0); + + pc = (uint64_t)tf->pc; + sp = (uint64_t)tf->sp; + ra = (uint64_t)tf->ra; + n++; + + /* + * Unwind, and unwind, and unwind + */ + while (1) { + if (dtrace_next_uframe(&pc, &sp, &ra) < 0) + break; + n++; + } + + return (n); +} + +void +dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) +{ + printf("IMPLEMENT ME: %s\n", __func__); +} + +/*ARGSUSED*/ +uint64_t +dtrace_getarg(int arg, int aframes) +{ + int i; + register_t sp, ra, pc; + /* XXX: Fix this ugly code */ + register_t args[8]; + int valid[8]; + + sp = (register_t)(intptr_t)__builtin_frame_address(0); + ra = (register_t)(intptr_t)__builtin_return_address(0); + + __asm __volatile( + "jal 99f\n" + "nop\n" + "99:\n" + "move %0, $31\n" /* get ra */ + "move $31, %1\n" /* restore ra */ + : "=r" (pc) + : "r" (ra)); + + for (i = 0; i <= aframes + 1; i++) { + if (dtrace_next_frame(&pc, &sp, args, valid) < 0) { + printf("%s: stack ends at frame #%d\n", __func__, i); + return (0); + } + } + + if (arg < 8) { + if (valid[arg]) + return (args[arg]); + else + printf("%s: request arg%d is not valid\n", __func__, arg); + } + + return (0); +} + +int +dtrace_getstackdepth(int aframes) +{ + register_t sp, ra, pc; + int depth = 0; + + sp = (register_t)(intptr_t)__builtin_frame_address(0); + ra = (register_t)(intptr_t)__builtin_return_address(0); + + __asm __volatile( + "jal 99f\n" + "nop\n" + "99:\n" + "move %0, $31\n" /* get ra */ + "move $31, %1\n" /* restore ra */ + : "=r" (pc) + : "r" (ra)); + + for (;;) { + if (dtrace_next_frame(&pc, &sp, NULL, NULL) < 0) + break; + depth++; + } + + if (depth < aframes) + return 0; + else + return depth - aframes; +} + +ulong_t +dtrace_getreg(struct trapframe *rp, uint_t reg) +{ + + return (0); +} + +static int +dtrace_next_frame(register_t *pc, register_t *sp, + register_t *args, int *valid_args) +{ + InstFmt i; + /* + * Arrays for a0..a3 registers and flags if content + * of these registers is valid, e.g. obtained from the stack + */ + uintptr_t va; + unsigned instr, mask; + unsigned int frames = 0; + int more, stksize; + register_t ra = 0; + int arg, r; + vm_offset_t addr; + + /* + * Invalidate arguments values + */ + if (valid_args) { + for (r = 0; r < 8; r++) + valid_args[r] = 0; + } + + /* Jump here after a nonstandard (interrupt handler) frame */ + stksize = 0; + if (frames++ > 100) { + /* return breaks stackframe-size heuristics with gcc -O2 */ + goto error; /* XXX */ + } + + /* check for bad SP: could foul up next frame */ + if (!MIPS_IS_VALID_KERNELADDR(*sp)) { + goto error; + } + + /* check for bad PC */ + if (!MIPS_IS_VALID_KERNELADDR(*pc)) { + goto error; + } + + /* + * Find the beginning of the current subroutine by scanning + * backwards from the current PC for the end of the previous + * subroutine. + */ + va = *pc - sizeof(int); + while (1) { + instr = kdbpeek((int *)va); + + /* [d]addiu sp,sp,-X */ + if (((instr & 0xffff8000) == 0x27bd8000) + || ((instr & 0xffff8000) == 0x67bd8000)) + break; + + /* jr ra */ + if (instr == 0x03e00008) { + /* skip over branch-delay slot instruction */ + va += 2 * sizeof(int); + break; + } + + va -= sizeof(int); + } + + /* skip over nulls which might separate .o files */ + while ((instr = kdbpeek((int *)va)) == 0) + va += sizeof(int); + + /* scan forwards to find stack size and any saved registers */ + stksize = 0; + more = 3; + mask = 0; + for (; more; va += sizeof(int), + more = (more == 3) ? 3 : more - 1) { + /* stop if hit our current position */ + if (va >= *pc) + break; + instr = kdbpeek((int *)va); + i.word = instr; + switch (i.JType.op) { + case OP_SPECIAL: + switch (i.RType.func) { + case OP_JR: + case OP_JALR: + more = 2; /* stop after next instruction */ + break; + + case OP_SYSCALL: + case OP_BREAK: + more = 1; /* stop now */ + }; + break; + + case OP_BCOND: + case OP_J: + case OP_JAL: + case OP_BEQ: + case OP_BNE: + case OP_BLEZ: + case OP_BGTZ: + more = 2; /* stop after next instruction */ + break; + + case OP_COP0: + case OP_COP1: + case OP_COP2: + case OP_COP3: + switch (i.RType.rs) { + case OP_BCx: + case OP_BCy: + more = 2; /* stop after next instruction */ + }; + break; + + case OP_SW: + /* look for saved registers on the stack */ + if (i.IType.rs != 29) + break; + /* only restore the first one */ + if (mask & (1 << i.IType.rt)) + break; + mask |= (1 << i.IType.rt); + addr = (vm_offset_t)(*sp + (short)i.IType.imm); + switch (i.IType.rt) { + case 4:/* a0 */ + case 5:/* a1 */ + case 6:/* a2 */ + case 7:/* a3 */ +#if defined(__mips_n64) || defined(__mips_n32) + case 8:/* a4 */ + case 9:/* a5 */ + case 10:/* a6 */ + case 11:/* a7 */ +#endif + arg = i.IType.rt - 4; + if (args) + args[arg] = kdbpeek((int*)addr); + if (valid_args) + valid_args[arg] = 1; + break; + case 31: /* ra */ + ra = kdbpeek((int *)addr); + } + break; + + case OP_SD: + /* look for saved registers on the stack */ + if (i.IType.rs != 29) + break; + /* only restore the first one */ + if (mask & (1 << i.IType.rt)) + break; + mask |= (1 << i.IType.rt); + addr = (vm_offset_t)(*sp + (short)i.IType.imm); + switch (i.IType.rt) { + case 4:/* a0 */ + case 5:/* a1 */ + case 6:/* a2 */ + case 7:/* a3 */ +#if defined(__mips_n64) || defined(__mips_n32) + case 8:/* a4 */ + case 9:/* a5 */ + case 10:/* a6 */ + case 11:/* a7 */ +#endif + arg = i.IType.rt - 4; + if (args) + args[arg] = kdbpeekd((int *)addr); + if (valid_args) + valid_args[arg] = 1; + break; + + case 31: /* ra */ + ra = kdbpeekd((int *)addr); + } + break; + + case OP_ADDI: + case OP_ADDIU: + case OP_DADDI: + case OP_DADDIU: + /* look for stack pointer adjustment */ + if (i.IType.rs != 29 || i.IType.rt != 29) + break; + stksize = -((short)i.IType.imm); + } + } + + if (!MIPS_IS_VALID_KERNELADDR(ra)) + return (-1); + + *pc = ra; + *sp += stksize; + +#if defined(__mips_o32) + /* + * For MIPS32 fill out arguments 5..8 from the stack + */ + for (arg = 4; arg < 8; arg++) { + addr = (vm_offset_t)(*sp + arg*sizeof(register_t)); + if (args) + args[arg] = kdbpeekd((int *)addr); + if (valid_args) + valid_args[arg] = 1; + } +#endif + + return (0); +error: + return (-1); +} + +static int +dtrace_next_uframe(register_t *pc, register_t *sp, register_t *ra) +{ + int offset, registers_on_stack; + uint32_t opcode, mask; + register_t function_start; + int stksize; + InstFmt i; + + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + + registers_on_stack = 0; + mask = 0; + function_start = 0; + offset = 0; + stksize = 0; + + while (offset < MAX_FUNCTION_SIZE) { + opcode = dtrace_fuword32((void *)(vm_offset_t)(*pc - offset)); + + if (*flags & CPU_DTRACE_FAULT) + goto fault; + + /* [d]addiu sp, sp, -X*/ + if (((opcode & 0xffff8000) == 0x27bd8000) + || ((opcode & 0xffff8000) == 0x67bd8000)) { + function_start = *pc - offset; + registers_on_stack = 1; + break; + } + + /* lui gp, X */ + if ((opcode & 0xffff8000) == 0x3c1c0000) { + /* + * Function might start with this instruction + * Keep an eye on "jr ra" and sp correction + * with positive value further on + */ + function_start = *pc - offset; + } + + if (function_start) { + /* + * Stop looking further. Possible end of + * function instruction: it means there is no + * stack modifications, sp is unchanged + */ + + /* [d]addiu sp,sp,X */ + if (((opcode & 0xffff8000) == 0x27bd0000) + || ((opcode & 0xffff8000) == 0x67bd0000)) + break; + + if (opcode == 0x03e00008) + break; + } + + offset += sizeof(int); + } + + if (!function_start) + return (-1); + + if (registers_on_stack) { + offset = 0; + while ((offset < MAX_PROLOGUE_SIZE) + && ((function_start + offset) < *pc)) { + i.word = + dtrace_fuword32((void *)(vm_offset_t)(function_start + offset)); + switch (i.JType.op) { + case OP_SW: + /* look for saved registers on the stack */ + if (i.IType.rs != 29) + break; + /* only restore the first one */ + if (mask & (1 << i.IType.rt)) + break; + mask |= (1 << i.IType.rt); + if (i.IType.rt == 31) + *ra = dtrace_fuword32((void *)(vm_offset_t)(*sp + (short)i.IType.imm)); + break; + + case OP_SD: + /* look for saved registers on the stack */ + if (i.IType.rs != 29) + break; + /* only restore the first one */ + if (mask & (1 << i.IType.rt)) + break; + mask |= (1 << i.IType.rt); + /* ra */ + if (i.IType.rt == 31) + *ra = dtrace_fuword64((void *)(vm_offset_t)(*sp + (short)i.IType.imm)); + break; + + case OP_ADDI: + case OP_ADDIU: + case OP_DADDI: + case OP_DADDIU: + /* look for stack pointer adjustment */ + if (i.IType.rs != 29 || i.IType.rt != 29) + break; + stksize = -((short)i.IType.imm); + } + + offset += sizeof(int); + + if (*flags & CPU_DTRACE_FAULT) + goto fault; + } + } + + /* + * We reached the end of backtrace + */ + if (*pc == *ra) + return (-1); + + *pc = *ra; + *sp += stksize; + + return (0); +fault: + /* + * We just got lost in backtrace, no big deal + */ + *flags &= ~CPU_DTRACE_FAULT; + return (-1); +} + +static int +dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) +{ + + if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = uaddr; + return (0); + } + + return (1); +} + +void +dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(uaddr, kaddr, size); +} + +void +dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(kaddr, uaddr, size); +} + +void +dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(uaddr, kaddr, size, flags); +} + +void +dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(kaddr, uaddr, size, flags); +} + +uint8_t +dtrace_fuword8(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword8_nocheck(uaddr)); +} + +uint16_t +dtrace_fuword16(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword16_nocheck(uaddr)); +} + +uint32_t +dtrace_fuword32(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword32_nocheck(uaddr)); +} + +uint64_t +dtrace_fuword64(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword64_nocheck(uaddr)); +} diff --git a/sys/cddl/dev/dtrace/mips/dtrace_subr.c b/sys/cddl/dev/dtrace/mips/dtrace_subr.c new file mode 100644 index 000000000000..56ae129370fd --- /dev/null +++ b/sys/cddl/dev/dtrace/mips/dtrace_subr.c @@ -0,0 +1,286 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/kmem.h> +#include <sys/smp.h> +#include <sys/dtrace_impl.h> +#include <sys/dtrace_bsd.h> +#include <machine/clock.h> +#include <machine/frame.h> +#include <machine/trap.h> +#include <vm/pmap.h> + +#define DELAYBRANCH(x) ((int)(x) < 0) + +extern int (*dtrace_invop_jump_addr)(struct trapframe *); +extern dtrace_id_t dtrace_probeid_error; + +int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t); +void dtrace_invop_init(void); +void dtrace_invop_uninit(void); + +typedef struct dtrace_invop_hdlr { + int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t); + struct dtrace_invop_hdlr *dtih_next; +} dtrace_invop_hdlr_t; + +dtrace_invop_hdlr_t *dtrace_invop_hdlr; + +int +dtrace_invop(uintptr_t addr, struct trapframe *stack, uintptr_t eax) +{ + dtrace_invop_hdlr_t *hdlr; + int rval; + + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) + if ((rval = hdlr->dtih_func(addr, stack, eax)) != 0) + return (rval); + + return (0); +} + +void +dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr; + + hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); + hdlr->dtih_func = func; + hdlr->dtih_next = dtrace_invop_hdlr; + dtrace_invop_hdlr = hdlr; +} + +void +dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr, *prev; + + hdlr = dtrace_invop_hdlr; + prev = NULL; + + for (;;) { + if (hdlr == NULL) + panic("attempt to remove non-existent invop handler"); + + if (hdlr->dtih_func == func) + break; + + prev = hdlr; + hdlr = hdlr->dtih_next; + } + + if (prev == NULL) { + ASSERT(dtrace_invop_hdlr == hdlr); + dtrace_invop_hdlr = hdlr->dtih_next; + } else { + ASSERT(dtrace_invop_hdlr != hdlr); + prev->dtih_next = hdlr->dtih_next; + } + + kmem_free(hdlr, 0); +} + +/*ARGSUSED*/ +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + /* + * No toxic regions? + */ +} + +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) +{ + cpuset_t cpus; + + if (cpu == DTRACE_CPUALL) + cpus = all_cpus; + else + CPU_SETOF(cpu, &cpus); + + smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func, + smp_no_rendezvous_barrier, arg); +} + +static void +dtrace_sync_func(void) +{ +} + +void +dtrace_sync(void) +{ + dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); +} + +/* + * DTrace needs a high resolution time function which can + * be called from a probe context and guaranteed not to have + * instrumented with probes itself. + * + * Returns nanoseconds since boot. + */ +uint64_t +dtrace_gethrtime() +{ + struct timespec curtime; + + nanouptime(&curtime); + + return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec); + +} + +uint64_t +dtrace_gethrestime(void) +{ + struct timespec curtime; + + getnanotime(&curtime); + + return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec); +} + +/* Function to handle DTrace traps during probes. See amd64/amd64/trap.c */ +int +dtrace_trap(struct trapframe *frame, u_int type) +{ + + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in its per-cpu flags to indicate that it doesn't + * want to fault. On returning from the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * Check if DTrace has enabled 'no-fault' mode: + */ + if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) { + /* + * There are only a couple of trap types that are expected. + * All the rest will be handled in the usual way. + */ + switch (type) { + /* Page fault. */ + case T_TLB_ST_MISS: + case T_ADDR_ERR_ST: + case T_TLB_LD_MISS: + case T_ADDR_ERR_LD: + case T_BUS_ERR_IFETCH: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = frame->badvaddr; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + if (DELAYBRANCH(frame->cause)) /* Check BD bit */ + { + /* XXX: check MipsEmulateBranch on MIPS64 + frame->pc = MipsEmulateBranch(frame, frame->pc, + 0, 0); + */ + panic("%s: delay slot at %jx, badvaddr = %jx\n", + __func__, + (intmax_t)frame->pc, (intmax_t)frame->badvaddr); + } + else + frame->pc += sizeof(int); + return (1); + default: + /* Handle all other traps in the usual way. */ + break; + } + } + + /* Handle the trap in the usual way. */ + return (0); +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fault, int fltoffs, uintptr_t illval) +{ + + dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state, + (uintptr_t)epid, + (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs); +} + +static int +dtrace_invop_start(struct trapframe *frame) +{ + register_t *sp; + int16_t offs; + int invop; + + invop = dtrace_invop(frame->pc, frame, frame->pc); + offs = (invop & LDSD_DATA_MASK); + sp = (register_t *)((uint8_t *)frame->sp + offs); + + switch (invop & LDSD_RA_SP_MASK) { + case LD_RA_SP: + frame->ra = *sp; + frame->pc += INSN_SIZE; + break; + case SD_RA_SP: + *(sp) = frame->ra; + frame->pc += INSN_SIZE; + break; + default: + printf("%s: 0x%x undefined\n", __func__, invop); + return (-1); + }; + + return (0); +} + +void +dtrace_invop_init(void) +{ + + dtrace_invop_jump_addr = dtrace_invop_start; +} + +void +dtrace_invop_uninit(void) +{ + + dtrace_invop_jump_addr = 0; +} diff --git a/sys/cddl/dev/dtrace/mips/regset.h b/sys/cddl/dev/dtrace/mips/regset.h new file mode 100644 index 000000000000..d83a45e52a6e --- /dev/null +++ b/sys/cddl/dev/dtrace/mips/regset.h @@ -0,0 +1,62 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +#ifndef _REGSET_H +#define _REGSET_H + +/* + * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * XXXDTRACE: define registers properly + */ + +#if 0 +#define REG_PC PC +#define REG_FP EBP +#define REG_SP SP +#define REG_PS EFL +#define REG_R0 EAX +#define REG_R1 EDX +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _REGSET_H */ diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S b/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S new file mode 100644 index 000000000000..3371068896db --- /dev/null +++ b/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S @@ -0,0 +1,191 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Portions Copyright 2012,2013 Justin Hibbits <jhibbits@freebsd.org> + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include "assym.inc" + +#define _ASM + +#include <sys/cpuvar_defs.h> +#include <sys/dtrace.h> + +#include <machine/asm.h> +/* +#include <machine/cpu.h> +*/ + +/* + * Primitives + */ + + .text + +/* +void dtrace_membar_producer(void) +*/ +ASENTRY_NOPROF(dtrace_membar_producer) + sync + blr +END(dtrace_membar_producer) + +/* +void dtrace_membar_consumer(void) +*/ +ASENTRY_NOPROF(dtrace_membar_consumer) + isync + blr +END(dtrace_membar_consumer) + +/* +dtrace_icookie_t dtrace_interrupt_disable(void) +*/ +ASENTRY_NOPROF(dtrace_interrupt_disable) + mfmsr %r3 +#ifdef __powerpc64__ + /* Two-instruction sequence to clear EE flag */ + rldicl %r0,%r3,48,1 + rotldi %r0,%r0,16 +#else + rlwinm %r0,%r3,0,17,15 /* Clear EE flag */ +#endif + mtmsr %r0 + blr +END(dtrace_interrupt_disable) + +/* +void dtrace_interrupt_enable(dtrace_icookie_t cookie) +*/ +ASENTRY_NOPROF(dtrace_interrupt_enable) + mtmsr %r3 + blr +END(dtrace_interrupt_enable) + +/* +uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) +*/ +ASENTRY_NOPROF(dtrace_cas32) +1: + lwarx %r0,0,%r3 + cmpw %r4,%r0 + bne 2f + stwcx. %r5,0,%r3 + bne 1b +2: mr %r3,%r0 + blr +END(dtrace_cas32) + +/* +void * +dtrace_casptr(void *target, void *cmp, void *new) +*/ +ASENTRY_NOPROF(dtrace_casptr) +#ifdef __powerpc64__ +1: + ldarx %r0,0,%r3 + cmpd %r4,%r0 + bne 2f + stdcx. %r5,0,%r3 + bne 1b +#else +1: + lwarx %r0,0,%r3 + cmpw %r4,%r0 + bne 2f + stwcx. %r5,0,%r3 + bne 1b +#endif +2: mr %r3,%r0 + blr +END(dtrace_casptr) + + +/* +XXX: unoptimized +void +dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) +*/ +ASENTRY_NOPROF(dtrace_copy) + subi %r7,%r3,1 + subi %r8,%r4,1 + mtctr %r5 +1: + lbzu %r3,1(%r7) + stbu %r3,1(%r8) + bdnz 1b +2: + blr +END(dtrace_copy) + +/* +void +dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +*/ +ASENTRY_NOPROF(dtrace_copystr) + subi %r7,%r3,1 + subi %r8,%r4,1 +1: + lbzu %r3,1(%r7) + stbu %r3,1(%r8) + subi %r5,%r5,1 +#ifdef __powerpc64__ + cmpldi %r5,0 +#else + cmplwi %r5,0 +#endif + beq 2f + cmplwi %r3,0 + beq 2f + andi. %r0,%r5,0x0fff + beq 2f + lwz %r0,0(%r6) + andi. %r0,%r0,CPU_DTRACE_BADADDR + beq 1b +2: + blr +END(dtrace_copystr) + +/* +uintptr_t +dtrace_caller(int aframes) +*/ +ASENTRY_NOPROF(dtrace_caller) + li %r3, -1 + blr +END(dtrace_caller) + +/* +greg_t +dtrace_getfp(void) +*/ +ASENTRY_NOPROF(dtrace_getfp) + mr %r3,%r31 + blr +END(dtrace_getfp) + diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c new file mode 100644 index 000000000000..a188eafa777d --- /dev/null +++ b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c @@ -0,0 +1,692 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Portions Copyright 2012,2013 Justin Hibbits <jhibbits@freebsd.org> + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#include <sys/cdefs.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/stack.h> +#include <sys/sysent.h> +#include <sys/pcpu.h> + +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/psl.h> +#include <machine/reg.h> +#include <machine/stack.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> + +#include "regset.h" + +/* Offset to the LR Save word (ppc32) */ +#define RETURN_OFFSET 4 +/* Offset to LR Save word (ppc64). CR Save area sits between back chain and LR */ +#define RETURN_OFFSET64 16 + +#ifdef __powerpc64__ +#define OFFSET 4 /* Account for the TOC reload slot */ +#define FRAME_OFFSET 48 +#else +#define OFFSET 0 +#define FRAME_OFFSET 8 +#endif + +#define INKERNEL(x) (((x) <= VM_MAX_KERNEL_ADDRESS && \ + (x) >= VM_MIN_KERNEL_ADDRESS) || \ + (PMAP_HAS_DMAP && (x) >= DMAP_BASE_ADDRESS && \ + (x) <= DMAP_MAX_ADDRESS)) + +static __inline int +dtrace_sp_inkernel(uintptr_t sp) +{ + struct trapframe *frame; + vm_offset_t callpc; + + /* Not within the kernel, or not aligned. */ + if (!INKERNEL(sp) || (sp & 0xf) != 0) + return (0); +#ifdef __powerpc64__ + callpc = *(vm_offset_t *)(sp + RETURN_OFFSET64); +#else + callpc = *(vm_offset_t *)(sp + RETURN_OFFSET); +#endif + if ((callpc & 3) || (callpc < 0x100)) + return (0); + + /* + * trapexit() and asttrapexit() are sentinels + * for kernel stack tracing. + */ + if (callpc + OFFSET == (vm_offset_t) &trapexit || + callpc + OFFSET == (vm_offset_t) &asttrapexit) { + frame = (struct trapframe *)(sp + FRAME_OFFSET); + + return ((frame->srr1 & PSL_PR) == 0); + } + + return (1); +} + +static __inline void +dtrace_next_sp_pc(uintptr_t sp, uintptr_t *nsp, uintptr_t *pc) +{ + vm_offset_t callpc; + struct trapframe *frame; + +#ifdef __powerpc64__ + callpc = *(vm_offset_t *)(sp + RETURN_OFFSET64); +#else + callpc = *(vm_offset_t *)(sp + RETURN_OFFSET); +#endif + + /* + * trapexit() and asttrapexit() are sentinels + * for kernel stack tracing. + */ + if ((callpc + OFFSET == (vm_offset_t) &trapexit || + callpc + OFFSET == (vm_offset_t) &asttrapexit)) { + /* Access the trap frame */ + frame = (struct trapframe *)(sp + FRAME_OFFSET); + + if (nsp != NULL) + *nsp = frame->fixreg[1]; + if (pc != NULL) + *pc = frame->srr0; + return; + } + + if (nsp != NULL) + *nsp = *(uintptr_t *)sp; + if (pc != NULL) + *pc = callpc; +} + +void +dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, + uint32_t *intrpc) +{ + int depth = 0; + uintptr_t osp, sp; + vm_offset_t callpc; + pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller; + + osp = PAGE_SIZE; + if (intrpc != 0) + pcstack[depth++] = (pc_t) intrpc; + + aframes++; + + sp = (uintptr_t)__builtin_frame_address(0); + + while (depth < pcstack_limit) { + if (sp <= osp) + break; + + if (!dtrace_sp_inkernel(sp)) + break; + osp = sp; + dtrace_next_sp_pc(osp, &sp, &callpc); + + if (aframes > 0) { + aframes--; + if ((aframes == 0) && (caller != 0)) { + pcstack[depth++] = caller; + } + } + else { + pcstack[depth++] = callpc; + } + } + + for (; depth < pcstack_limit; depth++) { + pcstack[depth] = 0; + } +} + +static int +dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc, + uintptr_t sp) +{ + proc_t *p = curproc; + int ret = 0; + + ASSERT(pcstack == NULL || pcstack_limit > 0); + + while (pc != 0) { + ret++; + if (pcstack != NULL) { + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (sp == 0) + break; + + if (SV_PROC_FLAG(p, SV_ILP32)) { + pc = dtrace_fuword32((void *)(sp + RETURN_OFFSET)); + sp = dtrace_fuword32((void *)sp); + } + else { + pc = dtrace_fuword64((void *)(sp + RETURN_OFFSET64)); + sp = dtrace_fuword64((void *)sp); + } + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, sp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + int n; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = tf->srr0; + sp = tf->fixreg[1]; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = tf->lr; + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, sp); + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +int +dtrace_getustackdepth(void) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, sp; + int n = 0; + + if (p == NULL || (tf = curthread->td_frame) == NULL) + return (0); + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) + return (-1); + + pc = tf->srr0; + sp = tf->fixreg[1]; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + + if (SV_PROC_FLAG(p, SV_ILP32)) { + pc = dtrace_fuword32((void *) sp); + } + else + pc = dtrace_fuword64((void *) sp); + n++; + } + + n += dtrace_getustack_common(NULL, 0, pc, sp); + + return (n); +} + +void +dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) +{ + proc_t *p = curproc; + struct trapframe *tf; + uintptr_t pc, sp; + volatile uint16_t *flags = + (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; +#ifdef notyet /* XXX signal stack */ + uintptr_t oldcontext; + size_t s1, s2; +#endif + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = tf->srr0; + sp = tf->fixreg[1]; + +#ifdef notyet /* XXX signal stack */ + oldcontext = lwp->lwp_oldcontext; + s1 = sizeof (struct xframe) + 2 * sizeof (long); + s2 = s1 + sizeof (siginfo_t); +#endif + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t)pc; + *fpstack++ = 0; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + if (SV_PROC_FLAG(p, SV_ILP32)) { + pc = dtrace_fuword32((void *)sp); + } + else { + pc = dtrace_fuword64((void *)sp); + } + } + + while (pc != 0) { + *pcstack++ = (uint64_t)pc; + *fpstack++ = sp; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + + if (sp == 0) + break; + +#ifdef notyet /* XXX signal stack */ + if (oldcontext == sp + s1 || oldcontext == sp + s2) { + ucontext_t *ucp = (ucontext_t *)oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fulword(&gregs[REG_FP]); + pc = dtrace_fulword(&gregs[REG_PC]); + + oldcontext = dtrace_fulword(&ucp->uc_link); + } else +#endif /* XXX */ + { + if (SV_PROC_FLAG(p, SV_ILP32)) { + pc = dtrace_fuword32((void *)(sp + RETURN_OFFSET)); + sp = dtrace_fuword32((void *)sp); + } + else { + pc = dtrace_fuword64((void *)(sp + RETURN_OFFSET64)); + sp = dtrace_fuword64((void *)sp); + } + } + + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } + } + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +/*ARGSUSED*/ +uint64_t +dtrace_getarg(int arg, int aframes) +{ + uintptr_t val; + uintptr_t *fp = (uintptr_t *)__builtin_frame_address(0); + uintptr_t *stack; + int i; + + /* + * A total of 8 arguments are passed via registers; any argument with + * index of 7 or lower is therefore in a register. + */ + int inreg = 7; + + for (i = 1; i <= aframes; i++) { + fp = (uintptr_t *)*fp; + + /* + * On ppc32 trapexit() is the immediately following label. On + * ppc64 AIM trapexit() follows a nop. + */ +#ifdef __powerpc64__ + if ((long)(fp[2]) + 4 == (long)trapexit) { +#else + if ((long)(fp[1]) == (long)trapexit) { +#endif + /* + * In the case of powerpc, we will use the pointer to the regs + * structure that was pushed when we took the trap. To get this + * structure, we must increment beyond the frame structure. If the + * argument that we're seeking is passed on the stack, we'll pull + * the true stack pointer out of the saved registers and decrement + * our argument by the number of arguments passed in registers; if + * the argument we're seeking is passed in regsiters, we can just + * load it directly. + */ +#ifdef __powerpc64__ + struct reg *rp = (struct reg *)((uintptr_t)fp[0] + 48); +#else + struct reg *rp = (struct reg *)((uintptr_t)fp[0] + 8); +#endif + + if (arg <= inreg) { + stack = &rp->fixreg[3]; + } else { + stack = (uintptr_t *)(rp->fixreg[1]); + arg -= inreg; + } + goto load; + } + + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- the provider simply called dtrace_probe() + * directly. As this is the case, we need to shift the argument + * that we're looking for: the probe ID is the first argument to + * dtrace_probe(), so the argument n will actually be found where + * one would expect to find argument (n + 1). + */ + arg++; + + if (arg <= inreg) { + /* + * This shouldn't happen. If the argument is passed in a + * register then it should have been, well, passed in a + * register... + */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + arg -= (inreg + 1); + stack = fp + 2; + +load: + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + val = stack[arg]; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + + return (val); +} + +int +dtrace_getstackdepth(int aframes) +{ + int depth = 0; + uintptr_t osp, sp; + vm_offset_t callpc; + + osp = PAGE_SIZE; + sp = (uintptr_t)__builtin_frame_address(0); + for(;;) { + if (sp <= osp) + break; + + if (!dtrace_sp_inkernel(sp)) + break; + + depth++; + osp = sp; + dtrace_next_sp_pc(sp, &sp, NULL); + } + if (depth < aframes) + return (0); + + return (depth - aframes); +} + +ulong_t +dtrace_getreg(struct trapframe *rp, uint_t reg) +{ + if (reg < 32) + return (rp->fixreg[reg]); + + switch (reg) { + case 32: + return (rp->lr); + case 33: + return (rp->cr); + case 34: + return (rp->xer); + case 35: + return (rp->ctr); + case 36: + return (rp->srr0); + case 37: + return (rp->srr1); + case 38: + return (rp->exc); + default: + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } +} + +static int +dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) +{ + ASSERT(INKERNEL(kaddr) && kaddr + size >= kaddr); + + if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = uaddr; + return (0); + } + + return (1); +} + +void +dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + if (copyin((const void *)uaddr, (void *)kaddr, size)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + } +} + +void +dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) { + if (copyout((const void *)kaddr, (void *)uaddr, size)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + } + } +} + +void +dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + size_t actual; + int error; + + if (dtrace_copycheck(uaddr, kaddr, size)) { + error = copyinstr((const void *)uaddr, (void *)kaddr, + size, &actual); + + /* ENAMETOOLONG is not a fault condition. */ + if (error && error != ENAMETOOLONG) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + } + } +} + +/* + * The bulk of this function could be replaced to match dtrace_copyinstr() + * if we ever implement a copyoutstr(). + */ +void +dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + size_t len; + + if (dtrace_copycheck(uaddr, kaddr, size)) { + len = strlen((const char *)kaddr); + if (len > size) + len = size; + + if (copyout((const void *)kaddr, (void *)uaddr, len)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + } + } +} + +uint8_t +dtrace_fuword8(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (fubyte(uaddr)); +} + +uint16_t +dtrace_fuword16(void *uaddr) +{ + uint16_t ret = 0; + + if (dtrace_copycheck((uintptr_t)uaddr, (uintptr_t)&ret, sizeof(ret))) { + if (copyin((const void *)uaddr, (void *)&ret, sizeof(ret))) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + } + } + return ret; +} + +uint32_t +dtrace_fuword32(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (fuword32(uaddr)); +} + +uint64_t +dtrace_fuword64(void *uaddr) +{ + uint64_t ret = 0; + + if (dtrace_copycheck((uintptr_t)uaddr, (uintptr_t)&ret, sizeof(ret))) { + if (copyin((const void *)uaddr, (void *)&ret, sizeof(ret))) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + } + } + return ret; +} + +uintptr_t +dtrace_fulword(void *uaddr) +{ + uintptr_t ret = 0; + + if (dtrace_copycheck((uintptr_t)uaddr, (uintptr_t)&ret, sizeof(ret))) { + if (copyin((const void *)uaddr, (void *)&ret, sizeof(ret))) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + } + } + return ret; +} diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c b/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c new file mode 100644 index 000000000000..ed171f1135e9 --- /dev/null +++ b/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c @@ -0,0 +1,367 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/kmem.h> +#include <sys/smp.h> +#include <sys/dtrace_impl.h> +#include <sys/dtrace_bsd.h> +#include <machine/clock.h> +#include <machine/frame.h> +#include <machine/trap.h> +#include <vm/pmap.h> + +#define DELAYBRANCH(x) ((int)(x) < 0) + +extern dtrace_id_t dtrace_probeid_error; +extern int (*dtrace_invop_jump_addr)(struct trapframe *); + +extern void dtrace_getnanotime(struct timespec *tsp); + +int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t); +void dtrace_invop_init(void); +void dtrace_invop_uninit(void); + +typedef struct dtrace_invop_hdlr { + int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t); + struct dtrace_invop_hdlr *dtih_next; +} dtrace_invop_hdlr_t; + +dtrace_invop_hdlr_t *dtrace_invop_hdlr; + +int +dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t arg0) +{ + dtrace_invop_hdlr_t *hdlr; + int rval; + + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) + if ((rval = hdlr->dtih_func(addr, frame, arg0)) != 0) + return (rval); + + return (0); +} + +void +dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr; + + hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); + hdlr->dtih_func = func; + hdlr->dtih_next = dtrace_invop_hdlr; + dtrace_invop_hdlr = hdlr; +} + +void +dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL; + + for (;;) { + if (hdlr == NULL) + panic("attempt to remove non-existent invop handler"); + + if (hdlr->dtih_func == func) + break; + + prev = hdlr; + hdlr = hdlr->dtih_next; + } + + if (prev == NULL) { + ASSERT(dtrace_invop_hdlr == hdlr); + dtrace_invop_hdlr = hdlr->dtih_next; + } else { + ASSERT(dtrace_invop_hdlr != hdlr); + prev->dtih_next = hdlr->dtih_next; + } + + kmem_free(hdlr, 0); +} + + +/*ARGSUSED*/ +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + /* + * No toxic regions? + */ +} + +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) +{ + cpuset_t cpus; + + if (cpu == DTRACE_CPUALL) + cpus = all_cpus; + else + CPU_SETOF(cpu, &cpus); + + smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func, + smp_no_rendezvous_barrier, arg); +} + +static void +dtrace_sync_func(void) +{ +} + +void +dtrace_sync(void) +{ + dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); +} + +static int64_t tgt_cpu_tsc; +static int64_t hst_cpu_tsc; +static int64_t timebase_skew[MAXCPU]; +static uint64_t nsec_scale; + +/* See below for the explanation of this macro. */ +/* This is taken from the amd64 dtrace_subr, to provide a synchronized timer + * between multiple processors in dtrace. Since PowerPC Timebases can be much + * lower than x86, the scale shift is 26 instead of 28, allowing for a 15.63MHz + * timebase. + */ +#define SCALE_SHIFT 26 + +static void +dtrace_gethrtime_init_cpu(void *arg) +{ + uintptr_t cpu = (uintptr_t) arg; + + if (cpu == curcpu) + tgt_cpu_tsc = mftb(); + else + hst_cpu_tsc = mftb(); +} + +static void +dtrace_gethrtime_init(void *arg) +{ + struct pcpu *pc; + uint64_t tb_f; + cpuset_t map; + int i; + + tb_f = cpu_tickrate(); + + /* + * The following line checks that nsec_scale calculated below + * doesn't overflow 32-bit unsigned integer, so that it can multiply + * another 32-bit integer without overflowing 64-bit. + * Thus minimum supported Timebase frequency is 15.63MHz. + */ + KASSERT(tb_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("Timebase frequency is too low")); + + /* + * We scale up NANOSEC/tb_f ratio to preserve as much precision + * as possible. + * 2^26 factor was chosen quite arbitrarily from practical + * considerations: + * - it supports TSC frequencies as low as 15.63MHz (see above); + */ + nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tb_f; + + /* The current CPU is the reference one. */ + sched_pin(); + timebase_skew[curcpu] = 0; + CPU_FOREACH(i) { + if (i == curcpu) + continue; + + pc = pcpu_find(i); + CPU_SETOF(PCPU_GET(cpuid), &map); + CPU_SET(pc->pc_cpuid, &map); + + smp_rendezvous_cpus(map, NULL, + dtrace_gethrtime_init_cpu, + smp_no_rendezvous_barrier, (void *)(uintptr_t) i); + + timebase_skew[i] = tgt_cpu_tsc - hst_cpu_tsc; + } + sched_unpin(); +} +#ifdef EARLY_AP_STARTUP +SYSINIT(dtrace_gethrtime_init, SI_SUB_DTRACE, SI_ORDER_ANY, + dtrace_gethrtime_init, NULL); +#else +SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, + NULL); +#endif + +/* + * DTrace needs a high resolution time function which can + * be called from a probe context and guaranteed not to have + * instrumented with probes itself. + * + * Returns nanoseconds since boot. + */ +uint64_t +dtrace_gethrtime() +{ + uint64_t timebase; + uint32_t lo; + uint32_t hi; + + /* + * We split timebase value into lower and higher 32-bit halves and separately + * scale them with nsec_scale, then we scale them down by 2^28 + * (see nsec_scale calculations) taking into account 32-bit shift of + * the higher half and finally add. + */ + timebase = mftb() - timebase_skew[curcpu]; + lo = timebase; + hi = timebase >> 32; + return (((lo * nsec_scale) >> SCALE_SHIFT) + + ((hi * nsec_scale) << (32 - SCALE_SHIFT))); +} + +uint64_t +dtrace_gethrestime(void) +{ + struct timespec curtime; + + dtrace_getnanotime(&curtime); + + return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec); +} + +/* Function to handle DTrace traps during probes. See powerpc/powerpc/trap.c */ +int +dtrace_trap(struct trapframe *frame, u_int type) +{ + uint16_t nofault; + + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in its per-cpu flags to indicate that it doesn't + * want to fault. On returning from the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * Check if DTrace has enabled 'no-fault' mode: + */ + sched_pin(); + nofault = cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT; + sched_unpin(); + if (nofault) { + KASSERT((frame->srr1 & PSL_EE) == 0, ("interrupts enabled")); + /* + * There are only a couple of trap types that are expected. + * All the rest will be handled in the usual way. + */ + switch (type) { + /* Page fault. */ + case EXC_DSI: + case EXC_DSE: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = frame->dar; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->srr0 += sizeof(int); + return (1); + case EXC_ISI: + case EXC_ISE: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = frame->srr0; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->srr0 += sizeof(int); + return (1); + default: + /* Handle all other traps in the usual way. */ + break; + } + } + + /* Handle the trap in the usual way. */ + return (0); +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fault, int fltoffs, uintptr_t illval) +{ + + dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state, + (uintptr_t)epid, + (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs); +} + +static int +dtrace_invop_start(struct trapframe *frame) +{ + + switch (dtrace_invop(frame->srr0, frame, frame->fixreg[3])) { + case DTRACE_INVOP_JUMP: + break; + case DTRACE_INVOP_BCTR: + frame->srr0 = frame->ctr; + break; + case DTRACE_INVOP_BLR: + frame->srr0 = frame->lr; + break; + case DTRACE_INVOP_MFLR_R0: + frame->fixreg[0] = frame->lr; + frame->srr0 = frame->srr0 + 4; + break; + default: + return (-1); + } + return (0); +} + +void dtrace_invop_init(void) +{ + dtrace_invop_jump_addr = dtrace_invop_start; +} + +void dtrace_invop_uninit(void) +{ + dtrace_invop_jump_addr = 0; +} diff --git a/sys/cddl/dev/dtrace/powerpc/regset.h b/sys/cddl/dev/dtrace/powerpc/regset.h new file mode 100644 index 000000000000..64973885fc96 --- /dev/null +++ b/sys/cddl/dev/dtrace/powerpc/regset.h @@ -0,0 +1,63 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +#ifndef _REGSET_H +#define _REGSET_H + +/* + * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * XXXDTRACE: define registers properly + */ + +#if 0 +#define REG_PC PC +#define REG_FP EBP +#define REG_SP SP +#define REG_PS EFL +#define REG_R0 EAX +#define REG_R1 EDX +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _REGSET_H */ + diff --git a/sys/cddl/dev/dtrace/riscv/dtrace_asm.S b/sys/cddl/dev/dtrace/riscv/dtrace_asm.S new file mode 100644 index 000000000000..eeaf64061675 --- /dev/null +++ b/sys/cddl/dev/dtrace/riscv/dtrace_asm.S @@ -0,0 +1,177 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Portions Copyright 2016 Ruslan Bukin <br@bsdpad.com> + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#define _ASM +#define _LOCORE + +#include <sys/cpuvar_defs.h> +#include <sys/dtrace.h> + +#include <machine/riscvreg.h> +#include <machine/asm.h> + +#include "assym.inc" + +/* +void dtrace_membar_producer(void) +*/ +ENTRY(dtrace_membar_producer) + RET +END(dtrace_membar_producer) + +/* +void dtrace_membar_consumer(void) +*/ +ENTRY(dtrace_membar_consumer) + RET +END(dtrace_membar_consumer) + +/* +dtrace_icookie_t dtrace_interrupt_disable(void) +*/ +ENTRY(dtrace_interrupt_disable) + csrrci a0, sstatus, (SSTATUS_SIE) + andi a0, a0, (SSTATUS_SIE) + RET +END(dtrace_interrupt_disable) + +/* +void dtrace_interrupt_enable(dtrace_icookie_t cookie) +*/ +ENTRY(dtrace_interrupt_enable) + csrs sstatus, a0 + RET +END(dtrace_interrupt_enable) +/* +uint8_t +dtrace_fuword8_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword8_nocheck) + lb a0, 0(a0) + RET +END(dtrace_fuword8_nocheck) + +/* +uint16_t +dtrace_fuword16_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword16_nocheck) + lh a0, 0(a0) + RET +END(dtrace_fuword16_nocheck) + +/* +uint32_t +dtrace_fuword32_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword32_nocheck) + lw a0, 0(a0) + RET +END(dtrace_fuword32_nocheck) + +/* +uint64_t +dtrace_fuword64_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword64_nocheck) + ld a0, 0(a0) + RET +END(dtrace_fuword64_nocheck) + +/* +void +dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) +*/ +ENTRY(dtrace_copy) + beqz a2, 2f /* If len == 0 then skip loop */ +1: + lb a4, 0(a0) /* Load from uaddr */ + addi a0, a0, 1 + sb a4, 0(a1) /* Store in kaddr */ + addi a1, a1, 1 + addi a2, a2, -1 /* len-- */ + bnez a2, 1b +2: + RET +END(dtrace_copy) + +/* +void +dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +XXX: Check for flags? +*/ +ENTRY(dtrace_copystr) + beqz a2, 2f /* If len == 0 then skip loop */ + lb a4, 0(a0) /* Load from uaddr */ + addi a0, a0, 1 + sb a4, 0(a1) /* Store in kaddr */ + addi a1, a1, 1 + beqz a4, 2f /* If == 0 then break */ + addi a2, a2, -1 /* len-- */ + bnez a2, 1b +2: + RET +END(dtrace_copystr) + +/* +uintptr_t +dtrace_caller(int aframes) +*/ +ENTRY(dtrace_caller) + li a0, -1 + RET +END(dtrace_caller) + +/* +uint32_t +dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) +*/ +ENTRY(dtrace_cas32) +1: lr.w a3, 0(a0) /* Load target */ + bne a3, a1, 2f /* *target != cmp ? return */ + sc.w a4, a2, 0(a0) /* Store new to target */ + bnez a4, 1b /* Try again if store not succeed */ +2: mv a0, a3 /* Return the value loaded from target */ + RET +END(dtrace_cas32) + +/* +void * +dtrace_casptr(volatile void *target, volatile void *cmp, volatile void *new) +*/ +ENTRY(dtrace_casptr) +1: lr.d a3, 0(a0) /* Load target */ + bne a3, a1, 2f /* *target != cmp ? return */ + sc.d a4, a2, 0(a0) /* Store new to target */ + bnez a4, 1b /* Try again if store not succeed */ +2: mv a0, a3 /* Return the value loaded from target */ + RET +END(dtrace_casptr) diff --git a/sys/cddl/dev/dtrace/riscv/dtrace_isa.c b/sys/cddl/dev/dtrace/riscv/dtrace_isa.c new file mode 100644 index 000000000000..955d2cb10f65 --- /dev/null +++ b/sys/cddl/dev/dtrace/riscv/dtrace_isa.c @@ -0,0 +1,394 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Portions Copyright 2016 Ruslan Bukin <br@bsdpad.com> + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#include <sys/cdefs.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/stack.h> +#include <sys/pcpu.h> + +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/reg.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> + +#include <machine/atomic.h> +#include <machine/db_machdep.h> +#include <machine/md_var.h> +#include <machine/stack.h> +#include <ddb/db_sym.h> +#include <ddb/ddb.h> +#include <sys/kdb.h> + +#include "regset.h" + +/* + * Wee need some reasonable default to prevent backtrace code + * from wandering too far + */ +#define MAX_FUNCTION_SIZE 0x10000 +#define MAX_PROLOGUE_SIZE 0x100 +#define MAX_USTACK_DEPTH 2048 + +uint8_t dtrace_fuword8_nocheck(void *); +uint16_t dtrace_fuword16_nocheck(void *); +uint32_t dtrace_fuword32_nocheck(void *); +uint64_t dtrace_fuword64_nocheck(void *); + +void +dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, + uint32_t *intrpc) +{ + struct unwind_state state; + int scp_offset; + register_t sp; + int depth; + + depth = 0; + + if (intrpc != 0) { + pcstack[depth++] = (pc_t) intrpc; + } + + aframes++; + + __asm __volatile("mv %0, sp" : "=&r" (sp)); + + state.fp = (uint64_t)__builtin_frame_address(0); + state.sp = sp; + state.pc = (uint64_t)dtrace_getpcstack; + + while (depth < pcstack_limit) { + if (unwind_frame(&state)) + break; + + if (!INKERNEL(state.pc) || !INKERNEL(state.fp)) + break; + + /* + * NB: Unlike some other architectures, we don't need to + * explicitly insert cpu_dtrace_caller as it appears in the + * normal kernel stack trace rather than a special trap frame. + */ + if (aframes > 0) { + aframes--; + } else { + pcstack[depth++] = state.pc; + } + + } + + for (; depth < pcstack_limit; depth++) { + pcstack[depth] = 0; + } +} + +static int +dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc, + uintptr_t fp) +{ + volatile uint16_t *flags; + uintptr_t oldfp; + int ret; + + ret = 0; + flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + + ASSERT(pcstack == NULL || pcstack_limit > 0); + + while (pc != 0) { + /* + * We limit the number of times we can go around this + * loop to account for a circular stack. + */ + if (ret++ >= MAX_USTACK_DEPTH) { + *flags |= CPU_DTRACE_BADSTACK; + cpu_core[curcpu].cpuc_dtrace_illval = fp; + break; + } + + if (pcstack != NULL) { + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (fp == 0) + break; + + pc = dtrace_fuword64((void *)(fp + + offsetof(struct riscv_frame, f_retaddr))); + fp = dtrace_fuword64((void *)fp); + + if (fp == oldfp) { + *flags |= CPU_DTRACE_BADSTACK; + cpu_core[curcpu].cpuc_dtrace_illval = fp; + break; + } + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) +{ + volatile uint16_t *flags; + struct trapframe *tf; + uintptr_t pc, sp, fp; + proc_t *p; + int n; + + p = curproc; + flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (p == NULL || (tf = curthread->td_frame) == NULL) + goto zero; + + *pcstack++ = (uint64_t)p->p_pid; + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = tf->tf_sepc; + sp = tf->tf_sp; + fp = tf->tf_s[0]; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + + *pcstack++ = (uint64_t)pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = tf->tf_ra; + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp); + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0; +} + +int +dtrace_getustackdepth(void) +{ + + printf("IMPLEMENT ME: %s\n", __func__); + + return (0); +} + +void +dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) +{ + + printf("IMPLEMENT ME: %s\n", __func__); +} + +/*ARGSUSED*/ +uint64_t +dtrace_getarg(int arg, int aframes) +{ + + printf("IMPLEMENT ME: %s\n", __func__); + + return (0); +} + +int +dtrace_getstackdepth(int aframes) +{ + struct unwind_state state; + int scp_offset; + register_t sp; + int depth; + int done; + + depth = 1; + done = 0; + + __asm __volatile("mv %0, sp" : "=&r" (sp)); + + state.fp = (uint64_t)__builtin_frame_address(0); + state.sp = sp; + state.pc = (uint64_t)dtrace_getstackdepth; + + do { + done = unwind_frame(&state); + if (!INKERNEL(state.pc) || !INKERNEL(state.fp)) + break; + depth++; + } while (!done); + + if (depth < aframes) + return (0); + else + return (depth - aframes); +} + +ulong_t +dtrace_getreg(struct trapframe *rp, uint_t reg) +{ + + printf("IMPLEMENT ME: %s\n", __func__); + + return (0); +} + +static int +dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) +{ + + if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = uaddr; + return (0); + } + + return (1); +} + +void +dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(uaddr, kaddr, size); +} + +void +dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(kaddr, uaddr, size); +} + +void +dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(uaddr, kaddr, size, flags); +} + +void +dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(kaddr, uaddr, size, flags); +} + +uint8_t +dtrace_fuword8(void *uaddr) +{ + + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + + return (dtrace_fuword8_nocheck(uaddr)); +} + +uint16_t +dtrace_fuword16(void *uaddr) +{ + + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + + return (dtrace_fuword16_nocheck(uaddr)); +} + +uint32_t +dtrace_fuword32(void *uaddr) +{ + + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + + return (dtrace_fuword32_nocheck(uaddr)); +} + +uint64_t +dtrace_fuword64(void *uaddr) +{ + + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + + return (dtrace_fuword64_nocheck(uaddr)); +} diff --git a/sys/cddl/dev/dtrace/riscv/dtrace_subr.c b/sys/cddl/dev/dtrace/riscv/dtrace_subr.c new file mode 100644 index 000000000000..68d98db30c83 --- /dev/null +++ b/sys/cddl/dev/dtrace/riscv/dtrace_subr.c @@ -0,0 +1,310 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Portions Copyright 2016-2018 Ruslan Bukin <br@bsdpad.com> + * + * $FreeBSD$ + * + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/kmem.h> +#include <sys/smp.h> +#include <sys/dtrace_impl.h> +#include <sys/dtrace_bsd.h> +#include <machine/vmparam.h> +#include <machine/encoding.h> +#include <machine/riscvreg.h> +#include <machine/clock.h> +#include <machine/frame.h> +#include <machine/trap.h> +#include <vm/pmap.h> + +extern dtrace_id_t dtrace_probeid_error; +extern int (*dtrace_invop_jump_addr)(struct trapframe *); +extern void dtrace_getnanotime(struct timespec *tsp); + +int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t); +void dtrace_invop_init(void); +void dtrace_invop_uninit(void); + +typedef struct dtrace_invop_hdlr { + int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t); + struct dtrace_invop_hdlr *dtih_next; +} dtrace_invop_hdlr_t; + +dtrace_invop_hdlr_t *dtrace_invop_hdlr; + +int +dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) +{ + dtrace_invop_hdlr_t *hdlr; + int rval; + + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) + if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) + return (rval); + + return (0); +} + +void +dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr; + + hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); + hdlr->dtih_func = func; + hdlr->dtih_next = dtrace_invop_hdlr; + dtrace_invop_hdlr = hdlr; +} + +void +dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr, *prev; + + hdlr = dtrace_invop_hdlr; + prev = NULL; + + for (;;) { + if (hdlr == NULL) + panic("attempt to remove non-existent invop handler"); + + if (hdlr->dtih_func == func) + break; + + prev = hdlr; + hdlr = hdlr->dtih_next; + } + + if (prev == NULL) { + ASSERT(dtrace_invop_hdlr == hdlr); + dtrace_invop_hdlr = hdlr->dtih_next; + } else { + ASSERT(dtrace_invop_hdlr != hdlr); + prev->dtih_next = hdlr->dtih_next; + } + + kmem_free(hdlr, 0); +} + +/*ARGSUSED*/ +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + + (*func)(0, (uintptr_t)VM_MIN_KERNEL_ADDRESS); +} + +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) +{ + cpuset_t cpus; + + if (cpu == DTRACE_CPUALL) + cpus = all_cpus; + else + CPU_SETOF(cpu, &cpus); + + smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func, + smp_no_rendezvous_barrier, arg); +} + +static void +dtrace_sync_func(void) +{ + +} + +void +dtrace_sync(void) +{ + + dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); +} + +/* + * DTrace needs a high resolution time function which can + * be called from a probe context and guaranteed not to have + * instrumented with probes itself. + * + * Returns nanoseconds since boot. + */ +uint64_t +dtrace_gethrtime() +{ + struct timespec curtime; + + nanouptime(&curtime); + + return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec); + +} + +uint64_t +dtrace_gethrestime(void) +{ + struct timespec current_time; + + dtrace_getnanotime(¤t_time); + + return (current_time.tv_sec * 1000000000UL + current_time.tv_nsec); +} + +/* Function to handle DTrace traps during probes. See riscv/riscv/trap.c */ +int +dtrace_trap(struct trapframe *frame, u_int type) +{ + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in its per-cpu flags to indicate that it doesn't + * want to fault. On returning from the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * Check if DTrace has enabled 'no-fault' mode: + * + */ + + if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) { + /* + * There are only a couple of trap types that are expected. + * All the rest will be handled in the usual way. + */ + switch (type) { + case EXCP_FAULT_LOAD: + case EXCP_FAULT_STORE: + case EXCP_FAULT_FETCH: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = 0; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->tf_sepc += 4; + + return (1); + default: + /* Handle all other traps in the usual way. */ + break; + } + } + + /* Handle the trap in the usual way. */ + return (0); +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fault, int fltoffs, uintptr_t illval) +{ + + dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state, + (uintptr_t)epid, + (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs); +} + +static int +match_opcode(uint32_t insn, int match, int mask) +{ + + if (((insn ^ match) & mask) == 0) + return (1); + + return (0); +} + +static int +dtrace_invop_start(struct trapframe *frame) +{ + register_t *sp; + uint32_t uimm; + uint32_t imm; + int invop; + + invop = dtrace_invop(frame->tf_sepc, frame, frame->tf_sepc); + + if (match_opcode(invop, (MATCH_SD | RS2_RA | RS1_SP), + (MASK_SD | RS2_MASK | RS1_MASK))) { + /* Non-compressed store of ra to sp */ + imm = (invop >> 7) & 0x1f; + imm |= ((invop >> 25) & 0x7f) << 5; + sp = (register_t *)((uint8_t *)frame->tf_sp + imm); + *sp = frame->tf_ra; + frame->tf_sepc += INSN_SIZE; + return (0); + } + + if (match_opcode(invop, (MATCH_JALR | (X_RA << RS1_SHIFT)), + (MASK_JALR | RD_MASK | RS1_MASK | IMM_MASK))) { + /* Non-compressed ret */ + frame->tf_sepc = frame->tf_ra; + return (0); + } + + if (match_opcode(invop, (MATCH_C_SDSP | RS2_C_RA), + (MASK_C_SDSP | RS2_C_MASK))) { + /* 'C'-compressed store of ra to sp */ + uimm = ((invop >> 10) & 0x7) << 3; + uimm |= ((invop >> 7) & 0x7) << 6; + sp = (register_t *)((uint8_t *)frame->tf_sp + uimm); + *sp = frame->tf_ra; + frame->tf_sepc += INSN_C_SIZE; + return (0); + } + + if (match_opcode(invop, (MATCH_C_JR | (X_RA << RD_SHIFT)), + (MASK_C_JR | RD_MASK))) { + /* 'C'-compressed ret */ + frame->tf_sepc = frame->tf_ra; + return (0); + } + + return (-1); +} + +void +dtrace_invop_init(void) +{ + + dtrace_invop_jump_addr = dtrace_invop_start; +} + +void +dtrace_invop_uninit(void) +{ + + dtrace_invop_jump_addr = 0; +} diff --git a/sys/cddl/dev/dtrace/riscv/regset.h b/sys/cddl/dev/dtrace/riscv/regset.h new file mode 100644 index 000000000000..f99b48f8354f --- /dev/null +++ b/sys/cddl/dev/dtrace/riscv/regset.h @@ -0,0 +1,51 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +#ifndef _REGSET_H +#define _REGSET_H + +/* + * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Place here */ + +#ifdef __cplusplus +} +#endif + +#endif /* _REGSET_H */ diff --git a/sys/cddl/dev/dtrace/x86/dis_tables.c b/sys/cddl/dev/dtrace/x86/dis_tables.c new file mode 100644 index 000000000000..d130b3ac335e --- /dev/null +++ b/sys/cddl/dev/dtrace/x86/dis_tables.c @@ -0,0 +1,5597 @@ +/* + * + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright 2016 Joyent, Inc. + */ + +/* + * Copyright (c) 2010, Intel Corporation. + * All rights reserved. + */ + +/* Copyright (c) 1988 AT&T */ +/* All Rights Reserved */ + +/* + * $FreeBSD$ + */ + +#include "dis_tables.h" + +/* BEGIN CSTYLED */ + +/* + * Disassembly begins in dis_distable, which is equivalent to the One-byte + * Opcode Map in the Intel IA32 ISA Reference (page A-6 in my copy). The + * decoding loops then traverse out through the other tables as necessary to + * decode a given instruction. + * + * The behavior of this file can be controlled by one of the following flags: + * + * DIS_TEXT Include text for disassembly + * DIS_MEM Include memory-size calculations + * + * Either or both of these can be defined. + * + * This file is not, and will never be, cstyled. If anything, the tables should + * be taken out another tab stop or two so nothing overlaps. + */ + +/* + * These functions must be provided for the consumer to do disassembly. + */ +#ifdef DIS_TEXT +extern char *strncpy(char *, const char *, size_t); +extern size_t strlen(const char *); +extern int strcmp(const char *, const char *); +extern int strncmp(const char *, const char *, size_t); +extern size_t strlcat(char *, const char *, size_t); +#endif + + +#define TERM 0 /* used to indicate that the 'indirect' */ + /* field terminates - no pointer. */ + +/* Used to decode instructions. */ +typedef struct instable { + struct instable *it_indirect; /* for decode op codes */ + uchar_t it_adrmode; +#ifdef DIS_TEXT + char it_name[NCPS]; + uint_t it_suffix:1; /* mnem + "w", "l", or "d" */ +#endif +#ifdef DIS_MEM + uint_t it_size:16; +#endif + uint_t it_invalid64:1; /* opcode invalid in amd64 */ + uint_t it_always64:1; /* 64 bit when in 64 bit mode */ + uint_t it_invalid32:1; /* invalid in IA32 */ + uint_t it_stackop:1; /* push/pop stack operation */ + uint_t it_vexwoxmm:1; /* VEX instructions that don't use XMM/YMM */ + uint_t it_avxsuf:1; /* AVX suffix required */ +} instable_t; + +/* + * Instruction formats. + */ +enum { + UNKNOWN, + MRw, + IMlw, + IMw, + IR, + OA, + AO, + MS, + SM, + Mv, + Mw, + M, /* register or memory */ + MG9, /* register or memory in group 9 (prefix optional) */ + Mb, /* register or memory, always byte sized */ + MO, /* memory only (no registers) */ + PREF, + SWAPGS_RDTSCP, + MONITOR_MWAIT, + R, + RA, + SEG, + MR, + RM, + RM_66r, /* RM, but with a required 0x66 prefix */ + IA, + MA, + SD, + AD, + SA, + D, + INM, + SO, + BD, + I, + P, + V, + DSHIFT, /* for double shift that has an 8-bit immediate */ + U, + OVERRIDE, + NORM, /* instructions w/o ModR/M byte, no memory access */ + IMPLMEM, /* instructions w/o ModR/M byte, implicit mem access */ + O, /* for call */ + JTAB, /* jump table */ + IMUL, /* for 186 iimul instr */ + CBW, /* so data16 can be evaluated for cbw and variants */ + MvI, /* for 186 logicals */ + ENTER, /* for 186 enter instr */ + RMw, /* for 286 arpl instr */ + Ib, /* for push immediate byte */ + F, /* for 287 instructions */ + FF, /* for 287 instructions */ + FFC, /* for 287 instructions */ + DM, /* 16-bit data */ + AM, /* 16-bit addr */ + LSEG, /* for 3-bit seg reg encoding */ + MIb, /* for 386 logicals */ + SREG, /* for 386 special registers */ + PREFIX, /* a REP instruction prefix */ + LOCK, /* a LOCK instruction prefix */ + INT3, /* The int 3 instruction, which has a fake operand */ + INTx, /* The normal int instruction, with explicit int num */ + DSHIFTcl, /* for double shift that implicitly uses %cl */ + CWD, /* so data16 can be evaluated for cwd and variants */ + RET, /* single immediate 16-bit operand */ + MOVZ, /* for movs and movz, with different size operands */ + CRC32, /* for crc32, with different size operands */ + XADDB, /* for xaddb */ + MOVSXZ, /* AMD64 mov sign extend 32 to 64 bit instruction */ + MOVBE, /* movbe instruction */ + +/* + * MMX/SIMD addressing modes. + */ + + MMO, /* Prefixable MMX/SIMD-Int mm/mem -> mm */ + MMOIMPL, /* Prefixable MMX/SIMD-Int mm -> mm (mem) */ + MMO3P, /* Prefixable MMX/SIMD-Int mm -> r32,imm8 */ + MMOM3, /* Prefixable MMX/SIMD-Int mm -> r32 */ + MMOS, /* Prefixable MMX/SIMD-Int mm -> mm/mem */ + MMOMS, /* Prefixable MMX/SIMD-Int mm -> mem */ + MMOPM, /* MMX/SIMD-Int mm/mem -> mm,imm8 */ + MMOPM_66o, /* MMX/SIMD-Int 0x66 optional mm/mem -> mm,imm8 */ + MMOPRM, /* Prefixable MMX/SIMD-Int r32/mem -> mm,imm8 */ + MMOSH, /* Prefixable MMX mm,imm8 */ + MM, /* MMX/SIMD-Int mm/mem -> mm */ + MMS, /* MMX/SIMD-Int mm -> mm/mem */ + MMSH, /* MMX mm,imm8 */ + XMMO, /* Prefixable SIMD xmm/mem -> xmm */ + XMMOS, /* Prefixable SIMD xmm -> xmm/mem */ + XMMOPM, /* Prefixable SIMD xmm/mem w/to xmm,imm8 */ + XMMOMX, /* Prefixable SIMD mm/mem -> xmm */ + XMMOX3, /* Prefixable SIMD xmm -> r32 */ + XMMOXMM, /* Prefixable SIMD xmm/mem -> mm */ + XMMOM, /* Prefixable SIMD xmm -> mem */ + XMMOMS, /* Prefixable SIMD mem -> xmm */ + XMM, /* SIMD xmm/mem -> xmm */ + XMM_66r, /* SIMD 0x66 prefix required xmm/mem -> xmm */ + XMM_66o, /* SIMD 0x66 prefix optional xmm/mem -> xmm */ + XMMXIMPL, /* SIMD xmm -> xmm (mem) */ + XMM3P, /* SIMD xmm -> r32,imm8 */ + XMM3PM_66r, /* SIMD 0x66 prefix required xmm -> r32/mem,imm8 */ + XMMP, /* SIMD xmm/mem w/to xmm,imm8 */ + XMMP_66o, /* SIMD 0x66 prefix optional xmm/mem w/to xmm,imm8 */ + XMMP_66r, /* SIMD 0x66 prefix required xmm/mem w/to xmm,imm8 */ + XMMPRM, /* SIMD r32/mem -> xmm,imm8 */ + XMMPRM_66r, /* SIMD 0x66 prefix required r32/mem -> xmm,imm8 */ + XMMS, /* SIMD xmm -> xmm/mem */ + XMMM, /* SIMD mem -> xmm */ + XMMM_66r, /* SIMD 0x66 prefix required mem -> xmm */ + XMMMS, /* SIMD xmm -> mem */ + XMM3MX, /* SIMD r32/mem -> xmm */ + XMM3MXS, /* SIMD xmm -> r32/mem */ + XMMSH, /* SIMD xmm,imm8 */ + XMMXM3, /* SIMD xmm/mem -> r32 */ + XMMX3, /* SIMD xmm -> r32 */ + XMMXMM, /* SIMD xmm/mem -> mm */ + XMMMX, /* SIMD mm -> xmm */ + XMMXM, /* SIMD xmm -> mm */ + XMMX2I, /* SIMD xmm -> xmm, imm, imm */ + XMM2I, /* SIMD xmm, imm, imm */ + XMMFENCE, /* SIMD lfence or mfence */ + XMMSFNC, /* SIMD sfence (none or mem) */ + XGETBV_XSETBV, + VEX_NONE, /* VEX no operand */ + VEX_MO, /* VEX mod_rm -> implicit reg */ + VEX_RMrX, /* VEX VEX.vvvv, mod_rm -> mod_reg */ + VEX_VRMrX, /* VEX mod_rm, VEX.vvvv -> mod_rm */ + VEX_RRX, /* VEX VEX.vvvv, mod_reg -> mod_rm */ + VEX_RMRX, /* VEX VEX.vvvv, mod_rm, imm8[7:4] -> mod_reg */ + VEX_MX, /* VEX mod_rm -> mod_reg */ + VEX_MXI, /* VEX mod_rm, imm8 -> mod_reg */ + VEX_XXI, /* VEX mod_rm, imm8 -> VEX.vvvv */ + VEX_MR, /* VEX mod_rm -> mod_reg */ + VEX_RRI, /* VEX mod_reg, mod_rm -> implicit(eflags/r32) */ + VEX_RX, /* VEX mod_reg -> mod_rm */ + VEX_RR, /* VEX mod_rm -> mod_reg */ + VEX_RRi, /* VEX mod_rm, imm8 -> mod_reg */ + VEX_RM, /* VEX mod_reg -> mod_rm */ + VEX_RIM, /* VEX mod_reg, imm8 -> mod_rm */ + VEX_RRM, /* VEX VEX.vvvv, mod_reg -> mod_rm */ + VEX_RMX, /* VEX VEX.vvvv, mod_rm -> mod_reg */ + VEX_SbVM, /* VEX SIB, VEX.vvvv -> mod_rm */ + VMx, /* vmcall/vmlaunch/vmresume/vmxoff */ + VMxo, /* VMx instruction with optional prefix */ + SVM, /* AMD SVM instructions */ + BLS, /* BLSR, BLSMSK, BLSI */ + FMA, /* FMA instructions, all VEX_RMrX */ + ADX /* ADX instructions, support REX.w, mod_rm->mod_reg */ +}; + +/* + * VEX prefixes + */ +#define VEX_2bytes 0xC5 /* the first byte of two-byte form */ +#define VEX_3bytes 0xC4 /* the first byte of three-byte form */ + +#define FILL 0x90 /* Fill byte used for alignment (nop) */ + +/* +** Register numbers for the i386 +*/ +#define EAX_REGNO 0 +#define ECX_REGNO 1 +#define EDX_REGNO 2 +#define EBX_REGNO 3 +#define ESP_REGNO 4 +#define EBP_REGNO 5 +#define ESI_REGNO 6 +#define EDI_REGNO 7 + +/* + * modes for immediate values + */ +#define MODE_NONE 0 +#define MODE_IPREL 1 /* signed IP relative value */ +#define MODE_SIGNED 2 /* sign extended immediate */ +#define MODE_IMPLIED 3 /* constant value implied from opcode */ +#define MODE_OFFSET 4 /* offset part of an address */ +#define MODE_RIPREL 5 /* like IPREL, but from %rip (amd64) */ + +/* + * The letters used in these macros are: + * IND - indirect to another to another table + * "T" - means to Terminate indirections (this is the final opcode) + * "S" - means "operand length suffix required" + * "Sa" - means AVX2 suffix (d/q) required + * "NS" - means "no suffix" which is the operand length suffix of the opcode + * "Z" - means instruction size arg required + * "u" - means the opcode is invalid in IA32 but valid in amd64 + * "x" - means the opcode is invalid in amd64, but not IA32 + * "y" - means the operand size is always 64 bits in 64 bit mode + * "p" - means push/pop stack operation + * "vr" - means VEX instruction that operates on normal registers, not fpu + */ + +#if defined(DIS_TEXT) && defined(DIS_MEM) +#define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0, 0} +#define INDx(table) {(instable_t *)table, 0, "", 0, 0, 1, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, name, 0, 0, 0, 0, 1, 0} +#define TNSx(name, amode) {TERM, amode, name, 0, 0, 1, 0, 0, 0} +#define TNSy(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 0} +#define TNSyp(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 1} +#define TNSZ(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, name, 0, sz, 0, 1, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0, 1} +#define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, name, 1, 0, 0, 1, 0, 0} +#define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 1} +#define TSZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, name, 1, sz, 1, 0, 0, 0} +#define TSZy(name, amode, sz) {TERM, amode, name, 1, sz, 0, 1, 0, 0} +#define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0} +#elif defined(DIS_TEXT) +#define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0} +#define INDx(table) {(instable_t *)table, 0, "", 0, 1, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0} +#define TNSx(name, amode) {TERM, amode, name, 0, 1, 0, 0, 0} +#define TNSy(name, amode) {TERM, amode, name, 0, 0, 1, 0, 0} +#define TNSyp(name, amode) {TERM, amode, name, 0, 0, 1, 0, 1} +#define TNSZ(name, amode, sz) {TERM, amode, name, 0, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, name, 0, 0, 1, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, name, 0, 0, 0, 0, 0, 1} +#define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, name, 1, 1, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0} +#define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 1} +#define TSZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, name, 1, 1, 0, 0, 0} +#define TSZy(name, amode, sz) {TERM, amode, name, 1, 0, 1, 0, 0} +#define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0} +#elif defined(DIS_MEM) +#define IND(table) {(instable_t *)table, 0, 0, 0, 0, 0, 0} +#define INDx(table) {(instable_t *)table, 0, 0, 1, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, 0, 0, 0, 1, 0} +#define TNSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0} +#define TNSyp(name, amode) {TERM, amode, 0, 0, 1, 0, 1} +#define TNSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0} +#define TNSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 1} +#define TS(name, amode) {TERM, amode, 0, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0} +#define TSp(name, amode) {TERM, amode, 0, 0, 0, 0, 1} +#define TSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, sz, 1, 0, 0, 0} +#define TSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0} +#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0, 0} +#else +#define IND(table) {(instable_t *)table, 0, 0, 0, 0, 0} +#define INDx(table) {(instable_t *)table, 0, 1, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, 0, 0, 1, 0} +#define TNSy(name, amode) {TERM, amode, 0, 1, 0, 0} +#define TNSyp(name, amode) {TERM, amode, 0, 1, 0, 1} +#define TNSx(name, amode) {TERM, amode, 1, 0, 0, 0} +#define TNSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 1} +#define TS(name, amode) {TERM, amode, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, 1, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, 0, 1, 0, 0} +#define TSp(name, amode) {TERM, amode, 0, 0, 0, 1} +#define TSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, 1, 0, 0, 0} +#define TSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0} +#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0} +#endif + +#ifdef DIS_TEXT +/* + * this decodes the r_m field for mode's 0, 1, 2 in 16 bit mode + */ +const char *const dis_addr16[3][8] = { +"(%bx,%si)", "(%bx,%di)", "(%bp,%si)", "(%bp,%di)", "(%si)", "(%di)", "", + "(%bx)", +"(%bx,%si)", "(%bx,%di)", "(%bp,%si)", "(%bp,%di)", "(%si)", "(%di", "(%bp)", + "(%bx)", +"(%bx,%si)", "(%bx,%di)", "(%bp,%si)", "(%bp,%di)", "(%si)", "(%di)", "(%bp)", + "(%bx)", +}; + + +/* + * This decodes 32 bit addressing mode r_m field for modes 0, 1, 2 + */ +const char *const dis_addr32_mode0[16] = { + "(%eax)", "(%ecx)", "(%edx)", "(%ebx)", "", "", "(%esi)", "(%edi)", + "(%r8d)", "(%r9d)", "(%r10d)", "(%r11d)", "", "", "(%r14d)", "(%r15d)" +}; + +const char *const dis_addr32_mode12[16] = { + "(%eax)", "(%ecx)", "(%edx)", "(%ebx)", "", "(%ebp)", "(%esi)", "(%edi)", + "(%r8d)", "(%r9d)", "(%r10d)", "(%r11d)", "", "(%r13d)", "(%r14d)", "(%r15d)" +}; + +/* + * This decodes 64 bit addressing mode r_m field for modes 0, 1, 2 + */ +const char *const dis_addr64_mode0[16] = { + "(%rax)", "(%rcx)", "(%rdx)", "(%rbx)", "", "(%rip)", "(%rsi)", "(%rdi)", + "(%r8)", "(%r9)", "(%r10)", "(%r11)", "(%r12)", "(%rip)", "(%r14)", "(%r15)" +}; +const char *const dis_addr64_mode12[16] = { + "(%rax)", "(%rcx)", "(%rdx)", "(%rbx)", "", "(%rbp)", "(%rsi)", "(%rdi)", + "(%r8)", "(%r9)", "(%r10)", "(%r11)", "(%r12)", "(%r13)", "(%r14)", "(%r15)" +}; + +/* + * decode for scale from SIB byte + */ +const char *const dis_scale_factor[4] = { ")", ",2)", ",4)", ",8)" }; + +/* + * decode for scale from VSIB byte, note that we always include the scale factor + * to match gas. + */ +const char *const dis_vscale_factor[4] = { ",1)", ",2)", ",4)", ",8)" }; + +/* + * register decoding for normal references to registers (ie. not addressing) + */ +const char *const dis_REG8[16] = { + "%al", "%cl", "%dl", "%bl", "%ah", "%ch", "%dh", "%bh", + "%r8b", "%r9b", "%r10b", "%r11b", "%r12b", "%r13b", "%r14b", "%r15b" +}; + +const char *const dis_REG8_REX[16] = { + "%al", "%cl", "%dl", "%bl", "%spl", "%bpl", "%sil", "%dil", + "%r8b", "%r9b", "%r10b", "%r11b", "%r12b", "%r13b", "%r14b", "%r15b" +}; + +const char *const dis_REG16[16] = { + "%ax", "%cx", "%dx", "%bx", "%sp", "%bp", "%si", "%di", + "%r8w", "%r9w", "%r10w", "%r11w", "%r12w", "%r13w", "%r14w", "%r15w" +}; + +const char *const dis_REG32[16] = { + "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi", + "%r8d", "%r9d", "%r10d", "%r11d", "%r12d", "%r13d", "%r14d", "%r15d" +}; + +const char *const dis_REG64[16] = { + "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi", + "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15" +}; + +const char *const dis_DEBUGREG[16] = { + "%db0", "%db1", "%db2", "%db3", "%db4", "%db5", "%db6", "%db7", + "%db8", "%db9", "%db10", "%db11", "%db12", "%db13", "%db14", "%db15" +}; + +const char *const dis_CONTROLREG[16] = { + "%cr0", "%cr1", "%cr2", "%cr3", "%cr4", "%cr5?", "%cr6?", "%cr7?", + "%cr8", "%cr9?", "%cr10?", "%cr11?", "%cr12?", "%cr13?", "%cr14?", "%cr15?" +}; + +const char *const dis_TESTREG[16] = { + "%tr0?", "%tr1?", "%tr2?", "%tr3", "%tr4", "%tr5", "%tr6", "%tr7", + "%tr0?", "%tr1?", "%tr2?", "%tr3", "%tr4", "%tr5", "%tr6", "%tr7" +}; + +const char *const dis_MMREG[16] = { + "%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7", + "%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7" +}; + +const char *const dis_XMMREG[16] = { + "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", + "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15" +}; + +const char *const dis_YMMREG[16] = { + "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", + "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15" +}; + +const char *const dis_SEGREG[16] = { + "%es", "%cs", "%ss", "%ds", "%fs", "%gs", "<reserved>", "<reserved>", + "%es", "%cs", "%ss", "%ds", "%fs", "%gs", "<reserved>", "<reserved>" +}; + +/* + * SIMD predicate suffixes + */ +const char *const dis_PREDSUFFIX[8] = { + "eq", "lt", "le", "unord", "neq", "nlt", "nle", "ord" +}; + +const char *const dis_AVXvgrp7[3][8] = { + /*0 1 2 3 4 5 6 7*/ +/*71*/ {"", "", "vpsrlw", "", "vpsraw", "", "vpsllw", ""}, +/*72*/ {"", "", "vpsrld", "", "vpsrad", "", "vpslld", ""}, +/*73*/ {"", "", "vpsrlq", "vpsrldq", "", "", "vpsllq", "vpslldq"} +}; + +#endif /* DIS_TEXT */ + +/* + * "decode table" for 64 bit mode MOVSXD instruction (opcode 0x63) + */ +const instable_t dis_opMOVSLD = TNS("movslq",MOVSXZ); + +/* + * "decode table" for pause and clflush instructions + */ +const instable_t dis_opPause = TNS("pause", NORM); + +/* + * Decode table for 0x0F00 opcodes + */ +const instable_t dis_op0F00[8] = { + +/* [0] */ TNS("sldt",M), TNS("str",M), TNSy("lldt",M), TNSy("ltr",M), +/* [4] */ TNSZ("verr",M,2), TNSZ("verw",M,2), INVALID, INVALID, +}; + + +/* + * Decode table for 0x0F01 opcodes + */ +const instable_t dis_op0F01[8] = { + +/* [0] */ TNSZ("sgdt",VMx,6), TNSZ("sidt",MONITOR_MWAIT,6), TNSZ("lgdt",XGETBV_XSETBV,6), TNSZ("lidt",SVM,6), +/* [4] */ TNSZ("smsw",M,2), INVALID, TNSZ("lmsw",M,2), TNS("invlpg",SWAPGS_RDTSCP), +}; + +/* + * Decode table for 0x0F18 opcodes -- SIMD prefetch + */ +const instable_t dis_op0F18[8] = { + +/* [0] */ TNS("prefetchnta",PREF),TNS("prefetcht0",PREF), TNS("prefetcht1",PREF), TNS("prefetcht2",PREF), +/* [4] */ INVALID, INVALID, INVALID, INVALID, +}; + +/* + * Decode table for 0x0FAE opcodes -- SIMD state save/restore + */ +const instable_t dis_op0FAE[8] = { +/* [0] */ TNSZ("fxsave",M,512), TNSZ("fxrstor",M,512), TNS("ldmxcsr",M), TNS("stmxcsr",M), +/* [4] */ TNSZ("xsave",M,512), TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE), TNS("sfence",XMMSFNC), +}; + +/* + * Decode table for 0x0FBA opcodes + */ + +const instable_t dis_op0FBA[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ TS("bt",MIb), TS("bts",MIb), TS("btr",MIb), TS("btc",MIb), +}; + +/* + * Decode table for 0x0FC7 opcode (group 9) + */ + +const instable_t dis_op0FC7[8] = { + +/* [0] */ INVALID, TNS("cmpxchg8b",M), INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmptrld",MG9), TNS("vmptrst",MG9), +}; + +/* + * Decode table for 0x0FC7 opcode (group 9) mode 3 + */ + +const instable_t dis_op0FC7m3[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("rdrand",MG9), TNS("rdseed", MG9), +}; + +/* + * Decode table for 0x0FC7 opcode with 0x66 prefix + */ + +const instable_t dis_op660FC7[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmclear",M), INVALID, +}; + +/* + * Decode table for 0x0FC7 opcode with 0xF3 prefix + */ + +const instable_t dis_opF30FC7[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmxon",M), INVALID, +}; + +/* + * Decode table for 0x0FC8 opcode -- 486 bswap instruction + * + *bit pattern: 0000 1111 1100 1reg + */ +const instable_t dis_op0FC8[4] = { +/* [0] */ TNS("bswap",R), INVALID, INVALID, INVALID, +}; + +/* + * Decode table for 0x0F71, 0x0F72, and 0x0F73 opcodes -- MMX instructions + */ +const instable_t dis_op0F7123[4][8] = { +{ +/* [70].0 */ INVALID, INVALID, INVALID, INVALID, +/* .4 */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [71].0 */ INVALID, INVALID, TNS("psrlw",MMOSH), INVALID, +/* .4 */ TNS("psraw",MMOSH), INVALID, TNS("psllw",MMOSH), INVALID, +}, { +/* [72].0 */ INVALID, INVALID, TNS("psrld",MMOSH), INVALID, +/* .4 */ TNS("psrad",MMOSH), INVALID, TNS("pslld",MMOSH), INVALID, +}, { +/* [73].0 */ INVALID, INVALID, TNS("psrlq",MMOSH), TNS("INVALID",MMOSH), +/* .4 */ INVALID, INVALID, TNS("psllq",MMOSH), TNS("INVALID",MMOSH), +} }; + +/* + * Decode table for SIMD extensions to above 0x0F71-0x0F73 opcodes. + */ +const instable_t dis_opSIMD7123[32] = { +/* [70].0 */ INVALID, INVALID, INVALID, INVALID, +/* .4 */ INVALID, INVALID, INVALID, INVALID, + +/* [71].0 */ INVALID, INVALID, TNS("psrlw",XMMSH), INVALID, +/* .4 */ TNS("psraw",XMMSH), INVALID, TNS("psllw",XMMSH), INVALID, + +/* [72].0 */ INVALID, INVALID, TNS("psrld",XMMSH), INVALID, +/* .4 */ TNS("psrad",XMMSH), INVALID, TNS("pslld",XMMSH), INVALID, + +/* [73].0 */ INVALID, INVALID, TNS("psrlq",XMMSH), TNS("psrldq",XMMSH), +/* .4 */ INVALID, INVALID, TNS("psllq",XMMSH), TNS("pslldq",XMMSH), +}; + +/* + * SIMD instructions have been wedged into the existing IA32 instruction + * set through the use of prefixes. That is, while 0xf0 0x58 may be + * addps, 0xf3 0xf0 0x58 (literally, repz addps) is a completely different + * instruction - addss. At present, three prefixes have been coopted in + * this manner - address size (0x66), repnz (0xf2) and repz (0xf3). The + * following tables are used to provide the prefixed instruction names. + * The arrays are sparse, but they're fast. + */ + +/* + * Decode table for SIMD instructions with the address size (0x66) prefix. + */ +const instable_t dis_opSIMDdata16[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNSZ("movupd",XMM,16), TNSZ("movupd",XMMS,16), TNSZ("movlpd",XMMM,8), TNSZ("movlpd",XMMMS,8), +/* [14] */ TNSZ("unpcklpd",XMM,16),TNSZ("unpckhpd",XMM,16),TNSZ("movhpd",XMMM,8), TNSZ("movhpd",XMMMS,8), +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ TNSZ("movapd",XMM,16), TNSZ("movapd",XMMS,16), TNSZ("cvtpi2pd",XMMOMX,8),TNSZ("movntpd",XMMOMS,16), +/* [2C] */ TNSZ("cvttpd2pi",XMMXMM,16),TNSZ("cvtpd2pi",XMMXMM,16),TNSZ("ucomisd",XMM,8),TNSZ("comisd",XMM,8), + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ TNS("movmskpd",XMMOX3), TNSZ("sqrtpd",XMM,16), INVALID, INVALID, +/* [54] */ TNSZ("andpd",XMM,16), TNSZ("andnpd",XMM,16), TNSZ("orpd",XMM,16), TNSZ("xorpd",XMM,16), +/* [58] */ TNSZ("addpd",XMM,16), TNSZ("mulpd",XMM,16), TNSZ("cvtpd2ps",XMM,16),TNSZ("cvtps2dq",XMM,16), +/* [5C] */ TNSZ("subpd",XMM,16), TNSZ("minpd",XMM,16), TNSZ("divpd",XMM,16), TNSZ("maxpd",XMM,16), + +/* [60] */ TNSZ("punpcklbw",XMM,16),TNSZ("punpcklwd",XMM,16),TNSZ("punpckldq",XMM,16),TNSZ("packsswb",XMM,16), +/* [64] */ TNSZ("pcmpgtb",XMM,16), TNSZ("pcmpgtw",XMM,16), TNSZ("pcmpgtd",XMM,16), TNSZ("packuswb",XMM,16), +/* [68] */ TNSZ("punpckhbw",XMM,16),TNSZ("punpckhwd",XMM,16),TNSZ("punpckhdq",XMM,16),TNSZ("packssdw",XMM,16), +/* [6C] */ TNSZ("punpcklqdq",XMM,16),TNSZ("punpckhqdq",XMM,16),TNSZ("movd",XMM3MX,4),TNSZ("movdqa",XMM,16), + +/* [70] */ TNSZ("pshufd",XMMP,16), INVALID, INVALID, INVALID, +/* [74] */ TNSZ("pcmpeqb",XMM,16), TNSZ("pcmpeqw",XMM,16), TNSZ("pcmpeqd",XMM,16), INVALID, +/* [78] */ TNSZ("extrq",XMM2I,16), TNSZ("extrq",XMM,16), INVALID, INVALID, +/* [7C] */ TNSZ("haddpd",XMM,16), TNSZ("hsubpd",XMM,16), TNSZ("movd",XMM3MXS,4), TNSZ("movdqa",XMMS,16), + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, TNSZ("cmppd",XMMP,16), INVALID, +/* [C4] */ TNSZ("pinsrw",XMMPRM,2),TNS("pextrw",XMM3P), TNSZ("shufpd",XMMP,16), INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ TNSZ("addsubpd",XMM,16),TNSZ("psrlw",XMM,16), TNSZ("psrld",XMM,16), TNSZ("psrlq",XMM,16), +/* [D4] */ TNSZ("paddq",XMM,16), TNSZ("pmullw",XMM,16), TNSZ("movq",XMMS,8), TNS("pmovmskb",XMMX3), +/* [D8] */ TNSZ("psubusb",XMM,16), TNSZ("psubusw",XMM,16), TNSZ("pminub",XMM,16), TNSZ("pand",XMM,16), +/* [DC] */ TNSZ("paddusb",XMM,16), TNSZ("paddusw",XMM,16), TNSZ("pmaxub",XMM,16), TNSZ("pandn",XMM,16), + +/* [E0] */ TNSZ("pavgb",XMM,16), TNSZ("psraw",XMM,16), TNSZ("psrad",XMM,16), TNSZ("pavgw",XMM,16), +/* [E4] */ TNSZ("pmulhuw",XMM,16), TNSZ("pmulhw",XMM,16), TNSZ("cvttpd2dq",XMM,16),TNSZ("movntdq",XMMS,16), +/* [E8] */ TNSZ("psubsb",XMM,16), TNSZ("psubsw",XMM,16), TNSZ("pminsw",XMM,16), TNSZ("por",XMM,16), +/* [EC] */ TNSZ("paddsb",XMM,16), TNSZ("paddsw",XMM,16), TNSZ("pmaxsw",XMM,16), TNSZ("pxor",XMM,16), + +/* [F0] */ INVALID, TNSZ("psllw",XMM,16), TNSZ("pslld",XMM,16), TNSZ("psllq",XMM,16), +/* [F4] */ TNSZ("pmuludq",XMM,16), TNSZ("pmaddwd",XMM,16), TNSZ("psadbw",XMM,16), TNSZ("maskmovdqu", XMMXIMPL,16), +/* [F8] */ TNSZ("psubb",XMM,16), TNSZ("psubw",XMM,16), TNSZ("psubd",XMM,16), TNSZ("psubq",XMM,16), +/* [FC] */ TNSZ("paddb",XMM,16), TNSZ("paddw",XMM,16), TNSZ("paddd",XMM,16), INVALID, +}; + +const instable_t dis_opAVX660F[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNSZ("vmovupd",VEX_MX,16), TNSZ("vmovupd",VEX_RX,16), TNSZ("vmovlpd",VEX_RMrX,8), TNSZ("vmovlpd",VEX_RM,8), +/* [14] */ TNSZ("vunpcklpd",VEX_RMrX,16),TNSZ("vunpckhpd",VEX_RMrX,16),TNSZ("vmovhpd",VEX_RMrX,8), TNSZ("vmovhpd",VEX_RM,8), +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ TNSZ("vmovapd",VEX_MX,16), TNSZ("vmovapd",VEX_RX,16), INVALID, TNSZ("vmovntpd",VEX_RM,16), +/* [2C] */ INVALID, INVALID, TNSZ("vucomisd",VEX_MX,8),TNSZ("vcomisd",VEX_MX,8), + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ TNS("vmovmskpd",VEX_MR), TNSZ("vsqrtpd",VEX_MX,16), INVALID, INVALID, +/* [54] */ TNSZ("vandpd",VEX_RMrX,16), TNSZ("vandnpd",VEX_RMrX,16), TNSZ("vorpd",VEX_RMrX,16), TNSZ("vxorpd",VEX_RMrX,16), +/* [58] */ TNSZ("vaddpd",VEX_RMrX,16), TNSZ("vmulpd",VEX_RMrX,16), TNSZ("vcvtpd2ps",VEX_MX,16),TNSZ("vcvtps2dq",VEX_MX,16), +/* [5C] */ TNSZ("vsubpd",VEX_RMrX,16), TNSZ("vminpd",VEX_RMrX,16), TNSZ("vdivpd",VEX_RMrX,16), TNSZ("vmaxpd",VEX_RMrX,16), + +/* [60] */ TNSZ("vpunpcklbw",VEX_RMrX,16),TNSZ("vpunpcklwd",VEX_RMrX,16),TNSZ("vpunpckldq",VEX_RMrX,16),TNSZ("vpacksswb",VEX_RMrX,16), +/* [64] */ TNSZ("vpcmpgtb",VEX_RMrX,16), TNSZ("vpcmpgtw",VEX_RMrX,16), TNSZ("vpcmpgtd",VEX_RMrX,16), TNSZ("vpackuswb",VEX_RMrX,16), +/* [68] */ TNSZ("vpunpckhbw",VEX_RMrX,16),TNSZ("vpunpckhwd",VEX_RMrX,16),TNSZ("vpunpckhdq",VEX_RMrX,16),TNSZ("vpackssdw",VEX_RMrX,16), +/* [6C] */ TNSZ("vpunpcklqdq",VEX_RMrX,16),TNSZ("vpunpckhqdq",VEX_RMrX,16),TNSZ("vmovd",VEX_MX,4),TNSZ("vmovdqa",VEX_MX,16), + +/* [70] */ TNSZ("vpshufd",VEX_MXI,16), TNSZ("vgrp71",VEX_XXI,16), TNSZ("vgrp72",VEX_XXI,16), TNSZ("vgrp73",VEX_XXI,16), +/* [74] */ TNSZ("vpcmpeqb",VEX_RMrX,16), TNSZ("vpcmpeqw",VEX_RMrX,16), TNSZ("vpcmpeqd",VEX_RMrX,16), INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ TNSZ("vhaddpd",VEX_RMrX,16), TNSZ("vhsubpd",VEX_RMrX,16), TNSZ("vmovd",VEX_RR,4), TNSZ("vmovdqa",VEX_RX,16), + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, TNSZ("vcmppd",VEX_RMRX,16), INVALID, +/* [C4] */ TNSZ("vpinsrw",VEX_RMRX,2),TNS("vpextrw",VEX_MR), TNSZ("vshufpd",VEX_RMRX,16), INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ TNSZ("vaddsubpd",VEX_RMrX,16),TNSZ("vpsrlw",VEX_RMrX,16), TNSZ("vpsrld",VEX_RMrX,16), TNSZ("vpsrlq",VEX_RMrX,16), +/* [D4] */ TNSZ("vpaddq",VEX_RMrX,16), TNSZ("vpmullw",VEX_RMrX,16), TNSZ("vmovq",VEX_RX,8), TNS("vpmovmskb",VEX_MR), +/* [D8] */ TNSZ("vpsubusb",VEX_RMrX,16), TNSZ("vpsubusw",VEX_RMrX,16), TNSZ("vpminub",VEX_RMrX,16), TNSZ("vpand",VEX_RMrX,16), +/* [DC] */ TNSZ("vpaddusb",VEX_RMrX,16), TNSZ("vpaddusw",VEX_RMrX,16), TNSZ("vpmaxub",VEX_RMrX,16), TNSZ("vpandn",VEX_RMrX,16), + +/* [E0] */ TNSZ("vpavgb",VEX_RMrX,16), TNSZ("vpsraw",VEX_RMrX,16), TNSZ("vpsrad",VEX_RMrX,16), TNSZ("vpavgw",VEX_RMrX,16), +/* [E4] */ TNSZ("vpmulhuw",VEX_RMrX,16), TNSZ("vpmulhw",VEX_RMrX,16), TNSZ("vcvttpd2dq",VEX_MX,16),TNSZ("vmovntdq",VEX_RM,16), +/* [E8] */ TNSZ("vpsubsb",VEX_RMrX,16), TNSZ("vpsubsw",VEX_RMrX,16), TNSZ("vpminsw",VEX_RMrX,16), TNSZ("vpor",VEX_RMrX,16), +/* [EC] */ TNSZ("vpaddsb",VEX_RMrX,16), TNSZ("vpaddsw",VEX_RMrX,16), TNSZ("vpmaxsw",VEX_RMrX,16), TNSZ("vpxor",VEX_RMrX,16), + +/* [F0] */ INVALID, TNSZ("vpsllw",VEX_RMrX,16), TNSZ("vpslld",VEX_RMrX,16), TNSZ("vpsllq",VEX_RMrX,16), +/* [F4] */ TNSZ("vpmuludq",VEX_RMrX,16), TNSZ("vpmaddwd",VEX_RMrX,16), TNSZ("vpsadbw",VEX_RMrX,16), TNS("vmaskmovdqu",VEX_MX), +/* [F8] */ TNSZ("vpsubb",VEX_RMrX,16), TNSZ("vpsubw",VEX_RMrX,16), TNSZ("vpsubd",VEX_RMrX,16), TNSZ("vpsubq",VEX_RMrX,16), +/* [FC] */ TNSZ("vpaddb",VEX_RMrX,16), TNSZ("vpaddw",VEX_RMrX,16), TNSZ("vpaddd",VEX_RMrX,16), INVALID, +}; + +/* + * Decode table for SIMD instructions with the repnz (0xf2) prefix. + */ +const instable_t dis_opSIMDrepnz[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNSZ("movsd",XMM,8), TNSZ("movsd",XMMS,8), TNSZ("movddup",XMM,8), INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, TNSZ("cvtsi2sd",XMM3MX,4),TNSZ("movntsd",XMMMS,8), +/* [2C] */ TNSZ("cvttsd2si",XMMXM3,8),TNSZ("cvtsd2si",XMMXM3,8),INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, TNSZ("sqrtsd",XMM,8), INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ TNSZ("addsd",XMM,8), TNSZ("mulsd",XMM,8), TNSZ("cvtsd2ss",XMM,8), INVALID, +/* [5C] */ TNSZ("subsd",XMM,8), TNSZ("minsd",XMM,8), TNSZ("divsd",XMM,8), TNSZ("maxsd",XMM,8), + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ TNSZ("pshuflw",XMMP,16),INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ TNSZ("insertq",XMMX2I,16),TNSZ("insertq",XMM,8),INVALID, INVALID, +/* [7C] */ TNSZ("haddps",XMM,16), TNSZ("hsubps",XMM,16), INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, TNSZ("cmpsd",XMMP,8), INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ TNSZ("addsubps",XMM,16),INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, TNS("movdq2q",XMMXM), INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, TNSZ("cvtpd2dq",XMM,16),INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ TNS("lddqu",XMMM), INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF20F[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNSZ("vmovsd",VEX_RMrX,8), TNSZ("vmovsd",VEX_RRX,8), TNSZ("vmovddup",VEX_MX,8), INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, TNSZ("vcvtsi2sd",VEX_RMrX,4),INVALID, +/* [2C] */ TNSZ("vcvttsd2si",VEX_MR,8),TNSZ("vcvtsd2si",VEX_MR,8),INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, TNSZ("vsqrtsd",VEX_RMrX,8), INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ TNSZ("vaddsd",VEX_RMrX,8), TNSZ("vmulsd",VEX_RMrX,8), TNSZ("vcvtsd2ss",VEX_RMrX,8), INVALID, +/* [5C] */ TNSZ("vsubsd",VEX_RMrX,8), TNSZ("vminsd",VEX_RMrX,8), TNSZ("vdivsd",VEX_RMrX,8), TNSZ("vmaxsd",VEX_RMrX,8), + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ TNSZ("vpshuflw",VEX_MXI,16),INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ TNSZ("vhaddps",VEX_RMrX,8), TNSZ("vhsubps",VEX_RMrX,8), INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, TNSZ("vcmpsd",VEX_RMRX,8), INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ TNSZ("vaddsubps",VEX_RMrX,8), INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, TNSZ("vcvtpd2dq",VEX_MX,16),INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ TNSZ("vlddqu",VEX_MX,16), INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF20F3A[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ TNSZvr("rorx",VEX_MXI,6),INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF20F38[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, TNSZvr("pdep",VEX_RMrX,5),TNSZvr("mulx",VEX_RMrX,5),TNSZvr("shrx",VEX_VRMrX,5), +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF30F38[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, TNSZvr("pext",VEX_RMrX,5),INVALID, TNSZvr("sarx",VEX_VRMrX,5), +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; +/* + * Decode table for SIMD instructions with the repz (0xf3) prefix. + */ +const instable_t dis_opSIMDrepz[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNSZ("movss",XMM,4), TNSZ("movss",XMMS,4), TNSZ("movsldup",XMM,16),INVALID, +/* [14] */ INVALID, INVALID, TNSZ("movshdup",XMM,16),INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, TNSZ("cvtsi2ss",XMM3MX,4),TNSZ("movntss",XMMMS,4), +/* [2C] */ TNSZ("cvttss2si",XMMXM3,4),TNSZ("cvtss2si",XMMXM3,4),INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, TNSZ("sqrtss",XMM,4), TNSZ("rsqrtss",XMM,4), TNSZ("rcpss",XMM,4), +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ TNSZ("addss",XMM,4), TNSZ("mulss",XMM,4), TNSZ("cvtss2sd",XMM,4), TNSZ("cvttps2dq",XMM,16), +/* [5C] */ TNSZ("subss",XMM,4), TNSZ("minss",XMM,4), TNSZ("divss",XMM,4), TNSZ("maxss",XMM,4), + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, TNSZ("movdqu",XMM,16), + +/* [70] */ TNSZ("pshufhw",XMMP,16),INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, TNSZ("movq",XMM,8), TNSZ("movdqu",XMMS,16), + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ TS("popcnt",MRw), INVALID, INVALID, INVALID, +/* [BC] */ TNSZ("tzcnt",MRw,5), TS("lzcnt",MRw), INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, TNSZ("cmpss",XMMP,4), INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, TNS("movq2dq",XMMMX), INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, TNSZ("cvtdq2pd",XMM,8), INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF30F[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNSZ("vmovss",VEX_RMrX,4), TNSZ("vmovss",VEX_RRX,4), TNSZ("vmovsldup",VEX_MX,4), INVALID, +/* [14] */ INVALID, INVALID, TNSZ("vmovshdup",VEX_MX,4), INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, TNSZ("vcvtsi2ss",VEX_RMrX,4),INVALID, +/* [2C] */ TNSZ("vcvttss2si",VEX_MR,4),TNSZ("vcvtss2si",VEX_MR,4),INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, TNSZ("vsqrtss",VEX_RMrX,4), TNSZ("vrsqrtss",VEX_RMrX,4), TNSZ("vrcpss",VEX_RMrX,4), +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ TNSZ("vaddss",VEX_RMrX,4), TNSZ("vmulss",VEX_RMrX,4), TNSZ("vcvtss2sd",VEX_RMrX,4), TNSZ("vcvttps2dq",VEX_MX,16), +/* [5C] */ TNSZ("vsubss",VEX_RMrX,4), TNSZ("vminss",VEX_RMrX,4), TNSZ("vdivss",VEX_RMrX,4), TNSZ("vmaxss",VEX_RMrX,4), + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, TNSZ("vmovdqu",VEX_MX,16), + +/* [70] */ TNSZ("vpshufhw",VEX_MXI,16),INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, TNSZ("vmovq",VEX_MX,8), TNSZ("vmovdqu",VEX_RX,16), + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, TNSZ("vcmpss",VEX_RMRX,4), INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, TNSZ("vcvtdq2pd",VEX_MX,8), INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; +/* + * The following two tables are used to encode crc32 and movbe + * since they share the same opcodes. + */ +const instable_t dis_op0F38F0[2] = { +/* [00] */ TNS("crc32b",CRC32), + TS("movbe",MOVBE), +}; + +const instable_t dis_op0F38F1[2] = { +/* [00] */ TS("crc32",CRC32), + TS("movbe",MOVBE), +}; + +/* + * The following table is used to distinguish between adox and adcx which share + * the same opcodes. + */ +const instable_t dis_op0F38F6[2] = { +/* [00] */ TNS("adcx",ADX), + TNS("adox",ADX), +}; + +const instable_t dis_op0F38[256] = { +/* [00] */ TNSZ("pshufb",XMM_66o,16),TNSZ("phaddw",XMM_66o,16),TNSZ("phaddd",XMM_66o,16),TNSZ("phaddsw",XMM_66o,16), +/* [04] */ TNSZ("pmaddubsw",XMM_66o,16),TNSZ("phsubw",XMM_66o,16), TNSZ("phsubd",XMM_66o,16),TNSZ("phsubsw",XMM_66o,16), +/* [08] */ TNSZ("psignb",XMM_66o,16),TNSZ("psignw",XMM_66o,16),TNSZ("psignd",XMM_66o,16),TNSZ("pmulhrsw",XMM_66o,16), +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNSZ("pblendvb",XMM_66r,16),INVALID, INVALID, INVALID, +/* [14] */ TNSZ("blendvps",XMM_66r,16),TNSZ("blendvpd",XMM_66r,16),INVALID, TNSZ("ptest",XMM_66r,16), +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ TNSZ("pabsb",XMM_66o,16),TNSZ("pabsw",XMM_66o,16),TNSZ("pabsd",XMM_66o,16),INVALID, + +/* [20] */ TNSZ("pmovsxbw",XMM_66r,16),TNSZ("pmovsxbd",XMM_66r,16),TNSZ("pmovsxbq",XMM_66r,16),TNSZ("pmovsxwd",XMM_66r,16), +/* [24] */ TNSZ("pmovsxwq",XMM_66r,16),TNSZ("pmovsxdq",XMM_66r,16),INVALID, INVALID, +/* [28] */ TNSZ("pmuldq",XMM_66r,16),TNSZ("pcmpeqq",XMM_66r,16),TNSZ("movntdqa",XMMM_66r,16),TNSZ("packusdw",XMM_66r,16), +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ TNSZ("pmovzxbw",XMM_66r,16),TNSZ("pmovzxbd",XMM_66r,16),TNSZ("pmovzxbq",XMM_66r,16),TNSZ("pmovzxwd",XMM_66r,16), +/* [34] */ TNSZ("pmovzxwq",XMM_66r,16),TNSZ("pmovzxdq",XMM_66r,16),INVALID, TNSZ("pcmpgtq",XMM_66r,16), +/* [38] */ TNSZ("pminsb",XMM_66r,16),TNSZ("pminsd",XMM_66r,16),TNSZ("pminuw",XMM_66r,16),TNSZ("pminud",XMM_66r,16), +/* [3C] */ TNSZ("pmaxsb",XMM_66r,16),TNSZ("pmaxsd",XMM_66r,16),TNSZ("pmaxuw",XMM_66r,16),TNSZ("pmaxud",XMM_66r,16), + +/* [40] */ TNSZ("pmulld",XMM_66r,16),TNSZ("phminposuw",XMM_66r,16),INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ TNSy("invept", RM_66r), TNSy("invvpid", RM_66r),TNSy("invpcid", RM_66r),INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ TNSZ("sha1nexte",XMM,16),TNSZ("sha1msg1",XMM,16),TNSZ("sha1msg2",XMM,16),TNSZ("sha256rnds2",XMM,16), +/* [CC] */ TNSZ("sha256msg1",XMM,16),TNSZ("sha256msg2",XMM,16),INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, TNSZ("aesimc",XMM_66r,16), +/* [DC] */ TNSZ("aesenc",XMM_66r,16),TNSZ("aesenclast",XMM_66r,16),TNSZ("aesdec",XMM_66r,16),TNSZ("aesdeclast",XMM_66r,16), + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, +/* [F0] */ IND(dis_op0F38F0), IND(dis_op0F38F1), INVALID, INVALID, +/* [F4] */ INVALID, INVALID, IND(dis_op0F38F6), INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVX660F38[256] = { +/* [00] */ TNSZ("vpshufb",VEX_RMrX,16),TNSZ("vphaddw",VEX_RMrX,16),TNSZ("vphaddd",VEX_RMrX,16),TNSZ("vphaddsw",VEX_RMrX,16), +/* [04] */ TNSZ("vpmaddubsw",VEX_RMrX,16),TNSZ("vphsubw",VEX_RMrX,16), TNSZ("vphsubd",VEX_RMrX,16),TNSZ("vphsubsw",VEX_RMrX,16), +/* [08] */ TNSZ("vpsignb",VEX_RMrX,16),TNSZ("vpsignw",VEX_RMrX,16),TNSZ("vpsignd",VEX_RMrX,16),TNSZ("vpmulhrsw",VEX_RMrX,16), +/* [0C] */ TNSZ("vpermilps",VEX_RMrX,8),TNSZ("vpermilpd",VEX_RMrX,16),TNSZ("vtestps",VEX_RRI,8), TNSZ("vtestpd",VEX_RRI,16), + +/* [10] */ INVALID, INVALID, INVALID, TNSZ("vcvtph2ps",VEX_MX,16), +/* [14] */ INVALID, INVALID, TNSZ("vpermps",VEX_RMrX,16),TNSZ("vptest",VEX_RRI,16), +/* [18] */ TNSZ("vbroadcastss",VEX_MX,4),TNSZ("vbroadcastsd",VEX_MX,8),TNSZ("vbroadcastf128",VEX_MX,16),INVALID, +/* [1C] */ TNSZ("vpabsb",VEX_MX,16),TNSZ("vpabsw",VEX_MX,16),TNSZ("vpabsd",VEX_MX,16),INVALID, + +/* [20] */ TNSZ("vpmovsxbw",VEX_MX,16),TNSZ("vpmovsxbd",VEX_MX,16),TNSZ("vpmovsxbq",VEX_MX,16),TNSZ("vpmovsxwd",VEX_MX,16), +/* [24] */ TNSZ("vpmovsxwq",VEX_MX,16),TNSZ("vpmovsxdq",VEX_MX,16),INVALID, INVALID, +/* [28] */ TNSZ("vpmuldq",VEX_RMrX,16),TNSZ("vpcmpeqq",VEX_RMrX,16),TNSZ("vmovntdqa",VEX_MX,16),TNSZ("vpackusdw",VEX_RMrX,16), +/* [2C] */ TNSZ("vmaskmovps",VEX_RMrX,8),TNSZ("vmaskmovpd",VEX_RMrX,16),TNSZ("vmaskmovps",VEX_RRM,8),TNSZ("vmaskmovpd",VEX_RRM,16), + +/* [30] */ TNSZ("vpmovzxbw",VEX_MX,16),TNSZ("vpmovzxbd",VEX_MX,16),TNSZ("vpmovzxbq",VEX_MX,16),TNSZ("vpmovzxwd",VEX_MX,16), +/* [34] */ TNSZ("vpmovzxwq",VEX_MX,16),TNSZ("vpmovzxdq",VEX_MX,16),TNSZ("vpermd",VEX_RMrX,16),TNSZ("vpcmpgtq",VEX_RMrX,16), +/* [38] */ TNSZ("vpminsb",VEX_RMrX,16),TNSZ("vpminsd",VEX_RMrX,16),TNSZ("vpminuw",VEX_RMrX,16),TNSZ("vpminud",VEX_RMrX,16), +/* [3C] */ TNSZ("vpmaxsb",VEX_RMrX,16),TNSZ("vpmaxsd",VEX_RMrX,16),TNSZ("vpmaxuw",VEX_RMrX,16),TNSZ("vpmaxud",VEX_RMrX,16), + +/* [40] */ TNSZ("vpmulld",VEX_RMrX,16),TNSZ("vphminposuw",VEX_MX,16),INVALID, INVALID, +/* [44] */ INVALID, TSaZ("vpsrlv",VEX_RMrX,16),TNSZ("vpsravd",VEX_RMrX,16),TSaZ("vpsllv",VEX_RMrX,16), +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ TNSZ("vpbroadcastd",VEX_MX,16),TNSZ("vpbroadcastq",VEX_MX,16),TNSZ("vbroadcasti128",VEX_MX,16),INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ TNSZ("vpbroadcastb",VEX_MX,16),TNSZ("vpbroadcastw",VEX_MX,16),INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ TSaZ("vpmaskmov",VEX_RMrX,16),INVALID, TSaZ("vpmaskmov",VEX_RRM,16),INVALID, + +/* [90] */ TNSZ("vpgatherd",VEX_SbVM,16),TNSZ("vpgatherq",VEX_SbVM,16),TNSZ("vgatherdp",VEX_SbVM,16),TNSZ("vgatherqp",VEX_SbVM,16), +/* [94] */ INVALID, INVALID, TNSZ("vfmaddsub132p",FMA,16),TNSZ("vfmsubadd132p",FMA,16), +/* [98] */ TNSZ("vfmadd132p",FMA,16),TNSZ("vfmadd132s",FMA,16),TNSZ("vfmsub132p",FMA,16),TNSZ("vfmsub132s",FMA,16), +/* [9C] */ TNSZ("vfnmadd132p",FMA,16),TNSZ("vfnmadd132s",FMA,16),TNSZ("vfnmsub132p",FMA,16),TNSZ("vfnmsub132s",FMA,16), + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, TNSZ("vfmaddsub213p",FMA,16),TNSZ("vfmsubadd213p",FMA,16), +/* [A8] */ TNSZ("vfmadd213p",FMA,16),TNSZ("vfmadd213s",FMA,16),TNSZ("vfmsub213p",FMA,16),TNSZ("vfmsub213s",FMA,16), +/* [AC] */ TNSZ("vfnmadd213p",FMA,16),TNSZ("vfnmadd213s",FMA,16),TNSZ("vfnmsub213p",FMA,16),TNSZ("vfnmsub213s",FMA,16), + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, TNSZ("vfmaddsub231p",FMA,16),TNSZ("vfmsubadd231p",FMA,16), +/* [B8] */ TNSZ("vfmadd231p",FMA,16),TNSZ("vfmadd231s",FMA,16),TNSZ("vfmsub231p",FMA,16),TNSZ("vfmsub231s",FMA,16), +/* [BC] */ TNSZ("vfnmadd231p",FMA,16),TNSZ("vfnmadd231s",FMA,16),TNSZ("vfnmsub231p",FMA,16),TNSZ("vfnmsub231s",FMA,16), + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, TNSZ("vaesimc",VEX_MX,16), +/* [DC] */ TNSZ("vaesenc",VEX_RMrX,16),TNSZ("vaesenclast",VEX_RMrX,16),TNSZ("vaesdec",VEX_RMrX,16),TNSZ("vaesdeclast",VEX_RMrX,16), + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, +/* [F0] */ IND(dis_op0F38F0), IND(dis_op0F38F1), INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, TNSZvr("shlx",VEX_VRMrX,5), +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_op0F3A[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ TNSZ("roundps",XMMP_66r,16),TNSZ("roundpd",XMMP_66r,16),TNSZ("roundss",XMMP_66r,16),TNSZ("roundsd",XMMP_66r,16), +/* [0C] */ TNSZ("blendps",XMMP_66r,16),TNSZ("blendpd",XMMP_66r,16),TNSZ("pblendw",XMMP_66r,16),TNSZ("palignr",XMMP_66o,16), + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ TNSZ("pextrb",XMM3PM_66r,8),TNSZ("pextrw",XMM3PM_66r,16),TSZ("pextr",XMM3PM_66r,16),TNSZ("extractps",XMM3PM_66r,16), +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ TNSZ("pinsrb",XMMPRM_66r,8),TNSZ("insertps",XMMP_66r,16),TSZ("pinsr",XMMPRM_66r,16),INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ TNSZ("dpps",XMMP_66r,16),TNSZ("dppd",XMMP_66r,16),TNSZ("mpsadbw",XMMP_66r,16),INVALID, +/* [44] */ TNSZ("pclmulqdq",XMMP_66r,16),INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ TNSZ("pcmpestrm",XMMP_66r,16),TNSZ("pcmpestri",XMMP_66r,16),TNSZ("pcmpistrm",XMMP_66r,16),TNSZ("pcmpistri",XMMP_66r,16), +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ TNSZ("sha1rnds4",XMMP,16),INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, TNSZ("aeskeygenassist",XMMP_66r,16), + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVX660F3A[256] = { +/* [00] */ TNSZ("vpermq",VEX_MXI,16),TNSZ("vpermpd",VEX_MXI,16),TNSZ("vpblendd",VEX_RMRX,16),INVALID, +/* [04] */ TNSZ("vpermilps",VEX_MXI,8),TNSZ("vpermilpd",VEX_MXI,16),TNSZ("vperm2f128",VEX_RMRX,16),INVALID, +/* [08] */ TNSZ("vroundps",VEX_MXI,16),TNSZ("vroundpd",VEX_MXI,16),TNSZ("vroundss",VEX_RMRX,16),TNSZ("vroundsd",VEX_RMRX,16), +/* [0C] */ TNSZ("vblendps",VEX_RMRX,16),TNSZ("vblendpd",VEX_RMRX,16),TNSZ("vpblendw",VEX_RMRX,16),TNSZ("vpalignr",VEX_RMRX,16), + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ TNSZ("vpextrb",VEX_RRi,8),TNSZ("vpextrw",VEX_RRi,16),TNSZ("vpextrd",VEX_RRi,16),TNSZ("vextractps",VEX_RM,16), +/* [18] */ TNSZ("vinsertf128",VEX_RMRX,16),TNSZ("vextractf128",VEX_RX,16),INVALID, INVALID, +/* [1C] */ INVALID, TNSZ("vcvtps2ph",VEX_RX,16), INVALID, INVALID, + +/* [20] */ TNSZ("vpinsrb",VEX_RMRX,8),TNSZ("vinsertps",VEX_RMRX,16),TNSZ("vpinsrd",VEX_RMRX,16),INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ TNSZ("vinserti128",VEX_RMRX,16),TNSZ("vextracti128",VEX_RIM,16),INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ TNSZ("vdpps",VEX_RMRX,16),TNSZ("vdppd",VEX_RMRX,16),TNSZ("vmpsadbw",VEX_RMRX,16),INVALID, +/* [44] */ TNSZ("vpclmulqdq",VEX_RMRX,16),INVALID, TNSZ("vperm2i128",VEX_RMRX,16),INVALID, +/* [48] */ INVALID, INVALID, TNSZ("vblendvps",VEX_RMRX,8), TNSZ("vblendvpd",VEX_RMRX,16), +/* [4C] */ TNSZ("vpblendvb",VEX_RMRX,16),INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ TNSZ("vpcmpestrm",VEX_MXI,16),TNSZ("vpcmpestri",VEX_MXI,16),TNSZ("vpcmpistrm",VEX_MXI,16),TNSZ("vpcmpistri",VEX_MXI,16), +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, TNSZ("vaeskeygenassist",VEX_MXI,16), + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +/* + * Decode table for 0x0F0D which uses the first byte of the mod_rm to + * indicate a sub-code. + */ +const instable_t dis_op0F0D[8] = { +/* [00] */ INVALID, TNS("prefetchw",PREF), TNS("prefetchwt1",PREF),INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +}; + +/* + * Decode table for 0x0F opcodes + */ + +const instable_t dis_op0F[16][16] = { +{ +/* [00] */ IND(dis_op0F00), IND(dis_op0F01), TNS("lar",MR), TNS("lsl",MR), +/* [04] */ INVALID, TNS("syscall",NORM), TNS("clts",NORM), TNS("sysret",NORM), +/* [08] */ TNS("invd",NORM), TNS("wbinvd",NORM), INVALID, TNS("ud2",NORM), +/* [0C] */ INVALID, IND(dis_op0F0D), INVALID, INVALID, +}, { +/* [10] */ TNSZ("movups",XMMO,16), TNSZ("movups",XMMOS,16),TNSZ("movlps",XMMO,8), TNSZ("movlps",XMMOS,8), +/* [14] */ TNSZ("unpcklps",XMMO,16),TNSZ("unpckhps",XMMO,16),TNSZ("movhps",XMMOM,8),TNSZ("movhps",XMMOMS,8), +/* [18] */ IND(dis_op0F18), INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, TS("nop",Mw), +}, { +/* [20] */ TSy("mov",SREG), TSy("mov",SREG), TSy("mov",SREG), TSy("mov",SREG), +/* [24] */ TSx("mov",SREG), INVALID, TSx("mov",SREG), INVALID, +/* [28] */ TNSZ("movaps",XMMO,16), TNSZ("movaps",XMMOS,16),TNSZ("cvtpi2ps",XMMOMX,8),TNSZ("movntps",XMMOS,16), +/* [2C] */ TNSZ("cvttps2pi",XMMOXMM,8),TNSZ("cvtps2pi",XMMOXMM,8),TNSZ("ucomiss",XMMO,4),TNSZ("comiss",XMMO,4), +}, { +/* [30] */ TNS("wrmsr",NORM), TNS("rdtsc",NORM), TNS("rdmsr",NORM), TNS("rdpmc",NORM), +/* [34] */ TNSx("sysenter",NORM), TNSx("sysexit",NORM), INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [40] */ TS("cmovx.o",MR), TS("cmovx.no",MR), TS("cmovx.b",MR), TS("cmovx.ae",MR), +/* [44] */ TS("cmovx.e",MR), TS("cmovx.ne",MR), TS("cmovx.be",MR), TS("cmovx.a",MR), +/* [48] */ TS("cmovx.s",MR), TS("cmovx.ns",MR), TS("cmovx.pe",MR), TS("cmovx.po",MR), +/* [4C] */ TS("cmovx.l",MR), TS("cmovx.ge",MR), TS("cmovx.le",MR), TS("cmovx.g",MR), +}, { +/* [50] */ TNS("movmskps",XMMOX3), TNSZ("sqrtps",XMMO,16), TNSZ("rsqrtps",XMMO,16),TNSZ("rcpps",XMMO,16), +/* [54] */ TNSZ("andps",XMMO,16), TNSZ("andnps",XMMO,16), TNSZ("orps",XMMO,16), TNSZ("xorps",XMMO,16), +/* [58] */ TNSZ("addps",XMMO,16), TNSZ("mulps",XMMO,16), TNSZ("cvtps2pd",XMMO,8),TNSZ("cvtdq2ps",XMMO,16), +/* [5C] */ TNSZ("subps",XMMO,16), TNSZ("minps",XMMO,16), TNSZ("divps",XMMO,16), TNSZ("maxps",XMMO,16), +}, { +/* [60] */ TNSZ("punpcklbw",MMO,4),TNSZ("punpcklwd",MMO,4),TNSZ("punpckldq",MMO,4),TNSZ("packsswb",MMO,8), +/* [64] */ TNSZ("pcmpgtb",MMO,8), TNSZ("pcmpgtw",MMO,8), TNSZ("pcmpgtd",MMO,8), TNSZ("packuswb",MMO,8), +/* [68] */ TNSZ("punpckhbw",MMO,8),TNSZ("punpckhwd",MMO,8),TNSZ("punpckhdq",MMO,8),TNSZ("packssdw",MMO,8), +/* [6C] */ TNSZ("INVALID",MMO,0), TNSZ("INVALID",MMO,0), TNSZ("movd",MMO,4), TNSZ("movq",MMO,8), +}, { +/* [70] */ TNSZ("pshufw",MMOPM,8), TNS("psrXXX",MR), TNS("psrXXX",MR), TNS("psrXXX",MR), +/* [74] */ TNSZ("pcmpeqb",MMO,8), TNSZ("pcmpeqw",MMO,8), TNSZ("pcmpeqd",MMO,8), TNS("emms",NORM), +/* [78] */ TNSy("vmread",RM), TNSy("vmwrite",MR), INVALID, INVALID, +/* [7C] */ INVALID, INVALID, TNSZ("movd",MMOS,4), TNSZ("movq",MMOS,8), +}, { +/* [80] */ TNS("jo",D), TNS("jno",D), TNS("jb",D), TNS("jae",D), +/* [84] */ TNS("je",D), TNS("jne",D), TNS("jbe",D), TNS("ja",D), +/* [88] */ TNS("js",D), TNS("jns",D), TNS("jp",D), TNS("jnp",D), +/* [8C] */ TNS("jl",D), TNS("jge",D), TNS("jle",D), TNS("jg",D), +}, { +/* [90] */ TNS("seto",Mb), TNS("setno",Mb), TNS("setb",Mb), TNS("setae",Mb), +/* [94] */ TNS("sete",Mb), TNS("setne",Mb), TNS("setbe",Mb), TNS("seta",Mb), +/* [98] */ TNS("sets",Mb), TNS("setns",Mb), TNS("setp",Mb), TNS("setnp",Mb), +/* [9C] */ TNS("setl",Mb), TNS("setge",Mb), TNS("setle",Mb), TNS("setg",Mb), +}, { +/* [A0] */ TSp("push",LSEG), TSp("pop",LSEG), TNS("cpuid",NORM), TS("bt",RMw), +/* [A4] */ TS("shld",DSHIFT), TS("shld",DSHIFTcl), INVALID, INVALID, +/* [A8] */ TSp("push",LSEG), TSp("pop",LSEG), TNS("rsm",NORM), TS("bts",RMw), +/* [AC] */ TS("shrd",DSHIFT), TS("shrd",DSHIFTcl), IND(dis_op0FAE), TS("imul",MRw), +}, { +/* [B0] */ TNS("cmpxchgb",RMw), TS("cmpxchg",RMw), TS("lss",MR), TS("btr",RMw), +/* [B4] */ TS("lfs",MR), TS("lgs",MR), TS("movzb",MOVZ), TNS("movzwl",MOVZ), +/* [B8] */ TNS("INVALID",MRw), INVALID, IND(dis_op0FBA), TS("btc",RMw), +/* [BC] */ TS("bsf",MRw), TS("bsr",MRw), TS("movsb",MOVZ), TNS("movswl",MOVZ), +}, { +/* [C0] */ TNS("xaddb",XADDB), TS("xadd",RMw), TNSZ("cmpps",XMMOPM,16),TNS("movnti",RM), +/* [C4] */ TNSZ("pinsrw",MMOPRM,2),TNS("pextrw",MMO3P), TNSZ("shufps",XMMOPM,16),IND(dis_op0FC7), +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [D0] */ INVALID, TNSZ("psrlw",MMO,8), TNSZ("psrld",MMO,8), TNSZ("psrlq",MMO,8), +/* [D4] */ TNSZ("paddq",MMO,8), TNSZ("pmullw",MMO,8), TNSZ("INVALID",MMO,0), TNS("pmovmskb",MMOM3), +/* [D8] */ TNSZ("psubusb",MMO,8), TNSZ("psubusw",MMO,8), TNSZ("pminub",MMO,8), TNSZ("pand",MMO,8), +/* [DC] */ TNSZ("paddusb",MMO,8), TNSZ("paddusw",MMO,8), TNSZ("pmaxub",MMO,8), TNSZ("pandn",MMO,8), +}, { +/* [E0] */ TNSZ("pavgb",MMO,8), TNSZ("psraw",MMO,8), TNSZ("psrad",MMO,8), TNSZ("pavgw",MMO,8), +/* [E4] */ TNSZ("pmulhuw",MMO,8), TNSZ("pmulhw",MMO,8), TNS("INVALID",XMMO), TNSZ("movntq",MMOMS,8), +/* [E8] */ TNSZ("psubsb",MMO,8), TNSZ("psubsw",MMO,8), TNSZ("pminsw",MMO,8), TNSZ("por",MMO,8), +/* [EC] */ TNSZ("paddsb",MMO,8), TNSZ("paddsw",MMO,8), TNSZ("pmaxsw",MMO,8), TNSZ("pxor",MMO,8), +}, { +/* [F0] */ INVALID, TNSZ("psllw",MMO,8), TNSZ("pslld",MMO,8), TNSZ("psllq",MMO,8), +/* [F4] */ TNSZ("pmuludq",MMO,8), TNSZ("pmaddwd",MMO,8), TNSZ("psadbw",MMO,8), TNSZ("maskmovq",MMOIMPL,8), +/* [F8] */ TNSZ("psubb",MMO,8), TNSZ("psubw",MMO,8), TNSZ("psubd",MMO,8), TNSZ("psubq",MMO,8), +/* [FC] */ TNSZ("paddb",MMO,8), TNSZ("paddw",MMO,8), TNSZ("paddd",MMO,8), INVALID, +} }; + +const instable_t dis_opAVX0F[16][16] = { +{ +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [10] */ TNSZ("vmovups",VEX_MX,16), TNSZ("vmovups",VEX_RM,16),TNSZ("vmovlps",VEX_RMrX,8), TNSZ("vmovlps",VEX_RM,8), +/* [14] */ TNSZ("vunpcklps",VEX_RMrX,16),TNSZ("vunpckhps",VEX_RMrX,16),TNSZ("vmovhps",VEX_RMrX,8),TNSZ("vmovhps",VEX_RM,8), +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ TNSZ("vmovaps",VEX_MX,16), TNSZ("vmovaps",VEX_RX,16),INVALID, TNSZ("vmovntps",VEX_RM,16), +/* [2C] */ INVALID, INVALID, TNSZ("vucomiss",VEX_MX,4),TNSZ("vcomiss",VEX_MX,4), +}, { +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [50] */ TNS("vmovmskps",VEX_MR), TNSZ("vsqrtps",VEX_MX,16), TNSZ("vrsqrtps",VEX_MX,16),TNSZ("vrcpps",VEX_MX,16), +/* [54] */ TNSZ("vandps",VEX_RMrX,16), TNSZ("vandnps",VEX_RMrX,16), TNSZ("vorps",VEX_RMrX,16), TNSZ("vxorps",VEX_RMrX,16), +/* [58] */ TNSZ("vaddps",VEX_RMrX,16), TNSZ("vmulps",VEX_RMrX,16), TNSZ("vcvtps2pd",VEX_MX,8),TNSZ("vcvtdq2ps",VEX_MX,16), +/* [5C] */ TNSZ("vsubps",VEX_RMrX,16), TNSZ("vminps",VEX_RMrX,16), TNSZ("vdivps",VEX_RMrX,16), TNSZ("vmaxps",VEX_RMrX,16), +}, { +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, TNS("vzeroupper", VEX_NONE), +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, TNSZ("vldmxcsr",VEX_MO,2), INVALID, +}, { +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [C0] */ INVALID, INVALID, TNSZ("vcmpps",VEX_RMRX,16),INVALID, +/* [C4] */ INVALID, INVALID, TNSZ("vshufps",VEX_RMRX,16),INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [F0] */ INVALID, INVALID, TNSZvr("andn",VEX_RMrX,5),TNSZvr("bls",BLS,5), +/* [F4] */ INVALID, TNSZvr("bzhi",VEX_VRMrX,5),INVALID, TNSZvr("bextr",VEX_VRMrX,5), +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +} }; + +/* + * Decode table for 0x80 opcodes + */ + +const instable_t dis_op80[8] = { + +/* [0] */ TNS("addb",IMlw), TNS("orb",IMw), TNS("adcb",IMlw), TNS("sbbb",IMlw), +/* [4] */ TNS("andb",IMw), TNS("subb",IMlw), TNS("xorb",IMw), TNS("cmpb",IMlw), +}; + + +/* + * Decode table for 0x81 opcodes. + */ + +const instable_t dis_op81[8] = { + +/* [0] */ TS("add",IMlw), TS("or",IMw), TS("adc",IMlw), TS("sbb",IMlw), +/* [4] */ TS("and",IMw), TS("sub",IMlw), TS("xor",IMw), TS("cmp",IMlw), +}; + + +/* + * Decode table for 0x82 opcodes. + */ + +const instable_t dis_op82[8] = { + +/* [0] */ TNSx("addb",IMlw), TNSx("orb",IMlw), TNSx("adcb",IMlw), TNSx("sbbb",IMlw), +/* [4] */ TNSx("andb",IMlw), TNSx("subb",IMlw), TNSx("xorb",IMlw), TNSx("cmpb",IMlw), +}; +/* + * Decode table for 0x83 opcodes. + */ + +const instable_t dis_op83[8] = { + +/* [0] */ TS("add",IMlw), TS("or",IMlw), TS("adc",IMlw), TS("sbb",IMlw), +/* [4] */ TS("and",IMlw), TS("sub",IMlw), TS("xor",IMlw), TS("cmp",IMlw), +}; + +/* + * Decode table for 0xC0 opcodes. + */ + +const instable_t dis_opC0[8] = { + +/* [0] */ TNS("rolb",MvI), TNS("rorb",MvI), TNS("rclb",MvI), TNS("rcrb",MvI), +/* [4] */ TNS("shlb",MvI), TNS("shrb",MvI), INVALID, TNS("sarb",MvI), +}; + +/* + * Decode table for 0xD0 opcodes. + */ + +const instable_t dis_opD0[8] = { + +/* [0] */ TNS("rolb",Mv), TNS("rorb",Mv), TNS("rclb",Mv), TNS("rcrb",Mv), +/* [4] */ TNS("shlb",Mv), TNS("shrb",Mv), TNS("salb",Mv), TNS("sarb",Mv), +}; + +/* + * Decode table for 0xC1 opcodes. + * 186 instruction set + */ + +const instable_t dis_opC1[8] = { + +/* [0] */ TS("rol",MvI), TS("ror",MvI), TS("rcl",MvI), TS("rcr",MvI), +/* [4] */ TS("shl",MvI), TS("shr",MvI), TS("sal",MvI), TS("sar",MvI), +}; + +/* + * Decode table for 0xD1 opcodes. + */ + +const instable_t dis_opD1[8] = { + +/* [0] */ TS("rol",Mv), TS("ror",Mv), TS("rcl",Mv), TS("rcr",Mv), +/* [4] */ TS("shl",Mv), TS("shr",Mv), TS("sal",Mv), TS("sar",Mv), +}; + + +/* + * Decode table for 0xD2 opcodes. + */ + +const instable_t dis_opD2[8] = { + +/* [0] */ TNS("rolb",Mv), TNS("rorb",Mv), TNS("rclb",Mv), TNS("rcrb",Mv), +/* [4] */ TNS("shlb",Mv), TNS("shrb",Mv), TNS("salb",Mv), TNS("sarb",Mv), +}; +/* + * Decode table for 0xD3 opcodes. + */ + +const instable_t dis_opD3[8] = { + +/* [0] */ TS("rol",Mv), TS("ror",Mv), TS("rcl",Mv), TS("rcr",Mv), +/* [4] */ TS("shl",Mv), TS("shr",Mv), TS("salb",Mv), TS("sar",Mv), +}; + + +/* + * Decode table for 0xF6 opcodes. + */ + +const instable_t dis_opF6[8] = { + +/* [0] */ TNS("testb",IMw), TNS("testb",IMw), TNS("notb",Mw), TNS("negb",Mw), +/* [4] */ TNS("mulb",MA), TNS("imulb",MA), TNS("divb",MA), TNS("idivb",MA), +}; + + +/* + * Decode table for 0xF7 opcodes. + */ + +const instable_t dis_opF7[8] = { + +/* [0] */ TS("test",IMw), TS("test",IMw), TS("not",Mw), TS("neg",Mw), +/* [4] */ TS("mul",MA), TS("imul",MA), TS("div",MA), TS("idiv",MA), +}; + + +/* + * Decode table for 0xFE opcodes. + */ + +const instable_t dis_opFE[8] = { + +/* [0] */ TNS("incb",Mw), TNS("decb",Mw), INVALID, INVALID, +/* [4] */ INVALID, INVALID, INVALID, INVALID, +}; +/* + * Decode table for 0xFF opcodes. + */ + +const instable_t dis_opFF[8] = { + +/* [0] */ TS("inc",Mw), TS("dec",Mw), TNSyp("call",INM), TNS("lcall",INM), +/* [4] */ TNSy("jmp",INM), TNS("ljmp",INM), TSp("push",M), INVALID, +}; + +/* for 287 instructions, which are a mess to decode */ + +const instable_t dis_opFP1n2[8][8] = { +{ +/* bit pattern: 1101 1xxx MODxx xR/M */ +/* [0,0] */ TNS("fadds",M), TNS("fmuls",M), TNS("fcoms",M), TNS("fcomps",M), +/* [0,4] */ TNS("fsubs",M), TNS("fsubrs",M), TNS("fdivs",M), TNS("fdivrs",M), +}, { +/* [1,0] */ TNS("flds",M), INVALID, TNS("fsts",M), TNS("fstps",M), +/* [1,4] */ TNSZ("fldenv",M,28), TNSZ("fldcw",M,2), TNSZ("fnstenv",M,28), TNSZ("fnstcw",M,2), +}, { +/* [2,0] */ TNS("fiaddl",M), TNS("fimull",M), TNS("ficoml",M), TNS("ficompl",M), +/* [2,4] */ TNS("fisubl",M), TNS("fisubrl",M), TNS("fidivl",M), TNS("fidivrl",M), +}, { +/* [3,0] */ TNS("fildl",M), TNSZ("tisttpl",M,4), TNS("fistl",M), TNS("fistpl",M), +/* [3,4] */ INVALID, TNSZ("fldt",M,10), INVALID, TNSZ("fstpt",M,10), +}, { +/* [4,0] */ TNSZ("faddl",M,8), TNSZ("fmull",M,8), TNSZ("fcoml",M,8), TNSZ("fcompl",M,8), +/* [4,1] */ TNSZ("fsubl",M,8), TNSZ("fsubrl",M,8), TNSZ("fdivl",M,8), TNSZ("fdivrl",M,8), +}, { +/* [5,0] */ TNSZ("fldl",M,8), TNSZ("fisttpll",M,8), TNSZ("fstl",M,8), TNSZ("fstpl",M,8), +/* [5,4] */ TNSZ("frstor",M,108), INVALID, TNSZ("fnsave",M,108), TNSZ("fnstsw",M,2), +}, { +/* [6,0] */ TNSZ("fiadd",M,2), TNSZ("fimul",M,2), TNSZ("ficom",M,2), TNSZ("ficomp",M,2), +/* [6,4] */ TNSZ("fisub",M,2), TNSZ("fisubr",M,2), TNSZ("fidiv",M,2), TNSZ("fidivr",M,2), +}, { +/* [7,0] */ TNSZ("fild",M,2), TNSZ("fisttp",M,2), TNSZ("fist",M,2), TNSZ("fistp",M,2), +/* [7,4] */ TNSZ("fbld",M,10), TNSZ("fildll",M,8), TNSZ("fbstp",M,10), TNSZ("fistpll",M,8), +} }; + +const instable_t dis_opFP3[8][8] = { +{ +/* bit pattern: 1101 1xxx 11xx xREG */ +/* [0,0] */ TNS("fadd",FF), TNS("fmul",FF), TNS("fcom",F), TNS("fcomp",F), +/* [0,4] */ TNS("fsub",FF), TNS("fsubr",FF), TNS("fdiv",FF), TNS("fdivr",FF), +}, { +/* [1,0] */ TNS("fld",F), TNS("fxch",F), TNS("fnop",NORM), TNS("fstp",F), +/* [1,4] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [2,0] */ INVALID, INVALID, INVALID, INVALID, +/* [2,4] */ INVALID, TNS("fucompp",NORM), INVALID, INVALID, +}, { +/* [3,0] */ INVALID, INVALID, INVALID, INVALID, +/* [3,4] */ INVALID, INVALID, INVALID, INVALID, +}, { +/* [4,0] */ TNS("fadd",FF), TNS("fmul",FF), TNS("fcom",F), TNS("fcomp",F), +/* [4,4] */ TNS("fsub",FF), TNS("fsubr",FF), TNS("fdiv",FF), TNS("fdivr",FF), +}, { +/* [5,0] */ TNS("ffree",F), TNS("fxch",F), TNS("fst",F), TNS("fstp",F), +/* [5,4] */ TNS("fucom",F), TNS("fucomp",F), INVALID, INVALID, +}, { +/* [6,0] */ TNS("faddp",FF), TNS("fmulp",FF), TNS("fcomp",F), TNS("fcompp",NORM), +/* [6,4] */ TNS("fsubp",FF), TNS("fsubrp",FF), TNS("fdivp",FF), TNS("fdivrp",FF), +}, { +/* [7,0] */ TNS("ffreep",F), TNS("fxch",F), TNS("fstp",F), TNS("fstp",F), +/* [7,4] */ TNS("fnstsw",M), TNS("fucomip",FFC), TNS("fcomip",FFC), INVALID, +} }; + +const instable_t dis_opFP4[4][8] = { +{ +/* bit pattern: 1101 1001 111x xxxx */ +/* [0,0] */ TNS("fchs",NORM), TNS("fabs",NORM), INVALID, INVALID, +/* [0,4] */ TNS("ftst",NORM), TNS("fxam",NORM), TNS("ftstp",NORM), INVALID, +}, { +/* [1,0] */ TNS("fld1",NORM), TNS("fldl2t",NORM), TNS("fldl2e",NORM), TNS("fldpi",NORM), +/* [1,4] */ TNS("fldlg2",NORM), TNS("fldln2",NORM), TNS("fldz",NORM), INVALID, +}, { +/* [2,0] */ TNS("f2xm1",NORM), TNS("fyl2x",NORM), TNS("fptan",NORM), TNS("fpatan",NORM), +/* [2,4] */ TNS("fxtract",NORM), TNS("fprem1",NORM), TNS("fdecstp",NORM), TNS("fincstp",NORM), +}, { +/* [3,0] */ TNS("fprem",NORM), TNS("fyl2xp1",NORM), TNS("fsqrt",NORM), TNS("fsincos",NORM), +/* [3,4] */ TNS("frndint",NORM), TNS("fscale",NORM), TNS("fsin",NORM), TNS("fcos",NORM), +} }; + +const instable_t dis_opFP5[8] = { +/* bit pattern: 1101 1011 111x xxxx */ +/* [0] */ TNS("feni",NORM), TNS("fdisi",NORM), TNS("fnclex",NORM), TNS("fninit",NORM), +/* [4] */ TNS("fsetpm",NORM), TNS("frstpm",NORM), INVALID, INVALID, +}; + +const instable_t dis_opFP6[8] = { +/* bit pattern: 1101 1011 11yy yxxx */ +/* [00] */ TNS("fcmov.nb",FF), TNS("fcmov.ne",FF), TNS("fcmov.nbe",FF), TNS("fcmov.nu",FF), +/* [04] */ INVALID, TNS("fucomi",F), TNS("fcomi",F), INVALID, +}; + +const instable_t dis_opFP7[8] = { +/* bit pattern: 1101 1010 11yy yxxx */ +/* [00] */ TNS("fcmov.b",FF), TNS("fcmov.e",FF), TNS("fcmov.be",FF), TNS("fcmov.u",FF), +/* [04] */ INVALID, INVALID, INVALID, INVALID, +}; + +/* + * Main decode table for the op codes. The first two nibbles + * will be used as an index into the table. If there is a + * a need to further decode an instruction, the array to be + * referenced is indicated with the other two entries being + * empty. + */ + +const instable_t dis_distable[16][16] = { +{ +/* [0,0] */ TNS("addb",RMw), TS("add",RMw), TNS("addb",MRw), TS("add",MRw), +/* [0,4] */ TNS("addb",IA), TS("add",IA), TSx("push",SEG), TSx("pop",SEG), +/* [0,8] */ TNS("orb",RMw), TS("or",RMw), TNS("orb",MRw), TS("or",MRw), +/* [0,C] */ TNS("orb",IA), TS("or",IA), TSx("push",SEG), IND(dis_op0F), +}, { +/* [1,0] */ TNS("adcb",RMw), TS("adc",RMw), TNS("adcb",MRw), TS("adc",MRw), +/* [1,4] */ TNS("adcb",IA), TS("adc",IA), TSx("push",SEG), TSx("pop",SEG), +/* [1,8] */ TNS("sbbb",RMw), TS("sbb",RMw), TNS("sbbb",MRw), TS("sbb",MRw), +/* [1,C] */ TNS("sbbb",IA), TS("sbb",IA), TSx("push",SEG), TSx("pop",SEG), +}, { +/* [2,0] */ TNS("andb",RMw), TS("and",RMw), TNS("andb",MRw), TS("and",MRw), +/* [2,4] */ TNS("andb",IA), TS("and",IA), TNSx("%es:",OVERRIDE), TNSx("daa",NORM), +/* [2,8] */ TNS("subb",RMw), TS("sub",RMw), TNS("subb",MRw), TS("sub",MRw), +/* [2,C] */ TNS("subb",IA), TS("sub",IA), TNS("%cs:",OVERRIDE), TNSx("das",NORM), +}, { +/* [3,0] */ TNS("xorb",RMw), TS("xor",RMw), TNS("xorb",MRw), TS("xor",MRw), +/* [3,4] */ TNS("xorb",IA), TS("xor",IA), TNSx("%ss:",OVERRIDE), TNSx("aaa",NORM), +/* [3,8] */ TNS("cmpb",RMw), TS("cmp",RMw), TNS("cmpb",MRw), TS("cmp",MRw), +/* [3,C] */ TNS("cmpb",IA), TS("cmp",IA), TNSx("%ds:",OVERRIDE), TNSx("aas",NORM), +}, { +/* [4,0] */ TSx("inc",R), TSx("inc",R), TSx("inc",R), TSx("inc",R), +/* [4,4] */ TSx("inc",R), TSx("inc",R), TSx("inc",R), TSx("inc",R), +/* [4,8] */ TSx("dec",R), TSx("dec",R), TSx("dec",R), TSx("dec",R), +/* [4,C] */ TSx("dec",R), TSx("dec",R), TSx("dec",R), TSx("dec",R), +}, { +/* [5,0] */ TSp("push",R), TSp("push",R), TSp("push",R), TSp("push",R), +/* [5,4] */ TSp("push",R), TSp("push",R), TSp("push",R), TSp("push",R), +/* [5,8] */ TSp("pop",R), TSp("pop",R), TSp("pop",R), TSp("pop",R), +/* [5,C] */ TSp("pop",R), TSp("pop",R), TSp("pop",R), TSp("pop",R), +}, { +/* [6,0] */ TSZx("pusha",IMPLMEM,28),TSZx("popa",IMPLMEM,28), TSx("bound",MR), TNS("arpl",RMw), +/* [6,4] */ TNS("%fs:",OVERRIDE), TNS("%gs:",OVERRIDE), TNS("data16",DM), TNS("addr16",AM), +/* [6,8] */ TSp("push",I), TS("imul",IMUL), TSp("push",Ib), TS("imul",IMUL), +/* [6,C] */ TNSZ("insb",IMPLMEM,1), TSZ("ins",IMPLMEM,4), TNSZ("outsb",IMPLMEM,1),TSZ("outs",IMPLMEM,4), +}, { +/* [7,0] */ TNSy("jo",BD), TNSy("jno",BD), TNSy("jb",BD), TNSy("jae",BD), +/* [7,4] */ TNSy("je",BD), TNSy("jne",BD), TNSy("jbe",BD), TNSy("ja",BD), +/* [7,8] */ TNSy("js",BD), TNSy("jns",BD), TNSy("jp",BD), TNSy("jnp",BD), +/* [7,C] */ TNSy("jl",BD), TNSy("jge",BD), TNSy("jle",BD), TNSy("jg",BD), +}, { +/* [8,0] */ IND(dis_op80), IND(dis_op81), INDx(dis_op82), IND(dis_op83), +/* [8,4] */ TNS("testb",RMw), TS("test",RMw), TNS("xchgb",RMw), TS("xchg",RMw), +/* [8,8] */ TNS("movb",RMw), TS("mov",RMw), TNS("movb",MRw), TS("mov",MRw), +/* [8,C] */ TNS("movw",SM), TS("lea",MR), TNS("movw",MS), TSp("pop",M), +}, { +/* [9,0] */ TNS("nop",NORM), TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), +/* [9,4] */ TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), +/* [9,8] */ TNS("cXtX",CBW), TNS("cXtX",CWD), TNSx("lcall",SO), TNS("fwait",NORM), +/* [9,C] */ TSZy("pushf",IMPLMEM,4),TSZy("popf",IMPLMEM,4), TNS("sahf",NORM), TNS("lahf",NORM), +}, { +/* [A,0] */ TNS("movb",OA), TS("mov",OA), TNS("movb",AO), TS("mov",AO), +/* [A,4] */ TNSZ("movsb",SD,1), TS("movs",SD), TNSZ("cmpsb",SD,1), TS("cmps",SD), +/* [A,8] */ TNS("testb",IA), TS("test",IA), TNS("stosb",AD), TS("stos",AD), +/* [A,C] */ TNS("lodsb",SA), TS("lods",SA), TNS("scasb",AD), TS("scas",AD), +}, { +/* [B,0] */ TNS("movb",IR), TNS("movb",IR), TNS("movb",IR), TNS("movb",IR), +/* [B,4] */ TNS("movb",IR), TNS("movb",IR), TNS("movb",IR), TNS("movb",IR), +/* [B,8] */ TS("mov",IR), TS("mov",IR), TS("mov",IR), TS("mov",IR), +/* [B,C] */ TS("mov",IR), TS("mov",IR), TS("mov",IR), TS("mov",IR), +}, { +/* [C,0] */ IND(dis_opC0), IND(dis_opC1), TNSyp("ret",RET), TNSyp("ret",NORM), +/* [C,4] */ TNSx("les",MR), TNSx("lds",MR), TNS("movb",IMw), TS("mov",IMw), +/* [C,8] */ TNSyp("enter",ENTER), TNSyp("leave",NORM), TNS("lret",RET), TNS("lret",NORM), +/* [C,C] */ TNS("int",INT3), TNS("int",INTx), TNSx("into",NORM), TNS("iret",NORM), +}, { +/* [D,0] */ IND(dis_opD0), IND(dis_opD1), IND(dis_opD2), IND(dis_opD3), +/* [D,4] */ TNSx("aam",U), TNSx("aad",U), TNSx("falc",NORM), TNSZ("xlat",IMPLMEM,1), + +/* 287 instructions. Note that although the indirect field */ +/* indicates opFP1n2 for further decoding, this is not necessarily */ +/* the case since the opFP arrays are not partitioned according to key1 */ +/* and key2. opFP1n2 is given only to indicate that we haven't */ +/* finished decoding the instruction. */ +/* [D,8] */ IND(dis_opFP1n2), IND(dis_opFP1n2), IND(dis_opFP1n2), IND(dis_opFP1n2), +/* [D,C] */ IND(dis_opFP1n2), IND(dis_opFP1n2), IND(dis_opFP1n2), IND(dis_opFP1n2), +}, { +/* [E,0] */ TNSy("loopnz",BD), TNSy("loopz",BD), TNSy("loop",BD), TNSy("jcxz",BD), +/* [E,4] */ TNS("inb",P), TS("in",P), TNS("outb",P), TS("out",P), +/* [E,8] */ TNSyp("call",D), TNSy("jmp",D), TNSx("ljmp",SO), TNSy("jmp",BD), +/* [E,C] */ TNS("inb",V), TS("in",V), TNS("outb",V), TS("out",V), +}, { +/* [F,0] */ TNS("lock",LOCK), TNS("icebp", NORM), TNS("repnz",PREFIX), TNS("repz",PREFIX), +/* [F,4] */ TNS("hlt",NORM), TNS("cmc",NORM), IND(dis_opF6), IND(dis_opF7), +/* [F,8] */ TNS("clc",NORM), TNS("stc",NORM), TNS("cli",NORM), TNS("sti",NORM), +/* [F,C] */ TNS("cld",NORM), TNS("std",NORM), IND(dis_opFE), IND(dis_opFF), +} }; + +/* END CSTYLED */ + +/* + * common functions to decode and disassemble an x86 or amd64 instruction + */ + +/* + * These are the individual fields of a REX prefix. Note that a REX + * prefix with none of these set is still needed to: + * - use the MOVSXD (sign extend 32 to 64 bits) instruction + * - access the %sil, %dil, %bpl, %spl registers + */ +#define REX_W 0x08 /* 64 bit operand size when set */ +#define REX_R 0x04 /* high order bit extension of ModRM reg field */ +#define REX_X 0x02 /* high order bit extension of SIB index field */ +#define REX_B 0x01 /* extends ModRM r_m, SIB base, or opcode reg */ + +/* + * These are the individual fields of a VEX prefix. + */ +#define VEX_R 0x08 /* REX.R in 1's complement form */ +#define VEX_X 0x04 /* REX.X in 1's complement form */ +#define VEX_B 0x02 /* REX.B in 1's complement form */ +/* Vector Length, 0: scalar or 128-bit vector, 1: 256-bit vector */ +#define VEX_L 0x04 +#define VEX_W 0x08 /* opcode specific, use like REX.W */ +#define VEX_m 0x1F /* VEX m-mmmm field */ +#define VEX_v 0x78 /* VEX register specifier */ +#define VEX_p 0x03 /* VEX pp field, opcode extension */ + +/* VEX m-mmmm field, only used by three bytes prefix */ +#define VEX_m_0F 0x01 /* implied 0F leading opcode byte */ +#define VEX_m_0F38 0x02 /* implied 0F 38 leading opcode byte */ +#define VEX_m_0F3A 0x03 /* implied 0F 3A leading opcode byte */ + +/* VEX pp field, providing equivalent functionality of a SIMD prefix */ +#define VEX_p_66 0x01 +#define VEX_p_F3 0x02 +#define VEX_p_F2 0x03 + +/* + * Even in 64 bit mode, usually only 4 byte immediate operands are supported. + */ +static int isize[] = {1, 2, 4, 4}; +static int isize64[] = {1, 2, 4, 8}; + +/* + * Just a bunch of useful macros. + */ +#define WBIT(x) (x & 0x1) /* to get w bit */ +#define REGNO(x) (x & 0x7) /* to get 3 bit register */ +#define VBIT(x) ((x)>>1 & 0x1) /* to get 'v' bit */ +#define OPSIZE(osize, wbit) ((wbit) ? isize[osize] : 1) +#define OPSIZE64(osize, wbit) ((wbit) ? isize64[osize] : 1) + +#define REG_ONLY 3 /* mode to indicate a register operand (not memory) */ + +#define BYTE_OPND 0 /* w-bit value indicating byte register */ +#define LONG_OPND 1 /* w-bit value indicating opnd_size register */ +#define MM_OPND 2 /* "value" used to indicate a mmx reg */ +#define XMM_OPND 3 /* "value" used to indicate a xmm reg */ +#define SEG_OPND 4 /* "value" used to indicate a segment reg */ +#define CONTROL_OPND 5 /* "value" used to indicate a control reg */ +#define DEBUG_OPND 6 /* "value" used to indicate a debug reg */ +#define TEST_OPND 7 /* "value" used to indicate a test reg */ +#define WORD_OPND 8 /* w-bit value indicating word size reg */ +#define YMM_OPND 9 /* "value" used to indicate a ymm reg */ + +/* + * The AVX2 gather instructions are a bit of a mess. While there's a pattern, + * there's not really a consistent scheme that we can use to know what the mode + * is supposed to be for a given type. Various instructions, like VPGATHERDD, + * always match the value of VEX_L. Other instructions like VPGATHERDQ, have + * some registers match VEX_L, but the VSIB is always XMM. + * + * The simplest way to deal with this is to just define a table based on the + * instruction opcodes, which are 0x90-0x93, so we subtract 0x90 to index into + * them. + * + * We further have to subdivide this based on the value of VEX_W and the value + * of VEX_L. The array is constructed to be indexed as: + * [opcode - 0x90][VEX_W][VEX_L]. + */ +/* w = 0, 0x90 */ +typedef struct dis_gather_regs { + uint_t dgr_arg0; /* src reg */ + uint_t dgr_arg1; /* vsib reg */ + uint_t dgr_arg2; /* dst reg */ + char *dgr_suffix; /* suffix to append */ +} dis_gather_regs_t; + +static dis_gather_regs_t dis_vgather[4][2][2] = { + { + /* op 0x90, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "d" } + }, + /* op 0x90, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "q" }, + { YMM_OPND, XMM_OPND, YMM_OPND, "q" } + } + }, + { + /* op 0x91, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { XMM_OPND, YMM_OPND, XMM_OPND, "d" }, + }, + /* op 0x91, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "q" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "q" }, + } + }, + { + /* op 0x92, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "s" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "s" } + }, + /* op 0x92, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, XMM_OPND, YMM_OPND, "d" } + } + }, + { + /* op 0x93, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "s" }, + { XMM_OPND, YMM_OPND, XMM_OPND, "s" } + }, + /* op 0x93, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "d" } + } + } +}; + +/* + * Get the next byte and separate the op code into the high and low nibbles. + */ +static int +dtrace_get_opcode(dis86_t *x, uint_t *high, uint_t *low) +{ + int byte; + + /* + * x86 instructions have a maximum length of 15 bytes. Bail out if + * we try to read more. + */ + if (x->d86_len >= 15) + return (x->d86_error = 1); + + if (x->d86_error) + return (1); + byte = x->d86_get_byte(x->d86_data); + if (byte < 0) + return (x->d86_error = 1); + x->d86_bytes[x->d86_len++] = byte; + *low = byte & 0xf; /* ----xxxx low 4 bits */ + *high = byte >> 4 & 0xf; /* xxxx---- bits 7 to 4 */ + return (0); +} + +/* + * Get and decode an SIB (scaled index base) byte + */ +static void +dtrace_get_SIB(dis86_t *x, uint_t *ss, uint_t *index, uint_t *base) +{ + int byte; + + if (x->d86_error) + return; + + byte = x->d86_get_byte(x->d86_data); + if (byte < 0) { + x->d86_error = 1; + return; + } + x->d86_bytes[x->d86_len++] = byte; + + *base = byte & 0x7; + *index = (byte >> 3) & 0x7; + *ss = (byte >> 6) & 0x3; +} + +/* + * Get the byte following the op code and separate it into the + * mode, register, and r/m fields. + */ +static void +dtrace_get_modrm(dis86_t *x, uint_t *mode, uint_t *reg, uint_t *r_m) +{ + if (x->d86_got_modrm == 0) { + if (x->d86_rmindex == -1) + x->d86_rmindex = x->d86_len; + dtrace_get_SIB(x, mode, reg, r_m); + x->d86_got_modrm = 1; + } +} + +/* + * Adjust register selection based on any REX prefix bits present. + */ +/*ARGSUSED*/ +static void +dtrace_rex_adjust(uint_t rex_prefix, uint_t mode, uint_t *reg, uint_t *r_m) +{ + if (reg != NULL && r_m == NULL) { + if (rex_prefix & REX_B) + *reg += 8; + } else { + if (reg != NULL && (REX_R & rex_prefix) != 0) + *reg += 8; + if (r_m != NULL && (REX_B & rex_prefix) != 0) + *r_m += 8; + } +} + +/* + * Adjust register selection based on any VEX prefix bits present. + * Notes: VEX.R, VEX.X and VEX.B use the inverted form compared with REX prefix + */ +/*ARGSUSED*/ +static void +dtrace_vex_adjust(uint_t vex_byte1, uint_t mode, uint_t *reg, uint_t *r_m) +{ + if (reg != NULL && r_m == NULL) { + if (!(vex_byte1 & VEX_B)) + *reg += 8; + } else { + if (reg != NULL && ((VEX_R & vex_byte1) == 0)) + *reg += 8; + if (r_m != NULL && ((VEX_B & vex_byte1) == 0)) + *r_m += 8; + } +} + +/* + * Get an immediate operand of the given size, with sign extension. + */ +static void +dtrace_imm_opnd(dis86_t *x, int wbit, int size, int opindex) +{ + int i; + int byte; + int valsize; + + if (x->d86_numopnds < opindex + 1) + x->d86_numopnds = opindex + 1; + + switch (wbit) { + case BYTE_OPND: + valsize = 1; + break; + case LONG_OPND: + if (x->d86_opnd_size == SIZE16) + valsize = 2; + else if (x->d86_opnd_size == SIZE32) + valsize = 4; + else + valsize = 8; + break; + case MM_OPND: + case XMM_OPND: + case YMM_OPND: + case SEG_OPND: + case CONTROL_OPND: + case DEBUG_OPND: + case TEST_OPND: + valsize = size; + break; + case WORD_OPND: + valsize = 2; + break; + } + if (valsize < size) + valsize = size; + + if (x->d86_error) + return; + x->d86_opnd[opindex].d86_value = 0; + for (i = 0; i < size; ++i) { + byte = x->d86_get_byte(x->d86_data); + if (byte < 0) { + x->d86_error = 1; + return; + } + x->d86_bytes[x->d86_len++] = byte; + x->d86_opnd[opindex].d86_value |= (uint64_t)byte << (i * 8); + } + /* Do sign extension */ + if (x->d86_bytes[x->d86_len - 1] & 0x80) { + for (; i < sizeof (uint64_t); i++) + x->d86_opnd[opindex].d86_value |= + (uint64_t)0xff << (i * 8); + } +#ifdef DIS_TEXT + x->d86_opnd[opindex].d86_mode = MODE_SIGNED; + x->d86_opnd[opindex].d86_value_size = valsize; + x->d86_imm_bytes += size; +#endif +} + +/* + * Get an ip relative operand of the given size, with sign extension. + */ +static void +dtrace_disp_opnd(dis86_t *x, int wbit, int size, int opindex) +{ + dtrace_imm_opnd(x, wbit, size, opindex); +#ifdef DIS_TEXT + x->d86_opnd[opindex].d86_mode = MODE_IPREL; +#endif +} + +/* + * Check to see if there is a segment override prefix pending. + * If so, print it in the current 'operand' location and set + * the override flag back to false. + */ +/*ARGSUSED*/ +static void +dtrace_check_override(dis86_t *x, int opindex) +{ +#ifdef DIS_TEXT + if (x->d86_seg_prefix) { + (void) strlcat(x->d86_opnd[opindex].d86_prefix, + x->d86_seg_prefix, PFIXLEN); + } +#endif + x->d86_seg_prefix = NULL; +} + + +/* + * Process a single instruction Register or Memory operand. + * + * mode = addressing mode from ModRM byte + * r_m = r_m (or reg if mode == 3) field from ModRM byte + * wbit = indicates which register (8bit, 16bit, ... MMX, etc.) set to use. + * o = index of operand that we are processing (0, 1 or 2) + * + * the value of reg or r_m must have already been adjusted for any REX prefix. + */ +/*ARGSUSED*/ +static void +dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) +{ + int have_SIB = 0; /* flag presence of scale-index-byte */ + uint_t ss; /* scale-factor from opcode */ + uint_t index; /* index register number */ + uint_t base; /* base register number */ + int dispsize; /* size of displacement in bytes */ +#ifdef DIS_TEXT + char *opnd = x->d86_opnd[opindex].d86_opnd; +#endif + + if (x->d86_numopnds < opindex + 1) + x->d86_numopnds = opindex + 1; + + if (x->d86_error) + return; + + /* + * first handle a simple register + */ + if (mode == REG_ONLY) { +#ifdef DIS_TEXT + switch (wbit) { + case MM_OPND: + (void) strlcat(opnd, dis_MMREG[r_m], OPLEN); + break; + case XMM_OPND: + (void) strlcat(opnd, dis_XMMREG[r_m], OPLEN); + break; + case YMM_OPND: + (void) strlcat(opnd, dis_YMMREG[r_m], OPLEN); + break; + case SEG_OPND: + (void) strlcat(opnd, dis_SEGREG[r_m], OPLEN); + break; + case CONTROL_OPND: + (void) strlcat(opnd, dis_CONTROLREG[r_m], OPLEN); + break; + case DEBUG_OPND: + (void) strlcat(opnd, dis_DEBUGREG[r_m], OPLEN); + break; + case TEST_OPND: + (void) strlcat(opnd, dis_TESTREG[r_m], OPLEN); + break; + case BYTE_OPND: + if (x->d86_rex_prefix == 0) + (void) strlcat(opnd, dis_REG8[r_m], OPLEN); + else + (void) strlcat(opnd, dis_REG8_REX[r_m], OPLEN); + break; + case WORD_OPND: + (void) strlcat(opnd, dis_REG16[r_m], OPLEN); + break; + case LONG_OPND: + if (x->d86_opnd_size == SIZE16) + (void) strlcat(opnd, dis_REG16[r_m], OPLEN); + else if (x->d86_opnd_size == SIZE32) + (void) strlcat(opnd, dis_REG32[r_m], OPLEN); + else + (void) strlcat(opnd, dis_REG64[r_m], OPLEN); + break; + } +#endif /* DIS_TEXT */ + return; + } + + /* + * if symbolic representation, skip override prefix, if any + */ + dtrace_check_override(x, opindex); + + /* + * Handle 16 bit memory references first, since they decode + * the mode values more simply. + * mode 1 is r_m + 8 bit displacement + * mode 2 is r_m + 16 bit displacement + * mode 0 is just r_m, unless r_m is 6 which is 16 bit disp + */ + if (x->d86_addr_size == SIZE16) { + if ((mode == 0 && r_m == 6) || mode == 2) + dtrace_imm_opnd(x, WORD_OPND, 2, opindex); + else if (mode == 1) + dtrace_imm_opnd(x, BYTE_OPND, 1, opindex); +#ifdef DIS_TEXT + if (mode == 0 && r_m == 6) + x->d86_opnd[opindex].d86_mode = MODE_SIGNED; + else if (mode == 0) + x->d86_opnd[opindex].d86_mode = MODE_NONE; + else + x->d86_opnd[opindex].d86_mode = MODE_OFFSET; + (void) strlcat(opnd, dis_addr16[mode][r_m], OPLEN); +#endif + return; + } + + /* + * 32 and 64 bit addressing modes are more complex since they + * can involve an SIB (scaled index and base) byte to decode. + */ + if (r_m == ESP_REGNO || r_m == ESP_REGNO + 8) { + have_SIB = 1; + dtrace_get_SIB(x, &ss, &index, &base); + if (x->d86_error) + return; + if (base != 5 || mode != 0) + if (x->d86_rex_prefix & REX_B) + base += 8; + if (x->d86_rex_prefix & REX_X) + index += 8; + } else { + base = r_m; + } + + /* + * Compute the displacement size and get its bytes + */ + dispsize = 0; + + if (mode == 1) + dispsize = 1; + else if (mode == 2) + dispsize = 4; + else if ((r_m & 7) == EBP_REGNO || + (have_SIB && (base & 7) == EBP_REGNO)) + dispsize = 4; + + if (dispsize > 0) { + dtrace_imm_opnd(x, dispsize == 4 ? LONG_OPND : BYTE_OPND, + dispsize, opindex); + if (x->d86_error) + return; + } + +#ifdef DIS_TEXT + if (dispsize > 0) + x->d86_opnd[opindex].d86_mode = MODE_OFFSET; + + if (have_SIB == 0) { + if (x->d86_mode == SIZE32) { + if (mode == 0) + (void) strlcat(opnd, dis_addr32_mode0[r_m], + OPLEN); + else + (void) strlcat(opnd, dis_addr32_mode12[r_m], + OPLEN); + } else { + if (mode == 0) { + (void) strlcat(opnd, dis_addr64_mode0[r_m], + OPLEN); + if (r_m == 5) { + x->d86_opnd[opindex].d86_mode = + MODE_RIPREL; + } + } else { + (void) strlcat(opnd, dis_addr64_mode12[r_m], + OPLEN); + } + } + } else { + uint_t need_paren = 0; + char **regs; + char **bregs; + const char *const *sf; + if (x->d86_mode == SIZE32) /* NOTE this is not addr_size! */ + regs = (char **)dis_REG32; + else + regs = (char **)dis_REG64; + + if (x->d86_vsib != 0) { + if (wbit == YMM_OPND) /* NOTE this is not addr_size! */ + bregs = (char **)dis_YMMREG; + else + bregs = (char **)dis_XMMREG; + sf = dis_vscale_factor; + } else { + bregs = regs; + sf = dis_scale_factor; + } + + /* + * print the base (if any) + */ + if (base == EBP_REGNO && mode == 0) { + if (index != ESP_REGNO || x->d86_vsib != 0) { + (void) strlcat(opnd, "(", OPLEN); + need_paren = 1; + } + } else { + (void) strlcat(opnd, "(", OPLEN); + (void) strlcat(opnd, regs[base], OPLEN); + need_paren = 1; + } + + /* + * print the index (if any) + */ + if (index != ESP_REGNO || x->d86_vsib) { + (void) strlcat(opnd, ",", OPLEN); + (void) strlcat(opnd, bregs[index], OPLEN); + (void) strlcat(opnd, sf[ss], OPLEN); + } else + if (need_paren) + (void) strlcat(opnd, ")", OPLEN); + } +#endif +} + +/* + * Operand sequence for standard instruction involving one register + * and one register/memory operand. + * wbit indicates a byte(0) or opnd_size(1) operation + * vbit indicates direction (0 for "opcode r,r_m") or (1 for "opcode r_m, r") + */ +#define STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, vbit) { \ + dtrace_get_modrm(x, &mode, ®, &r_m); \ + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); \ + dtrace_get_operand(x, mode, r_m, wbit, vbit); \ + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1 - vbit); \ +} + +/* + * Similar to above, but allows for the two operands to be of different + * classes (ie. wbit). + * wbit is for the r_m operand + * w2 is for the reg operand + */ +#define MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, w2, vbit) { \ + dtrace_get_modrm(x, &mode, ®, &r_m); \ + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); \ + dtrace_get_operand(x, mode, r_m, wbit, vbit); \ + dtrace_get_operand(x, REG_ONLY, reg, w2, 1 - vbit); \ +} + +/* + * Similar, but for 2 operands plus an immediate. + * vbit indicates direction + * 0 for "opcode imm, r, r_m" or + * 1 for "opcode imm, r_m, r" + */ +#define THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, w2, immsize, vbit) { \ + dtrace_get_modrm(x, &mode, ®, &r_m); \ + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); \ + dtrace_get_operand(x, mode, r_m, wbit, 2-vbit); \ + dtrace_get_operand(x, REG_ONLY, reg, w2, 1+vbit); \ + dtrace_imm_opnd(x, wbit, immsize, 0); \ +} + +/* + * Similar, but for 2 operands plus two immediates. + */ +#define FOUROPERAND(x, mode, reg, r_m, rex_prefix, wbit, w2, immsize) { \ + dtrace_get_modrm(x, &mode, ®, &r_m); \ + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); \ + dtrace_get_operand(x, mode, r_m, wbit, 2); \ + dtrace_get_operand(x, REG_ONLY, reg, w2, 3); \ + dtrace_imm_opnd(x, wbit, immsize, 1); \ + dtrace_imm_opnd(x, wbit, immsize, 0); \ +} + +/* + * 1 operands plus two immediates. + */ +#define ONEOPERAND_TWOIMM(x, mode, reg, r_m, rex_prefix, wbit, immsize) { \ + dtrace_get_modrm(x, &mode, ®, &r_m); \ + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); \ + dtrace_get_operand(x, mode, r_m, wbit, 2); \ + dtrace_imm_opnd(x, wbit, immsize, 1); \ + dtrace_imm_opnd(x, wbit, immsize, 0); \ +} + +/* + * Dissassemble a single x86 or amd64 instruction. + * + * Mode determines the default operating mode (SIZE16, SIZE32 or SIZE64) + * for interpreting instructions. + * + * returns non-zero for bad opcode + */ +int +dtrace_disx86(dis86_t *x, uint_t cpu_mode) +{ + instable_t *dp; /* decode table being used */ +#ifdef DIS_TEXT + uint_t i; +#endif +#ifdef DIS_MEM + uint_t nomem = 0; +#define NOMEM (nomem = 1) +#else +#define NOMEM /* nothing */ +#endif + uint_t opnd_size; /* SIZE16, SIZE32 or SIZE64 */ + uint_t addr_size; /* SIZE16, SIZE32 or SIZE64 */ + uint_t wbit; /* opcode wbit, 0 is 8 bit, !0 for opnd_size */ + uint_t w2; /* wbit value for second operand */ + uint_t vbit; + uint_t mode = 0; /* mode value from ModRM byte */ + uint_t reg; /* reg value from ModRM byte */ + uint_t r_m; /* r_m value from ModRM byte */ + + uint_t opcode1; /* high nibble of 1st byte */ + uint_t opcode2; /* low nibble of 1st byte */ + uint_t opcode3; /* extra opcode bits usually from ModRM byte */ + uint_t opcode4; /* high nibble of 2nd byte */ + uint_t opcode5; /* low nibble of 2nd byte */ + uint_t opcode6; /* high nibble of 3rd byte */ + uint_t opcode7; /* low nibble of 3rd byte */ + uint_t opcode_bytes = 1; + + /* + * legacy prefixes come in 5 flavors, you should have only one of each + */ + uint_t opnd_size_prefix = 0; + uint_t addr_size_prefix = 0; + uint_t segment_prefix = 0; + uint_t lock_prefix = 0; + uint_t rep_prefix = 0; + uint_t rex_prefix = 0; /* amd64 register extension prefix */ + + /* + * Intel VEX instruction encoding prefix and fields + */ + + /* 0xC4 means 3 bytes prefix, 0xC5 means 2 bytes prefix */ + uint_t vex_prefix = 0; + + /* + * VEX prefix byte 1, includes vex.r, vex.x and vex.b + * (for 3 bytes prefix) + */ + uint_t vex_byte1 = 0; + + /* + * For 32-bit mode, it should prefetch the next byte to + * distinguish between AVX and les/lds + */ + uint_t vex_prefetch = 0; + + uint_t vex_m = 0; + uint_t vex_v = 0; + uint_t vex_p = 0; + uint_t vex_R = 1; + uint_t vex_X = 1; + uint_t vex_B = 1; + uint_t vex_W = 0; + uint_t vex_L; + dis_gather_regs_t *vreg; + +#ifdef DIS_TEXT + /* Instruction name for BLS* family of instructions */ + char *blsinstr; +#endif + + size_t off; + + instable_t dp_mmx; + + x->d86_len = 0; + x->d86_rmindex = -1; + x->d86_error = 0; +#ifdef DIS_TEXT + x->d86_numopnds = 0; + x->d86_seg_prefix = NULL; + x->d86_mnem[0] = 0; + for (i = 0; i < 4; ++i) { + x->d86_opnd[i].d86_opnd[0] = 0; + x->d86_opnd[i].d86_prefix[0] = 0; + x->d86_opnd[i].d86_value_size = 0; + x->d86_opnd[i].d86_value = 0; + x->d86_opnd[i].d86_mode = MODE_NONE; + } +#endif + x->d86_rex_prefix = 0; + x->d86_got_modrm = 0; + x->d86_memsize = 0; + x->d86_vsib = 0; + + if (cpu_mode == SIZE16) { + opnd_size = SIZE16; + addr_size = SIZE16; + } else if (cpu_mode == SIZE32) { + opnd_size = SIZE32; + addr_size = SIZE32; + } else { + opnd_size = SIZE32; + addr_size = SIZE64; + } + + /* + * Get one opcode byte and check for zero padding that follows + * jump tables. + */ + if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0) + goto error; + + if (opcode1 == 0 && opcode2 == 0 && + x->d86_check_func != NULL && x->d86_check_func(x->d86_data)) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, ".byte\t0", OPLEN); +#endif + goto done; + } + + /* + * Gather up legacy x86 prefix bytes. + */ + for (;;) { + uint_t *which_prefix = NULL; + + dp = (instable_t *)&dis_distable[opcode1][opcode2]; + + switch (dp->it_adrmode) { + case PREFIX: + which_prefix = &rep_prefix; + break; + case LOCK: + which_prefix = &lock_prefix; + break; + case OVERRIDE: + which_prefix = &segment_prefix; +#ifdef DIS_TEXT + x->d86_seg_prefix = (char *)dp->it_name; +#endif + if (dp->it_invalid64 && cpu_mode == SIZE64) + goto error; + break; + case AM: + which_prefix = &addr_size_prefix; + break; + case DM: + which_prefix = &opnd_size_prefix; + break; + } + if (which_prefix == NULL) + break; + *which_prefix = (opcode1 << 4) | opcode2; + if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0) + goto error; + } + + /* + * Handle amd64 mode PREFIX values. + * Some of the segment prefixes are no-ops. (only FS/GS actually work) + * We might have a REX prefix (opcodes 0x40-0x4f) + */ + if (cpu_mode == SIZE64) { + if (segment_prefix != 0x64 && segment_prefix != 0x65) + segment_prefix = 0; + + if (opcode1 == 0x4) { + rex_prefix = (opcode1 << 4) | opcode2; + if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0) + goto error; + dp = (instable_t *)&dis_distable[opcode1][opcode2]; + } else if (opcode1 == 0xC && + (opcode2 == 0x4 || opcode2 == 0x5)) { + /* AVX instructions */ + vex_prefix = (opcode1 << 4) | opcode2; + x->d86_rex_prefix = 0x40; + } + } else if (opcode1 == 0xC && (opcode2 == 0x4 || opcode2 == 0x5)) { + /* LDS, LES or AVX */ + dtrace_get_modrm(x, &mode, ®, &r_m); + vex_prefetch = 1; + + if (mode == REG_ONLY) { + /* AVX */ + vex_prefix = (opcode1 << 4) | opcode2; + x->d86_rex_prefix = 0x40; + opcode3 = (((mode << 3) | reg)>>1) & 0x0F; + opcode4 = ((reg << 3) | r_m) & 0x0F; + } + } + + if (vex_prefix == VEX_2bytes) { + if (!vex_prefetch) { + if (dtrace_get_opcode(x, &opcode3, &opcode4) != 0) + goto error; + } + vex_R = ((opcode3 & VEX_R) & 0x0F) >> 3; + vex_L = ((opcode4 & VEX_L) & 0x0F) >> 2; + vex_v = (((opcode3 << 4) | opcode4) & VEX_v) >> 3; + vex_p = opcode4 & VEX_p; + /* + * The vex.x and vex.b bits are not defined in two bytes + * mode vex prefix, their default values are 1 + */ + vex_byte1 = (opcode3 & VEX_R) | VEX_X | VEX_B; + + if (vex_R == 0) + x->d86_rex_prefix |= REX_R; + + if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0) + goto error; + + switch (vex_p) { + case VEX_p_66: + dp = (instable_t *) + &dis_opAVX660F[(opcode1 << 4) | opcode2]; + break; + case VEX_p_F3: + dp = (instable_t *) + &dis_opAVXF30F[(opcode1 << 4) | opcode2]; + break; + case VEX_p_F2: + dp = (instable_t *) + &dis_opAVXF20F [(opcode1 << 4) | opcode2]; + break; + default: + dp = (instable_t *) + &dis_opAVX0F[opcode1][opcode2]; + + } + + } else if (vex_prefix == VEX_3bytes) { + if (!vex_prefetch) { + if (dtrace_get_opcode(x, &opcode3, &opcode4) != 0) + goto error; + } + vex_R = (opcode3 & VEX_R) >> 3; + vex_X = (opcode3 & VEX_X) >> 2; + vex_B = (opcode3 & VEX_B) >> 1; + vex_m = (((opcode3 << 4) | opcode4) & VEX_m); + vex_byte1 = opcode3 & (VEX_R | VEX_X | VEX_B); + + if (vex_R == 0) + x->d86_rex_prefix |= REX_R; + if (vex_X == 0) + x->d86_rex_prefix |= REX_X; + if (vex_B == 0) + x->d86_rex_prefix |= REX_B; + + if (dtrace_get_opcode(x, &opcode5, &opcode6) != 0) + goto error; + vex_W = (opcode5 & VEX_W) >> 3; + vex_L = (opcode6 & VEX_L) >> 2; + vex_v = (((opcode5 << 4) | opcode6) & VEX_v) >> 3; + vex_p = opcode6 & VEX_p; + + if (vex_W) + x->d86_rex_prefix |= REX_W; + + /* Only these three vex_m values valid; others are reserved */ + if ((vex_m != VEX_m_0F) && (vex_m != VEX_m_0F38) && + (vex_m != VEX_m_0F3A)) + goto error; + + if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0) + goto error; + + switch (vex_p) { + case VEX_p_66: + if (vex_m == VEX_m_0F) { + dp = (instable_t *) + &dis_opAVX660F + [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F38) { + dp = (instable_t *) + &dis_opAVX660F38 + [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F3A) { + dp = (instable_t *) + &dis_opAVX660F3A + [(opcode1 << 4) | opcode2]; + } else { + goto error; + } + break; + case VEX_p_F3: + if (vex_m == VEX_m_0F) { + dp = (instable_t *) + &dis_opAVXF30F + [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F38) { + dp = (instable_t *) + &dis_opAVXF30F38 + [(opcode1 << 4) | opcode2]; + } else { + goto error; + } + break; + case VEX_p_F2: + if (vex_m == VEX_m_0F) { + dp = (instable_t *) + &dis_opAVXF20F + [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F3A) { + dp = (instable_t *) + &dis_opAVXF20F3A + [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F38) { + dp = (instable_t *) + &dis_opAVXF20F38 + [(opcode1 << 4) | opcode2]; + } else { + goto error; + } + break; + default: + dp = (instable_t *) + &dis_opAVX0F[opcode1][opcode2]; + + } + } + if (vex_prefix) { + if (dp->it_vexwoxmm) { + wbit = LONG_OPND; + } else { + if (vex_L) + wbit = YMM_OPND; + else + wbit = XMM_OPND; + } + } + + /* + * Deal with selection of operand and address size now. + * Note that the REX.W bit being set causes opnd_size_prefix to be + * ignored. + */ + if (cpu_mode == SIZE64) { + if ((rex_prefix & REX_W) || vex_W) + opnd_size = SIZE64; + else if (opnd_size_prefix) + opnd_size = SIZE16; + + if (addr_size_prefix) + addr_size = SIZE32; + } else if (cpu_mode == SIZE32) { + if (opnd_size_prefix) + opnd_size = SIZE16; + if (addr_size_prefix) + addr_size = SIZE16; + } else { + if (opnd_size_prefix) + opnd_size = SIZE32; + if (addr_size_prefix) + addr_size = SIZE32; + } + /* + * The pause instruction - a repz'd nop. This doesn't fit + * with any of the other prefix goop added for SSE, so we'll + * special-case it here. + */ + if (rep_prefix == 0xf3 && opcode1 == 0x9 && opcode2 == 0x0) { + rep_prefix = 0; + dp = (instable_t *)&dis_opPause; + } + + /* + * Some 386 instructions have 2 bytes of opcode before the mod_r/m + * byte so we may need to perform a table indirection. + */ + if (dp->it_indirect == (instable_t *)dis_op0F) { + if (dtrace_get_opcode(x, &opcode4, &opcode5) != 0) + goto error; + opcode_bytes = 2; + if (opcode4 == 0x7 && opcode5 >= 0x1 && opcode5 <= 0x3) { + uint_t subcode; + + if (dtrace_get_opcode(x, &opcode6, &opcode7) != 0) + goto error; + opcode_bytes = 3; + subcode = ((opcode6 & 0x3) << 1) | + ((opcode7 & 0x8) >> 3); + dp = (instable_t *)&dis_op0F7123[opcode5][subcode]; + } else if ((opcode4 == 0xc) && (opcode5 >= 0x8)) { + dp = (instable_t *)&dis_op0FC8[0]; + } else if ((opcode4 == 0x3) && (opcode5 == 0xA)) { + opcode_bytes = 3; + if (dtrace_get_opcode(x, &opcode6, &opcode7) != 0) + goto error; + if (opnd_size == SIZE16) + opnd_size = SIZE32; + + dp = (instable_t *)&dis_op0F3A[(opcode6<<4)|opcode7]; +#ifdef DIS_TEXT + if (strcmp(dp->it_name, "INVALID") == 0) + goto error; +#endif + switch (dp->it_adrmode) { + case XMMP: + break; + case XMMP_66r: + case XMMPRM_66r: + case XMM3PM_66r: + if (opnd_size_prefix == 0) { + goto error; + } + break; + case XMMP_66o: + if (opnd_size_prefix == 0) { + /* SSSE3 MMX instructions */ + dp_mmx = *dp; + dp = &dp_mmx; + dp->it_adrmode = MMOPM_66o; +#ifdef DIS_MEM + dp->it_size = 8; +#endif + } + break; + default: + goto error; + } + } else if ((opcode4 == 0x3) && (opcode5 == 0x8)) { + opcode_bytes = 3; + if (dtrace_get_opcode(x, &opcode6, &opcode7) != 0) + goto error; + dp = (instable_t *)&dis_op0F38[(opcode6<<4)|opcode7]; + + /* + * Both crc32 and movbe have the same 3rd opcode + * byte of either 0xF0 or 0xF1, so we use another + * indirection to distinguish between the two. + */ + if (dp->it_indirect == (instable_t *)dis_op0F38F0 || + dp->it_indirect == (instable_t *)dis_op0F38F1) { + + dp = dp->it_indirect; + if (rep_prefix != 0xF2) { + /* It is movbe */ + dp++; + } + } + + /* + * The adx family of instructions (adcx and adox) + * continue the classic Intel tradition of abusing + * arbitrary prefixes without actually meaning the + * prefix bit. Therefore, if we find either the + * opnd_size_prefix or rep_prefix we end up zeroing it + * out after making our determination so as to ensure + * that we don't get confused and accidentally print + * repz prefixes and the like on these instructions. + * + * In addition, these instructions are actually much + * closer to AVX instructions in semantics. Importantly, + * they always default to having 32-bit operands. + * However, if the CPU is in 64-bit mode, then and only + * then, does it use REX.w promotes things to 64-bits + * and REX.r allows 64-bit mode to use register r8-r15. + */ + if (dp->it_indirect == (instable_t *)dis_op0F38F6) { + dp = dp->it_indirect; + if (opnd_size_prefix == 0 && + rep_prefix == 0xf3) { + /* It is adox */ + dp++; + } else if (opnd_size_prefix != 0x66 && + rep_prefix != 0) { + /* It isn't adcx */ + goto error; + } + opnd_size_prefix = 0; + rep_prefix = 0; + opnd_size = SIZE32; + if (rex_prefix & REX_W) + opnd_size = SIZE64; + } + +#ifdef DIS_TEXT + if (strcmp(dp->it_name, "INVALID") == 0) + goto error; +#endif + switch (dp->it_adrmode) { + case ADX: + case XMM: + break; + case RM_66r: + case XMM_66r: + case XMMM_66r: + if (opnd_size_prefix == 0) { + goto error; + } + break; + case XMM_66o: + if (opnd_size_prefix == 0) { + /* SSSE3 MMX instructions */ + dp_mmx = *dp; + dp = &dp_mmx; + dp->it_adrmode = MM; +#ifdef DIS_MEM + dp->it_size = 8; +#endif + } + break; + case CRC32: + if (rep_prefix != 0xF2) { + goto error; + } + rep_prefix = 0; + break; + case MOVBE: + if (rep_prefix != 0x0) { + goto error; + } + break; + default: + goto error; + } + } else { + dp = (instable_t *)&dis_op0F[opcode4][opcode5]; + } + } + + /* + * If still not at a TERM decode entry, then a ModRM byte + * exists and its fields further decode the instruction. + */ + x->d86_got_modrm = 0; + if (dp->it_indirect != TERM) { + dtrace_get_modrm(x, &mode, &opcode3, &r_m); + if (x->d86_error) + goto error; + reg = opcode3; + + /* + * decode 287 instructions (D8-DF) from opcodeN + */ + if (opcode1 == 0xD && opcode2 >= 0x8) { + if (opcode2 == 0xB && mode == 0x3 && opcode3 == 4) + dp = (instable_t *)&dis_opFP5[r_m]; + else if (opcode2 == 0xA && mode == 0x3 && opcode3 < 4) + dp = (instable_t *)&dis_opFP7[opcode3]; + else if (opcode2 == 0xB && mode == 0x3) + dp = (instable_t *)&dis_opFP6[opcode3]; + else if (opcode2 == 0x9 && mode == 0x3 && opcode3 >= 4) + dp = (instable_t *)&dis_opFP4[opcode3 - 4][r_m]; + else if (mode == 0x3) + dp = (instable_t *) + &dis_opFP3[opcode2 - 8][opcode3]; + else + dp = (instable_t *) + &dis_opFP1n2[opcode2 - 8][opcode3]; + } else { + dp = (instable_t *)dp->it_indirect + opcode3; + } + } + + /* + * In amd64 bit mode, ARPL opcode is changed to MOVSXD + * (sign extend 32bit to 64 bit) + */ + if ((vex_prefix == 0) && cpu_mode == SIZE64 && + opcode1 == 0x6 && opcode2 == 0x3) + dp = (instable_t *)&dis_opMOVSLD; + + /* + * at this point we should have a correct (or invalid) opcode + */ + if (cpu_mode == SIZE64 && dp->it_invalid64 || + cpu_mode != SIZE64 && dp->it_invalid32) + goto error; + if (dp->it_indirect != TERM) + goto error; + + /* + * Deal with MMX/SSE opcodes which are changed by prefixes. Note, we do + * need to include UNKNOWN below, as we may have instructions that + * actually have a prefix, but don't exist in any other form. + */ + switch (dp->it_adrmode) { + case UNKNOWN: + case MMO: + case MMOIMPL: + case MMO3P: + case MMOM3: + case MMOMS: + case MMOPM: + case MMOPRM: + case MMOS: + case XMMO: + case XMMOM: + case XMMOMS: + case XMMOPM: + case XMMOS: + case XMMOMX: + case XMMOX3: + case XMMOXMM: + /* + * This is horrible. Some SIMD instructions take the + * form 0x0F 0x?? ..., which is easily decoded using the + * existing tables. Other SIMD instructions use various + * prefix bytes to overload existing instructions. For + * Example, addps is F0, 58, whereas addss is F3 (repz), + * F0, 58. Presumably someone got a raise for this. + * + * If we see one of the instructions which can be + * modified in this way (if we've got one of the SIMDO* + * address modes), we'll check to see if the last prefix + * was a repz. If it was, we strip the prefix from the + * mnemonic, and we indirect using the dis_opSIMDrepz + * table. + */ + + /* + * Calculate our offset in dis_op0F + */ + if ((uintptr_t)dp - (uintptr_t)dis_op0F > sizeof (dis_op0F)) + goto error; + + off = ((uintptr_t)dp - (uintptr_t)dis_op0F) / + sizeof (instable_t); + + /* + * Rewrite if this instruction used one of the magic prefixes. + */ + if (rep_prefix) { + if (rep_prefix == 0xf2) + dp = (instable_t *)&dis_opSIMDrepnz[off]; + else + dp = (instable_t *)&dis_opSIMDrepz[off]; + rep_prefix = 0; + } else if (opnd_size_prefix) { + dp = (instable_t *)&dis_opSIMDdata16[off]; + opnd_size_prefix = 0; + if (opnd_size == SIZE16) + opnd_size = SIZE32; + } + break; + + case MG9: + /* + * More horribleness: the group 9 (0xF0 0xC7) instructions are + * allowed an optional prefix of 0x66 or 0xF3. This is similar + * to the SIMD business described above, but with a different + * addressing mode (and an indirect table), so we deal with it + * separately (if similarly). + * + * Intel further complicated this with the release of Ivy Bridge + * where they overloaded these instructions based on the ModR/M + * bytes. The VMX instructions have a mode of 0 since they are + * memory instructions but rdrand instructions have a mode of + * 0b11 (REG_ONLY) because they only operate on registers. While + * there are different prefix formats, for now it is sufficient + * to use a single different table. + */ + + /* + * Calculate our offset in dis_op0FC7 (the group 9 table) + */ + if ((uintptr_t)dp - (uintptr_t)dis_op0FC7 > sizeof (dis_op0FC7)) + goto error; + + off = ((uintptr_t)dp - (uintptr_t)dis_op0FC7) / + sizeof (instable_t); + + /* + * If we have a mode of 0b11 then we have to rewrite this. + */ + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode == REG_ONLY) { + dp = (instable_t *)&dis_op0FC7m3[off]; + break; + } + + /* + * Rewrite if this instruction used one of the magic prefixes. + */ + if (rep_prefix) { + if (rep_prefix == 0xf3) + dp = (instable_t *)&dis_opF30FC7[off]; + else + goto error; + rep_prefix = 0; + } else if (opnd_size_prefix) { + dp = (instable_t *)&dis_op660FC7[off]; + opnd_size_prefix = 0; + if (opnd_size == SIZE16) + opnd_size = SIZE32; + } + break; + + + case MMOSH: + /* + * As with the "normal" SIMD instructions, the MMX + * shuffle instructions are overloaded. These + * instructions, however, are special in that they use + * an extra byte, and thus an extra table. As of this + * writing, they only use the opnd_size prefix. + */ + + /* + * Calculate our offset in dis_op0F7123 + */ + if ((uintptr_t)dp - (uintptr_t)dis_op0F7123 > + sizeof (dis_op0F7123)) + goto error; + + if (opnd_size_prefix) { + off = ((uintptr_t)dp - (uintptr_t)dis_op0F7123) / + sizeof (instable_t); + dp = (instable_t *)&dis_opSIMD7123[off]; + opnd_size_prefix = 0; + if (opnd_size == SIZE16) + opnd_size = SIZE32; + } + break; + case MRw: + if (rep_prefix) { + if (rep_prefix == 0xf3) { + + /* + * Calculate our offset in dis_op0F + */ + if ((uintptr_t)dp - (uintptr_t)dis_op0F + > sizeof (dis_op0F)) + goto error; + + off = ((uintptr_t)dp - (uintptr_t)dis_op0F) / + sizeof (instable_t); + + dp = (instable_t *)&dis_opSIMDrepz[off]; + rep_prefix = 0; + } else { + goto error; + } + } + break; + } + + /* + * In 64 bit mode, some opcodes automatically use opnd_size == SIZE64. + */ + if (cpu_mode == SIZE64) + if (dp->it_always64 || (opnd_size == SIZE32 && dp->it_stackop)) + opnd_size = SIZE64; + +#ifdef DIS_TEXT + /* + * At this point most instructions can format the opcode mnemonic + * including the prefixes. + */ + if (lock_prefix) + (void) strlcat(x->d86_mnem, "lock ", OPLEN); + + if (rep_prefix == 0xf2) + (void) strlcat(x->d86_mnem, "repnz ", OPLEN); + else if (rep_prefix == 0xf3) + (void) strlcat(x->d86_mnem, "repz ", OPLEN); + + if (cpu_mode == SIZE64 && addr_size_prefix) + (void) strlcat(x->d86_mnem, "addr32 ", OPLEN); + + if (dp->it_adrmode != CBW && + dp->it_adrmode != CWD && + dp->it_adrmode != XMMSFNC) { + if (strcmp(dp->it_name, "INVALID") == 0) + goto error; + (void) strlcat(x->d86_mnem, dp->it_name, OPLEN); + if (dp->it_avxsuf && dp->it_suffix) { + (void) strlcat(x->d86_mnem, vex_W != 0 ? "q" : "d", + OPLEN); + } else if (dp->it_suffix) { + char *types[] = {"", "w", "l", "q"}; + if (opcode_bytes == 2 && opcode4 == 4) { + /* It's a cmovx.yy. Replace the suffix x */ + for (i = 5; i < OPLEN; i++) { + if (x->d86_mnem[i] == '.') + break; + } + x->d86_mnem[i - 1] = *types[opnd_size]; + } else if ((opnd_size == 2) && (opcode_bytes == 3) && + ((opcode6 == 1 && opcode7 == 6) || + (opcode6 == 2 && opcode7 == 2))) { + /* + * To handle PINSRD and PEXTRD + */ + (void) strlcat(x->d86_mnem, "d", OPLEN); + } else { + (void) strlcat(x->d86_mnem, types[opnd_size], + OPLEN); + } + } + } +#endif + + /* + * Process operands based on the addressing modes. + */ + x->d86_mode = cpu_mode; + /* + * In vex mode the rex_prefix has no meaning + */ + if (!vex_prefix) + x->d86_rex_prefix = rex_prefix; + x->d86_opnd_size = opnd_size; + x->d86_addr_size = addr_size; + vbit = 0; /* initialize for mem/reg -> reg */ + switch (dp->it_adrmode) { + /* + * amd64 instruction to sign extend 32 bit reg/mem operands + * into 64 bit register values + */ + case MOVSXZ: +#ifdef DIS_TEXT + if (rex_prefix == 0) + (void) strncpy(x->d86_mnem, "movzld", OPLEN); +#endif + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + x->d86_opnd_size = SIZE64; + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1); + x->d86_opnd_size = opnd_size = SIZE32; + wbit = LONG_OPND; + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + + /* + * movsbl movsbw movsbq (0x0FBE) or movswl movswq (0x0FBF) + * movzbl movzbw movzbq (0x0FB6) or movzwl movzwq (0x0FB7) + * wbit lives in 2nd byte, note that operands + * are different sized + */ + case MOVZ: + if (rex_prefix & REX_W) { + /* target register size = 64 bit */ + x->d86_mnem[5] = 'q'; + } + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1); + x->d86_opnd_size = opnd_size = SIZE16; + wbit = WBIT(opcode5); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + case CRC32: + opnd_size = SIZE32; + if (rex_prefix & REX_W) + opnd_size = SIZE64; + x->d86_opnd_size = opnd_size; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1); + wbit = WBIT(opcode7); + if (opnd_size_prefix) + x->d86_opnd_size = opnd_size = SIZE16; + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + case MOVBE: + opnd_size = SIZE32; + if (rex_prefix & REX_W) + opnd_size = SIZE64; + x->d86_opnd_size = opnd_size; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + wbit = WBIT(opcode7); + if (opnd_size_prefix) + x->d86_opnd_size = opnd_size = SIZE16; + if (wbit) { + /* reg -> mem */ + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 0); + dtrace_get_operand(x, mode, r_m, wbit, 1); + } else { + /* mem -> reg */ + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + } + break; + + /* + * imul instruction, with either 8-bit or longer immediate + * opcode 0x6B for byte, sign-extended displacement, 0x69 for word(s) + */ + case IMUL: + wbit = LONG_OPND; + THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, LONG_OPND, + OPSIZE(opnd_size, opcode2 == 0x9), 1); + break; + + /* memory or register operand to register, with 'w' bit */ + case MRw: + case ADX: + wbit = WBIT(opcode2); + STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0); + break; + + /* register to memory or register operand, with 'w' bit */ + /* arpl happens to fit here also because it is odd */ + case RMw: + if (opcode_bytes == 2) + wbit = WBIT(opcode5); + else + wbit = WBIT(opcode2); + STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1); + break; + + /* xaddb instruction */ + case XADDB: + wbit = 0; + STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1); + break; + + /* MMX register to memory or register operand */ + case MMS: + case MMOS: +#ifdef DIS_TEXT + wbit = strcmp(dp->it_name, "movd") ? MM_OPND : LONG_OPND; +#else + wbit = LONG_OPND; +#endif + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, MM_OPND, 1); + break; + + /* MMX register to memory */ + case MMOMS: + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode == REG_ONLY) + goto error; + wbit = MM_OPND; + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, MM_OPND, 1); + break; + + /* Double shift. Has immediate operand specifying the shift. */ + case DSHIFT: + wbit = LONG_OPND; + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 2); + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1); + dtrace_imm_opnd(x, wbit, 1, 0); + break; + + /* + * Double shift. With no immediate operand, specifies using %cl. + */ + case DSHIFTcl: + wbit = LONG_OPND; + STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1); + break; + + /* immediate to memory or register operand */ + case IMlw: + wbit = WBIT(opcode2); + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 1); + /* + * Have long immediate for opcode 0x81, but not 0x80 nor 0x83 + */ + dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, opcode2 == 1), 0); + break; + + /* immediate to memory or register operand with the */ + /* 'w' bit present */ + case IMw: + wbit = WBIT(opcode2); + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 1); + dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, wbit), 0); + break; + + /* immediate to register with register in low 3 bits */ + /* of op code */ + case IR: + /* w-bit here (with regs) is bit 3 */ + wbit = opcode2 >>3 & 0x1; + reg = REGNO(opcode2); + dtrace_rex_adjust(rex_prefix, mode, ®, NULL); + mode = REG_ONLY; + r_m = reg; + dtrace_get_operand(x, mode, r_m, wbit, 1); + dtrace_imm_opnd(x, wbit, OPSIZE64(opnd_size, wbit), 0); + break; + + /* MMX immediate shift of register */ + case MMSH: + case MMOSH: + wbit = MM_OPND; + goto mm_shift; /* in next case */ + + /* SIMD immediate shift of register */ + case XMMSH: + wbit = XMM_OPND; +mm_shift: + reg = REGNO(opcode7); + dtrace_rex_adjust(rex_prefix, mode, ®, NULL); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + dtrace_imm_opnd(x, wbit, 1, 0); + NOMEM; + break; + + /* accumulator to memory operand */ + case AO: + vbit = 1; + /*FALLTHROUGH*/ + + /* memory operand to accumulator */ + case OA: + wbit = WBIT(opcode2); + dtrace_get_operand(x, REG_ONLY, EAX_REGNO, wbit, 1 - vbit); + dtrace_imm_opnd(x, wbit, OPSIZE64(addr_size, LONG_OPND), vbit); +#ifdef DIS_TEXT + x->d86_opnd[vbit].d86_mode = MODE_OFFSET; +#endif + break; + + + /* segment register to memory or register operand */ + case SM: + vbit = 1; + /*FALLTHROUGH*/ + + /* memory or register operand to segment register */ + case MS: + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, LONG_OPND, vbit); + dtrace_get_operand(x, REG_ONLY, reg, SEG_OPND, 1 - vbit); + break; + + /* + * rotate or shift instructions, which may shift by 1 or + * consult the cl register, depending on the 'v' bit + */ + case Mv: + vbit = VBIT(opcode2); + wbit = WBIT(opcode2); + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 1); +#ifdef DIS_TEXT + if (vbit) { + (void) strlcat(x->d86_opnd[0].d86_opnd, "%cl", OPLEN); + } else { + x->d86_opnd[0].d86_mode = MODE_SIGNED; + x->d86_opnd[0].d86_value_size = 1; + x->d86_opnd[0].d86_value = 1; + } +#endif + break; + /* + * immediate rotate or shift instructions + */ + case MvI: + wbit = WBIT(opcode2); +normal_imm_mem: + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 1); + dtrace_imm_opnd(x, wbit, 1, 0); + break; + + /* bit test instructions */ + case MIb: + wbit = LONG_OPND; + goto normal_imm_mem; + + /* single memory or register operand with 'w' bit present */ + case Mw: + wbit = WBIT(opcode2); +just_mem: + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + + case SWAPGS_RDTSCP: + if (cpu_mode == SIZE64 && mode == 3 && r_m == 0) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "swapgs", OPLEN); +#endif + NOMEM; + break; + } else if (mode == 3 && r_m == 1) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "rdtscp", OPLEN); +#endif + NOMEM; + break; + } + + /*FALLTHROUGH*/ + + /* prefetch instruction - memory operand, but no memory acess */ + case PREF: + NOMEM; + /*FALLTHROUGH*/ + + /* single memory or register operand */ + case M: + case MG9: + wbit = LONG_OPND; + goto just_mem; + + /* single memory or register byte operand */ + case Mb: + wbit = BYTE_OPND; + goto just_mem; + + case VMx: + if (mode == 3) { +#ifdef DIS_TEXT + char *vminstr; + + switch (r_m) { + case 1: + vminstr = "vmcall"; + break; + case 2: + vminstr = "vmlaunch"; + break; + case 3: + vminstr = "vmresume"; + break; + case 4: + vminstr = "vmxoff"; + break; + default: + goto error; + } + + (void) strncpy(x->d86_mnem, vminstr, OPLEN); +#else + if (r_m < 1 || r_m > 4) + goto error; +#endif + + NOMEM; + break; + } + /*FALLTHROUGH*/ + case SVM: + if (mode == 3) { +#ifdef DIS_TEXT + char *vinstr; + + switch (r_m) { + case 0: + vinstr = "vmrun"; + break; + case 1: + vinstr = "vmmcall"; + break; + case 2: + vinstr = "vmload"; + break; + case 3: + vinstr = "vmsave"; + break; + case 4: + vinstr = "stgi"; + break; + case 5: + vinstr = "clgi"; + break; + case 6: + vinstr = "skinit"; + break; + case 7: + vinstr = "invlpga"; + break; + } + + (void) strncpy(x->d86_mnem, vinstr, OPLEN); +#endif + NOMEM; + break; + } + /*FALLTHROUGH*/ + case MONITOR_MWAIT: + if (mode == 3) { + if (r_m == 0) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "monitor", OPLEN); +#endif + NOMEM; + break; + } else if (r_m == 1) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "mwait", OPLEN); +#endif + NOMEM; + break; + } else if (r_m == 2) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "clac", OPLEN); +#endif + NOMEM; + break; + } else if (r_m == 3) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "stac", OPLEN); +#endif + NOMEM; + break; + } else { + goto error; + } + } + /*FALLTHROUGH*/ + case XGETBV_XSETBV: + if (mode == 3) { + if (r_m == 0) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "xgetbv", OPLEN); +#endif + NOMEM; + break; + } else if (r_m == 1) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "xsetbv", OPLEN); +#endif + NOMEM; + break; + } else { + goto error; + } + + } + /*FALLTHROUGH*/ + case MO: + /* Similar to M, but only memory (no direct registers) */ + wbit = LONG_OPND; + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode == 3) + goto error; + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + + /* move special register to register or reverse if vbit */ + case SREG: + switch (opcode5) { + + case 2: + vbit = 1; + /*FALLTHROUGH*/ + case 0: + wbit = CONTROL_OPND; + break; + + case 3: + vbit = 1; + /*FALLTHROUGH*/ + case 1: + wbit = DEBUG_OPND; + break; + + case 6: + vbit = 1; + /*FALLTHROUGH*/ + case 4: + wbit = TEST_OPND; + break; + + } + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, REG_ONLY, reg, wbit, vbit); + dtrace_get_operand(x, REG_ONLY, r_m, LONG_OPND, 1 - vbit); + NOMEM; + break; + + /* + * single register operand with register in the low 3 + * bits of op code + */ + case R: + if (opcode_bytes == 2) + reg = REGNO(opcode5); + else + reg = REGNO(opcode2); + dtrace_rex_adjust(rex_prefix, mode, ®, NULL); + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 0); + NOMEM; + break; + + /* + * register to accumulator with register in the low 3 + * bits of op code, xchg instructions + */ + case RA: + NOMEM; + reg = REGNO(opcode2); + dtrace_rex_adjust(rex_prefix, mode, ®, NULL); + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 0); + dtrace_get_operand(x, REG_ONLY, EAX_REGNO, LONG_OPND, 1); + break; + + /* + * single segment register operand, with register in + * bits 3-4 of op code byte + */ + case SEG: + NOMEM; + reg = (x->d86_bytes[x->d86_len - 1] >> 3) & 0x3; + dtrace_get_operand(x, REG_ONLY, reg, SEG_OPND, 0); + break; + + /* + * single segment register operand, with register in + * bits 3-5 of op code + */ + case LSEG: + NOMEM; + /* long seg reg from opcode */ + reg = (x->d86_bytes[x->d86_len - 1] >> 3) & 0x7; + dtrace_get_operand(x, REG_ONLY, reg, SEG_OPND, 0); + break; + + /* memory or register operand to register */ + case MR: + if (vex_prefetch) + x->d86_got_modrm = 1; + wbit = LONG_OPND; + STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0); + break; + + case RM: + case RM_66r: + wbit = LONG_OPND; + STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1); + break; + + /* MMX/SIMD-Int memory or mm reg to mm reg */ + case MM: + case MMO: +#ifdef DIS_TEXT + wbit = strcmp(dp->it_name, "movd") ? MM_OPND : LONG_OPND; +#else + wbit = LONG_OPND; +#endif + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, MM_OPND, 0); + break; + + case MMOIMPL: +#ifdef DIS_TEXT + wbit = strcmp(dp->it_name, "movd") ? MM_OPND : LONG_OPND; +#else + wbit = LONG_OPND; +#endif + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode != REG_ONLY) + goto error; + + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 0); + dtrace_get_operand(x, REG_ONLY, reg, MM_OPND, 1); + mode = 0; /* change for memory access size... */ + break; + + /* MMX/SIMD-Int and SIMD-FP predicated mm reg to r32 */ + case MMO3P: + wbit = MM_OPND; + goto xmm3p; + case XMM3P: + wbit = XMM_OPND; +xmm3p: + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode != REG_ONLY) + goto error; + + THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, LONG_OPND, 1, + 1); + NOMEM; + break; + + case XMM3PM_66r: + THREEOPERAND(x, mode, reg, r_m, rex_prefix, LONG_OPND, XMM_OPND, + 1, 0); + break; + + /* MMX/SIMD-Int predicated r32/mem to mm reg */ + case MMOPRM: + wbit = LONG_OPND; + w2 = MM_OPND; + goto xmmprm; + case XMMPRM: + case XMMPRM_66r: + wbit = LONG_OPND; + w2 = XMM_OPND; +xmmprm: + THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, w2, 1, 1); + break; + + /* MMX/SIMD-Int predicated mm/mem to mm reg */ + case MMOPM: + case MMOPM_66o: + wbit = w2 = MM_OPND; + goto xmmprm; + + /* MMX/SIMD-Int mm reg to r32 */ + case MMOM3: + NOMEM; + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode != REG_ONLY) + goto error; + wbit = MM_OPND; + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, LONG_OPND, 0); + break; + + /* SIMD memory or xmm reg operand to xmm reg */ + case XMM: + case XMM_66o: + case XMM_66r: + case XMMO: + case XMMXIMPL: + wbit = XMM_OPND; + STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0); + + if (dp->it_adrmode == XMMXIMPL && mode != REG_ONLY) + goto error; + +#ifdef DIS_TEXT + /* + * movlps and movhlps share opcodes. They differ in the + * addressing modes allowed for their operands. + * movhps and movlhps behave similarly. + */ + if (mode == REG_ONLY) { + if (strcmp(dp->it_name, "movlps") == 0) + (void) strncpy(x->d86_mnem, "movhlps", OPLEN); + else if (strcmp(dp->it_name, "movhps") == 0) + (void) strncpy(x->d86_mnem, "movlhps", OPLEN); + } +#endif + if (dp->it_adrmode == XMMXIMPL) + mode = 0; /* change for memory access size... */ + break; + + /* SIMD xmm reg to memory or xmm reg */ + case XMMS: + case XMMOS: + case XMMMS: + case XMMOMS: + dtrace_get_modrm(x, &mode, ®, &r_m); +#ifdef DIS_TEXT + if ((strcmp(dp->it_name, "movlps") == 0 || + strcmp(dp->it_name, "movhps") == 0 || + strcmp(dp->it_name, "movntps") == 0) && + mode == REG_ONLY) + goto error; +#endif + wbit = XMM_OPND; + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 1); + break; + + /* SIMD memory to xmm reg */ + case XMMM: + case XMMM_66r: + case XMMOM: + wbit = XMM_OPND; + dtrace_get_modrm(x, &mode, ®, &r_m); +#ifdef DIS_TEXT + if (mode == REG_ONLY) { + if (strcmp(dp->it_name, "movhps") == 0) + (void) strncpy(x->d86_mnem, "movlhps", OPLEN); + else + goto error; + } +#endif + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 0); + break; + + /* SIMD memory or r32 to xmm reg */ + case XMM3MX: + wbit = LONG_OPND; + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 0); + break; + + case XMM3MXS: + wbit = LONG_OPND; + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 1); + break; + + /* SIMD memory or mm reg to xmm reg */ + case XMMOMX: + /* SIMD mm to xmm */ + case XMMMX: + wbit = MM_OPND; + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 0); + break; + + /* SIMD memory or xmm reg to mm reg */ + case XMMXMM: + case XMMOXMM: + case XMMXM: + wbit = XMM_OPND; + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, MM_OPND, 0); + break; + + + /* SIMD memory or xmm reg to r32 */ + case XMMXM3: + wbit = XMM_OPND; + MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, LONG_OPND, 0); + break; + + /* SIMD xmm to r32 */ + case XMMX3: + case XMMOX3: + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode != REG_ONLY) + goto error; + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, XMM_OPND, 0); + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1); + NOMEM; + break; + + /* SIMD predicated memory or xmm reg with/to xmm reg */ + case XMMP: + case XMMP_66r: + case XMMP_66o: + case XMMOPM: + wbit = XMM_OPND; + THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 1, + 1); + +#ifdef DIS_TEXT + /* + * cmpps and cmpss vary their instruction name based + * on the value of imm8. Other XMMP instructions, + * such as shufps, require explicit specification of + * the predicate. + */ + if (dp->it_name[0] == 'c' && + dp->it_name[1] == 'm' && + dp->it_name[2] == 'p' && + strlen(dp->it_name) == 5) { + uchar_t pred = x->d86_opnd[0].d86_value & 0xff; + + if (pred >= (sizeof (dis_PREDSUFFIX) / sizeof (char *))) + goto error; + + (void) strncpy(x->d86_mnem, "cmp", OPLEN); + (void) strlcat(x->d86_mnem, dis_PREDSUFFIX[pred], + OPLEN); + (void) strlcat(x->d86_mnem, + dp->it_name + strlen(dp->it_name) - 2, + OPLEN); + x->d86_opnd[0] = x->d86_opnd[1]; + x->d86_opnd[1] = x->d86_opnd[2]; + x->d86_numopnds = 2; + } +#endif + break; + + case XMMX2I: + FOUROPERAND(x, mode, reg, r_m, rex_prefix, XMM_OPND, XMM_OPND, + 1); + NOMEM; + break; + + case XMM2I: + ONEOPERAND_TWOIMM(x, mode, reg, r_m, rex_prefix, XMM_OPND, 1); + NOMEM; + break; + + /* immediate operand to accumulator */ + case IA: + wbit = WBIT(opcode2); + dtrace_get_operand(x, REG_ONLY, EAX_REGNO, wbit, 1); + dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, wbit), 0); + NOMEM; + break; + + /* memory or register operand to accumulator */ + case MA: + wbit = WBIT(opcode2); + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + + /* si register to di register used to reference memory */ + case SD: +#ifdef DIS_TEXT + dtrace_check_override(x, 0); + x->d86_numopnds = 2; + if (addr_size == SIZE64) { + (void) strlcat(x->d86_opnd[0].d86_opnd, "(%rsi)", + OPLEN); + (void) strlcat(x->d86_opnd[1].d86_opnd, "(%rdi)", + OPLEN); + } else if (addr_size == SIZE32) { + (void) strlcat(x->d86_opnd[0].d86_opnd, "(%esi)", + OPLEN); + (void) strlcat(x->d86_opnd[1].d86_opnd, "(%edi)", + OPLEN); + } else { + (void) strlcat(x->d86_opnd[0].d86_opnd, "(%si)", + OPLEN); + (void) strlcat(x->d86_opnd[1].d86_opnd, "(%di)", + OPLEN); + } +#endif + wbit = LONG_OPND; + break; + + /* accumulator to di register */ + case AD: + wbit = WBIT(opcode2); +#ifdef DIS_TEXT + dtrace_check_override(x, 1); + x->d86_numopnds = 2; + dtrace_get_operand(x, REG_ONLY, EAX_REGNO, wbit, 0); + if (addr_size == SIZE64) + (void) strlcat(x->d86_opnd[1].d86_opnd, "(%rdi)", + OPLEN); + else if (addr_size == SIZE32) + (void) strlcat(x->d86_opnd[1].d86_opnd, "(%edi)", + OPLEN); + else + (void) strlcat(x->d86_opnd[1].d86_opnd, "(%di)", + OPLEN); +#endif + break; + + /* si register to accumulator */ + case SA: + wbit = WBIT(opcode2); +#ifdef DIS_TEXT + dtrace_check_override(x, 0); + x->d86_numopnds = 2; + if (addr_size == SIZE64) + (void) strlcat(x->d86_opnd[0].d86_opnd, "(%rsi)", + OPLEN); + else if (addr_size == SIZE32) + (void) strlcat(x->d86_opnd[0].d86_opnd, "(%esi)", + OPLEN); + else + (void) strlcat(x->d86_opnd[0].d86_opnd, "(%si)", + OPLEN); + dtrace_get_operand(x, REG_ONLY, EAX_REGNO, wbit, 1); +#endif + break; + + /* + * single operand, a 16/32 bit displacement + */ + case D: + wbit = LONG_OPND; + dtrace_disp_opnd(x, wbit, OPSIZE(opnd_size, LONG_OPND), 0); + NOMEM; + break; + + /* jmp/call indirect to memory or register operand */ + case INM: +#ifdef DIS_TEXT + (void) strlcat(x->d86_opnd[0].d86_prefix, "*", OPLEN); +#endif + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, LONG_OPND, 0); + wbit = LONG_OPND; + break; + + /* + * for long jumps and long calls -- a new code segment + * register and an offset in IP -- stored in object + * code in reverse order. Note - not valid in amd64 + */ + case SO: + dtrace_check_override(x, 1); + wbit = LONG_OPND; + dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, LONG_OPND), 1); +#ifdef DIS_TEXT + x->d86_opnd[1].d86_mode = MODE_SIGNED; +#endif + /* will now get segment operand */ + dtrace_imm_opnd(x, wbit, 2, 0); + break; + + /* + * jmp/call. single operand, 8 bit displacement. + * added to current EIP in 'compofff' + */ + case BD: + dtrace_disp_opnd(x, BYTE_OPND, 1, 0); + NOMEM; + break; + + /* single 32/16 bit immediate operand */ + case I: + wbit = LONG_OPND; + dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, LONG_OPND), 0); + break; + + /* single 8 bit immediate operand */ + case Ib: + wbit = LONG_OPND; + dtrace_imm_opnd(x, wbit, 1, 0); + break; + + case ENTER: + wbit = LONG_OPND; + dtrace_imm_opnd(x, wbit, 2, 0); + dtrace_imm_opnd(x, wbit, 1, 1); + switch (opnd_size) { + case SIZE64: + x->d86_memsize = (x->d86_opnd[1].d86_value + 1) * 8; + break; + case SIZE32: + x->d86_memsize = (x->d86_opnd[1].d86_value + 1) * 4; + break; + case SIZE16: + x->d86_memsize = (x->d86_opnd[1].d86_value + 1) * 2; + break; + } + + break; + + /* 16-bit immediate operand */ + case RET: + wbit = LONG_OPND; + dtrace_imm_opnd(x, wbit, 2, 0); + break; + + /* single 8 bit port operand */ + case P: + dtrace_check_override(x, 0); + dtrace_imm_opnd(x, BYTE_OPND, 1, 0); + NOMEM; + break; + + /* single operand, dx register (variable port instruction) */ + case V: + x->d86_numopnds = 1; + dtrace_check_override(x, 0); +#ifdef DIS_TEXT + (void) strlcat(x->d86_opnd[0].d86_opnd, "(%dx)", OPLEN); +#endif + NOMEM; + break; + + /* + * The int instruction, which has two forms: + * int 3 (breakpoint) or + * int n, where n is indicated in the subsequent + * byte (format Ib). The int 3 instruction (opcode 0xCC), + * where, although the 3 looks like an operand, + * it is implied by the opcode. It must be converted + * to the correct base and output. + */ + case INT3: +#ifdef DIS_TEXT + x->d86_numopnds = 1; + x->d86_opnd[0].d86_mode = MODE_SIGNED; + x->d86_opnd[0].d86_value_size = 1; + x->d86_opnd[0].d86_value = 3; +#endif + NOMEM; + break; + + /* single 8 bit immediate operand */ + case INTx: + dtrace_imm_opnd(x, BYTE_OPND, 1, 0); + NOMEM; + break; + + /* an unused byte must be discarded */ + case U: + if (x->d86_get_byte(x->d86_data) < 0) + goto error; + x->d86_len++; + NOMEM; + break; + + case CBW: +#ifdef DIS_TEXT + if (opnd_size == SIZE16) + (void) strlcat(x->d86_mnem, "cbtw", OPLEN); + else if (opnd_size == SIZE32) + (void) strlcat(x->d86_mnem, "cwtl", OPLEN); + else + (void) strlcat(x->d86_mnem, "cltq", OPLEN); +#endif + wbit = LONG_OPND; + NOMEM; + break; + + case CWD: +#ifdef DIS_TEXT + if (opnd_size == SIZE16) + (void) strlcat(x->d86_mnem, "cwtd", OPLEN); + else if (opnd_size == SIZE32) + (void) strlcat(x->d86_mnem, "cltd", OPLEN); + else + (void) strlcat(x->d86_mnem, "cqtd", OPLEN); +#endif + wbit = LONG_OPND; + NOMEM; + break; + + case XMMSFNC: + /* + * sfence is sfence if mode is REG_ONLY. If mode isn't + * REG_ONLY, mnemonic should be 'clflush'. + */ + dtrace_get_modrm(x, &mode, ®, &r_m); + + /* sfence doesn't take operands */ +#ifdef DIS_TEXT + if (mode == REG_ONLY) { + (void) strlcat(x->d86_mnem, "sfence", OPLEN); + } else { + (void) strlcat(x->d86_mnem, "clflush", OPLEN); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, BYTE_OPND, 0); + NOMEM; + } +#else + if (mode != REG_ONLY) { + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, LONG_OPND, 0); + NOMEM; + } +#endif + break; + + /* + * no disassembly, the mnemonic was all there was so go on + */ + case NORM: + if (dp->it_invalid32 && cpu_mode != SIZE64) + goto error; + NOMEM; + /*FALLTHROUGH*/ + case IMPLMEM: + break; + + case XMMFENCE: + /* + * XRSTOR and LFENCE share the same opcode but differ in mode + */ + dtrace_get_modrm(x, &mode, ®, &r_m); + + if (mode == REG_ONLY) { + /* + * Only the following exact byte sequences are allowed: + * + * 0f ae e8 lfence + * 0f ae f0 mfence + */ + if ((uint8_t)x->d86_bytes[x->d86_len - 1] != 0xe8 && + (uint8_t)x->d86_bytes[x->d86_len - 1] != 0xf0) + goto error; + } else { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "xrstor", OPLEN); +#endif + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, BYTE_OPND, 0); + } + break; + + /* float reg */ + case F: +#ifdef DIS_TEXT + x->d86_numopnds = 1; + (void) strlcat(x->d86_opnd[0].d86_opnd, "%st(X)", OPLEN); + x->d86_opnd[0].d86_opnd[4] = r_m + '0'; +#endif + NOMEM; + break; + + /* float reg to float reg, with ret bit present */ + case FF: + vbit = opcode2 >> 2 & 0x1; /* vbit = 1: st -> st(i) */ + /*FALLTHROUGH*/ + case FFC: /* case for vbit always = 0 */ +#ifdef DIS_TEXT + x->d86_numopnds = 2; + (void) strlcat(x->d86_opnd[1 - vbit].d86_opnd, "%st", OPLEN); + (void) strlcat(x->d86_opnd[vbit].d86_opnd, "%st(X)", OPLEN); + x->d86_opnd[vbit].d86_opnd[4] = r_m + '0'; +#endif + NOMEM; + break; + + /* AVX instructions */ + case VEX_MO: + /* op(ModR/M.r/m) */ + x->d86_numopnds = 1; + dtrace_get_modrm(x, &mode, ®, &r_m); +#ifdef DIS_TEXT + if ((dp == &dis_opAVX0F[0xA][0xE]) && (reg == 3)) + (void) strncpy(x->d86_mnem, "vstmxcsr", OPLEN); +#endif + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + case VEX_RMrX: + case FMA: + /* ModR/M.reg := op(VEX.vvvv, ModR/M.r/m) */ + x->d86_numopnds = 3; + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + /* + * In classic Intel fashion, the opcodes for all of the FMA + * instructions all have two possible mnemonics which vary by + * one letter, which is selected based on the value of the wbit. + * When wbit is one, they have the 'd' suffix and when 'wbit' is + * 0, they have the 's' suffix. Otherwise, the FMA instructions + * are all a standard VEX_RMrX. + */ +#ifdef DIS_TEXT + if (dp->it_adrmode == FMA) { + size_t len = strlen(dp->it_name); + (void) strncpy(x->d86_mnem, dp->it_name, OPLEN); + if (len + 1 < OPLEN) { + (void) strncpy(x->d86_mnem + len, + vex_W != 0 ? "d" : "s", OPLEN - len); + } + } +#endif + + if (mode != REG_ONLY) { + if ((dp == &dis_opAVXF20F[0x10]) || + (dp == &dis_opAVXF30F[0x10])) { + /* vmovsd <m64>, <xmm> */ + /* or vmovss <m64>, <xmm> */ + x->d86_numopnds = 2; + goto L_VEX_MX; + } + } + + dtrace_get_operand(x, REG_ONLY, reg, wbit, 2); + /* + * VEX prefix uses the 1's complement form to encode the + * XMM/YMM regs + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 1); + + if ((dp == &dis_opAVXF20F[0x2A]) || + (dp == &dis_opAVXF30F[0x2A])) { + /* + * vcvtsi2si </r,m>, <xmm>, <xmm> or vcvtsi2ss </r,m>, + * <xmm>, <xmm> + */ + wbit = LONG_OPND; + } +#ifdef DIS_TEXT + else if ((mode == REG_ONLY) && + (dp == &dis_opAVX0F[0x1][0x6])) { /* vmovlhps */ + (void) strncpy(x->d86_mnem, "vmovlhps", OPLEN); + } else if ((mode == REG_ONLY) && + (dp == &dis_opAVX0F[0x1][0x2])) { /* vmovhlps */ + (void) strncpy(x->d86_mnem, "vmovhlps", OPLEN); + } +#endif + dtrace_get_operand(x, mode, r_m, wbit, 0); + + break; + + case VEX_VRMrX: + /* ModR/M.reg := op(MODR/M.r/m, VEX.vvvv) */ + x->d86_numopnds = 3; + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, REG_ONLY, reg, wbit, 2); + /* + * VEX prefix uses the 1's complement form to encode the + * XMM/YMM regs + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 0); + + dtrace_get_operand(x, mode, r_m, wbit, 1); + break; + + case VEX_SbVM: + /* ModR/M.reg := op(MODR/M.r/m, VSIB, VEX.vvvv) */ + x->d86_numopnds = 3; + x->d86_vsib = 1; + + /* + * All instructions that use VSIB are currently a mess. See the + * comment around the dis_gather_regs_t structure definition. + */ + + vreg = &dis_vgather[opcode2][vex_W][vex_L]; + +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, dp->it_name, OPLEN); + (void) strlcat(x->d86_mnem + strlen(dp->it_name), + vreg->dgr_suffix, OPLEN - strlen(dp->it_name)); +#endif + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, REG_ONLY, reg, vreg->dgr_arg2, 2); + /* + * VEX prefix uses the 1's complement form to encode the + * XMM/YMM regs + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), vreg->dgr_arg0, + 0); + dtrace_get_operand(x, mode, r_m, vreg->dgr_arg1, 1); + break; + + case VEX_RRX: + /* ModR/M.rm := op(VEX.vvvv, ModR/M.reg) */ + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + if (mode != REG_ONLY) { + if ((dp == &dis_opAVXF20F[0x11]) || + (dp == &dis_opAVXF30F[0x11])) { + /* vmovsd <xmm>, <m64> */ + /* or vmovss <xmm>, <m64> */ + x->d86_numopnds = 2; + goto L_VEX_RM; + } + } + + dtrace_get_operand(x, mode, r_m, wbit, 2); + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 1); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 0); + break; + + case VEX_RMRX: + /* ModR/M.reg := op(VEX.vvvv, ModR/M.r_m, imm8[7:4]) */ + x->d86_numopnds = 4; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 3); + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 2); + if (dp == &dis_opAVX660F3A[0x18]) { + /* vinsertf128 <imm8>, <xmm>, <ymm>, <ymm> */ + dtrace_get_operand(x, mode, r_m, XMM_OPND, 1); + } else if ((dp == &dis_opAVX660F3A[0x20]) || + (dp == & dis_opAVX660F[0xC4])) { + /* vpinsrb <imm8>, <reg/mm>, <xmm>, <xmm> */ + /* or vpinsrw <imm8>, <reg/mm>, <xmm>, <xmm> */ + dtrace_get_operand(x, mode, r_m, LONG_OPND, 1); + } else if (dp == &dis_opAVX660F3A[0x22]) { + /* vpinsrd/q <imm8>, <reg/mm>, <xmm>, <xmm> */ +#ifdef DIS_TEXT + if (vex_W) + x->d86_mnem[6] = 'q'; +#endif + dtrace_get_operand(x, mode, r_m, LONG_OPND, 1); + } else { + dtrace_get_operand(x, mode, r_m, wbit, 1); + } + + /* one byte immediate number */ + dtrace_imm_opnd(x, wbit, 1, 0); + + /* vblendvpd, vblendvps, vblendvb use the imm encode the regs */ + if ((dp == &dis_opAVX660F3A[0x4A]) || + (dp == &dis_opAVX660F3A[0x4B]) || + (dp == &dis_opAVX660F3A[0x4C])) { +#ifdef DIS_TEXT + int regnum = (x->d86_opnd[0].d86_value & 0xF0) >> 4; +#endif + x->d86_opnd[0].d86_mode = MODE_NONE; +#ifdef DIS_TEXT + if (vex_L) + (void) strncpy(x->d86_opnd[0].d86_opnd, + dis_YMMREG[regnum], OPLEN); + else + (void) strncpy(x->d86_opnd[0].d86_opnd, + dis_XMMREG[regnum], OPLEN); +#endif + } + break; + + case VEX_MX: + /* ModR/M.reg := op(ModR/M.rm) */ + x->d86_numopnds = 2; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); +L_VEX_MX: + + if ((dp == &dis_opAVXF20F[0xE6]) || + (dp == &dis_opAVX660F[0x5A]) || + (dp == &dis_opAVX660F[0xE6])) { + /* vcvtpd2dq <ymm>, <xmm> */ + /* or vcvtpd2ps <ymm>, <xmm> */ + /* or vcvttpd2dq <ymm>, <xmm> */ + dtrace_get_operand(x, REG_ONLY, reg, XMM_OPND, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + } else if ((dp == &dis_opAVXF30F[0xE6]) || + (dp == &dis_opAVX0F[0x5][0xA]) || + (dp == &dis_opAVX660F38[0x13]) || + (dp == &dis_opAVX660F38[0x18]) || + (dp == &dis_opAVX660F38[0x19]) || + (dp == &dis_opAVX660F38[0x58]) || + (dp == &dis_opAVX660F38[0x78]) || + (dp == &dis_opAVX660F38[0x79]) || + (dp == &dis_opAVX660F38[0x59])) { + /* vcvtdq2pd <xmm>, <ymm> */ + /* or vcvtps2pd <xmm>, <ymm> */ + /* or vcvtph2ps <xmm>, <ymm> */ + /* or vbroadcasts* <xmm>, <ymm> */ + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + dtrace_get_operand(x, mode, r_m, XMM_OPND, 0); + } else if (dp == &dis_opAVX660F[0x6E]) { + /* vmovd/q <reg/mem 32/64>, <xmm> */ +#ifdef DIS_TEXT + if (vex_W) + x->d86_mnem[4] = 'q'; +#endif + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + dtrace_get_operand(x, mode, r_m, LONG_OPND, 0); + } else { + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + } + + break; + + case VEX_MXI: + /* ModR/M.reg := op(ModR/M.rm, imm8) */ + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, REG_ONLY, reg, wbit, 2); + dtrace_get_operand(x, mode, r_m, wbit, 1); + + /* one byte immediate number */ + dtrace_imm_opnd(x, wbit, 1, 0); + break; + + case VEX_XXI: + /* VEX.vvvv := op(ModR/M.rm, imm8) */ + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, dis_AVXvgrp7[opcode2 - 1][reg], + OPLEN); +#endif + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 2); + dtrace_get_operand(x, REG_ONLY, r_m, wbit, 1); + + /* one byte immediate number */ + dtrace_imm_opnd(x, wbit, 1, 0); + break; + + case VEX_MR: + /* ModR/M.reg (reg32/64) := op(ModR/M.rm) */ + if (dp == &dis_opAVX660F[0xC5]) { + /* vpextrw <imm8>, <xmm>, <reg> */ + x->d86_numopnds = 2; + vbit = 2; + } else { + x->d86_numopnds = 2; + vbit = 1; + } + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, vbit); + dtrace_get_operand(x, mode, r_m, wbit, vbit - 1); + + if (vbit == 2) + dtrace_imm_opnd(x, wbit, 1, 0); + + break; + + case VEX_RRI: + /* implicit(eflags/r32) := op(ModR/M.reg, ModR/M.rm) */ + x->d86_numopnds = 2; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + + case VEX_RX: + /* ModR/M.rm := op(ModR/M.reg) */ + /* vextractf128 || vcvtps2ph */ + if (dp == &dis_opAVX660F3A[0x19] || + dp == &dis_opAVX660F3A[0x1d]) { + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, mode, r_m, XMM_OPND, 2); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + + /* one byte immediate number */ + dtrace_imm_opnd(x, wbit, 1, 0); + break; + } + + x->d86_numopnds = 2; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 1); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 0); + break; + + case VEX_RR: + /* ModR/M.rm := op(ModR/M.reg) */ + x->d86_numopnds = 2; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + if (dp == &dis_opAVX660F[0x7E]) { + /* vmovd/q <reg/mem 32/64>, <xmm> */ +#ifdef DIS_TEXT + if (vex_W) + x->d86_mnem[4] = 'q'; +#endif + dtrace_get_operand(x, mode, r_m, LONG_OPND, 1); + } else + dtrace_get_operand(x, mode, r_m, wbit, 1); + + dtrace_get_operand(x, REG_ONLY, reg, wbit, 0); + break; + + case VEX_RRi: + /* ModR/M.rm := op(ModR/M.reg, imm) */ + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + +#ifdef DIS_TEXT + if (dp == &dis_opAVX660F3A[0x16]) { + /* vpextrd/q <imm>, <xmm>, <reg/mem 32/64> */ + if (vex_W) + x->d86_mnem[6] = 'q'; + } +#endif + dtrace_get_operand(x, mode, r_m, LONG_OPND, 2); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + + /* one byte immediate number */ + dtrace_imm_opnd(x, wbit, 1, 0); + break; + case VEX_RIM: + /* ModR/M.rm := op(ModR/M.reg, imm) */ + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, mode, r_m, XMM_OPND, 2); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + /* one byte immediate number */ + dtrace_imm_opnd(x, wbit, 1, 0); + break; + + case VEX_RM: + /* ModR/M.rm := op(ModR/M.reg) */ + if (dp == &dis_opAVX660F3A[0x17]) { /* vextractps */ + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, mode, r_m, LONG_OPND, 2); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + /* one byte immediate number */ + dtrace_imm_opnd(x, wbit, 1, 0); + break; + } + x->d86_numopnds = 2; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); +L_VEX_RM: + vbit = 1; + dtrace_get_operand(x, mode, r_m, wbit, vbit); + dtrace_get_operand(x, REG_ONLY, reg, wbit, vbit - 1); + + break; + + case VEX_RRM: + /* ModR/M.rm := op(VEX.vvvv, ModR/M.reg) */ + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 2); + /* VEX use the 1's complement form encode the XMM/YMM regs */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 1); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 0); + break; + + case VEX_RMX: + /* ModR/M.reg := op(VEX.vvvv, ModR/M.rm) */ + x->d86_numopnds = 3; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 2); + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 1); + dtrace_get_operand(x, REG_ONLY, r_m, wbit, 0); + break; + + case VEX_NONE: +#ifdef DIS_TEXT + if (vex_L) + (void) strncpy(x->d86_mnem, "vzeroall", OPLEN); +#endif + break; + case BLS: { + + /* + * The BLS instructions are VEX instructions that are based on + * VEX.0F38.F3; however, they are considered special group 17 + * and like everything else, they use the bits in 3-5 of the + * MOD R/M to determine the sub instruction. Unlike many others + * like the VMX instructions, these are valid both for memory + * and register forms. + */ + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + switch (reg) { + case 1: +#ifdef DIS_TEXT + blsinstr = "blsr"; +#endif + break; + case 2: +#ifdef DIS_TEXT + blsinstr = "blsmsk"; +#endif + break; + case 3: +#ifdef DIS_TEXT + blsinstr = "blsi"; +#endif + break; + default: + goto error; + } + + x->d86_numopnds = 2; +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, blsinstr, OPLEN); +#endif + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + } + /* an invalid op code */ + case AM: + case DM: + case OVERRIDE: + case PREFIX: + case UNKNOWN: + NOMEM; + default: + goto error; + } /* end switch */ + if (x->d86_error) + goto error; + +done: +#ifdef DIS_MEM + /* + * compute the size of any memory accessed by the instruction + */ + if (x->d86_memsize != 0) { + return (0); + } else if (dp->it_stackop) { + switch (opnd_size) { + case SIZE16: + x->d86_memsize = 2; + break; + case SIZE32: + x->d86_memsize = 4; + break; + case SIZE64: + x->d86_memsize = 8; + break; + } + } else if (nomem || mode == REG_ONLY) { + x->d86_memsize = 0; + + } else if (dp->it_size != 0) { + /* + * In 64 bit mode descriptor table entries + * go up to 10 bytes and popf/pushf are always 8 bytes + */ + if (x->d86_mode == SIZE64 && dp->it_size == 6) + x->d86_memsize = 10; + else if (x->d86_mode == SIZE64 && opcode1 == 0x9 && + (opcode2 == 0xc || opcode2 == 0xd)) + x->d86_memsize = 8; + else + x->d86_memsize = dp->it_size; + + } else if (wbit == 0) { + x->d86_memsize = 1; + + } else if (wbit == LONG_OPND) { + if (opnd_size == SIZE64) + x->d86_memsize = 8; + else if (opnd_size == SIZE32) + x->d86_memsize = 4; + else + x->d86_memsize = 2; + + } else if (wbit == SEG_OPND) { + x->d86_memsize = 4; + + } else { + x->d86_memsize = 8; + } +#endif + return (0); + +error: +#ifdef DIS_TEXT + (void) strlcat(x->d86_mnem, "undef", OPLEN); +#endif + return (1); +} + +#ifdef DIS_TEXT + +/* + * Some instructions should have immediate operands printed + * as unsigned integers. We compare against this table. + */ +static char *unsigned_ops[] = { + "or", "and", "xor", "test", "in", "out", "lcall", "ljmp", + "rcr", "rcl", "ror", "rol", "shl", "shr", "sal", "psr", "psl", + 0 +}; + + +static int +isunsigned_op(char *opcode) +{ + char *where; + int i; + int is_unsigned = 0; + + /* + * Work back to start of last mnemonic, since we may have + * prefixes on some opcodes. + */ + where = opcode + strlen(opcode) - 1; + while (where > opcode && *where != ' ') + --where; + if (*where == ' ') + ++where; + + for (i = 0; unsigned_ops[i]; ++i) { + if (strncmp(where, unsigned_ops[i], + strlen(unsigned_ops[i]))) + continue; + is_unsigned = 1; + break; + } + return (is_unsigned); +} + +/* + * Print a numeric immediate into end of buf, maximum length buflen. + * The immediate may be an address or a displacement. Mask is set + * for address size. If the immediate is a "small negative", or + * if it's a negative displacement of any magnitude, print as -<absval>. + * Respect the "octal" flag. "Small negative" is defined as "in the + * interval [NEG_LIMIT, 0)". + * + * Also, "isunsigned_op()" instructions never print negatives. + * + * Return whether we decided to print a negative value or not. + */ + +#define NEG_LIMIT -255 +enum {IMM, DISP}; +enum {POS, TRY_NEG}; + +static int +print_imm(dis86_t *dis, uint64_t usv, uint64_t mask, char *buf, + size_t buflen, int disp, int try_neg) +{ + int curlen; + int64_t sv = (int64_t)usv; + int octal = dis->d86_flags & DIS_F_OCTAL; + + curlen = strlen(buf); + + if (try_neg == TRY_NEG && sv < 0 && + (disp || sv >= NEG_LIMIT) && + !isunsigned_op(dis->d86_mnem)) { + dis->d86_sprintf_func(buf + curlen, buflen - curlen, + octal ? "-0%llo" : "-0x%llx", (-sv) & mask); + return (1); + } else { + if (disp == DISP) + dis->d86_sprintf_func(buf + curlen, buflen - curlen, + octal ? "+0%llo" : "+0x%llx", usv & mask); + else + dis->d86_sprintf_func(buf + curlen, buflen - curlen, + octal ? "0%llo" : "0x%llx", usv & mask); + return (0); + + } +} + + +static int +log2(int size) +{ + switch (size) { + case 1: return (0); + case 2: return (1); + case 4: return (2); + case 8: return (3); + } + return (0); +} + +/* ARGSUSED */ +void +dtrace_disx86_str(dis86_t *dis, uint_t mode, uint64_t pc, char *buf, + size_t buflen) +{ + uint64_t reltgt = 0; + uint64_t tgt = 0; + int curlen; + int (*lookup)(void *, uint64_t, char *, size_t); + int i; + int64_t sv; + uint64_t usv, mask, save_mask, save_usv; + static uint64_t masks[] = + {0xffU, 0xffffU, 0xffffffffU, 0xffffffffffffffffULL}; + save_usv = 0; + + dis->d86_sprintf_func(buf, buflen, "%-6s ", dis->d86_mnem); + + /* + * For PC-relative jumps, the pc is really the next pc after executing + * this instruction, so increment it appropriately. + */ + pc += dis->d86_len; + + for (i = 0; i < dis->d86_numopnds; i++) { + d86opnd_t *op = &dis->d86_opnd[i]; + + if (i != 0) + (void) strlcat(buf, ",", buflen); + + (void) strlcat(buf, op->d86_prefix, buflen); + + /* + * sv is for the signed, possibly-truncated immediate or + * displacement; usv retains the original size and + * unsignedness for symbol lookup. + */ + + sv = usv = op->d86_value; + + /* + * About masks: for immediates that represent + * addresses, the appropriate display size is + * the effective address size of the instruction. + * This includes MODE_OFFSET, MODE_IPREL, and + * MODE_RIPREL. Immediates that are simply + * immediate values should display in the operand's + * size, however, since they don't represent addresses. + */ + + /* d86_addr_size is SIZEnn, which is log2(real size) */ + mask = masks[dis->d86_addr_size]; + + /* d86_value_size and d86_imm_bytes are in bytes */ + if (op->d86_mode == MODE_SIGNED || + op->d86_mode == MODE_IMPLIED) + mask = masks[log2(op->d86_value_size)]; + + switch (op->d86_mode) { + + case MODE_NONE: + + (void) strlcat(buf, op->d86_opnd, buflen); + break; + + case MODE_SIGNED: + case MODE_IMPLIED: + case MODE_OFFSET: + + tgt = usv; + + if (dis->d86_seg_prefix) + (void) strlcat(buf, dis->d86_seg_prefix, + buflen); + + if (op->d86_mode == MODE_SIGNED || + op->d86_mode == MODE_IMPLIED) { + (void) strlcat(buf, "$", buflen); + } + + if (print_imm(dis, usv, mask, buf, buflen, + IMM, TRY_NEG) && + (op->d86_mode == MODE_SIGNED || + op->d86_mode == MODE_IMPLIED)) { + + /* + * We printed a negative value for an + * immediate that wasn't a + * displacement. Note that fact so we can + * print the positive value as an + * annotation. + */ + + save_usv = usv; + save_mask = mask; + } + (void) strlcat(buf, op->d86_opnd, buflen); + + break; + + case MODE_IPREL: + case MODE_RIPREL: + + reltgt = pc + sv; + + switch (mode) { + case SIZE16: + reltgt = (uint16_t)reltgt; + break; + case SIZE32: + reltgt = (uint32_t)reltgt; + break; + } + + (void) print_imm(dis, usv, mask, buf, buflen, + DISP, TRY_NEG); + + if (op->d86_mode == MODE_RIPREL) + (void) strlcat(buf, "(%rip)", buflen); + break; + } + } + + /* + * The symbol lookups may result in false positives, + * particularly on object files, where small numbers may match + * the 0-relative non-relocated addresses of symbols. + */ + + lookup = dis->d86_sym_lookup; + if (tgt != 0) { + if ((dis->d86_flags & DIS_F_NOIMMSYM) == 0 && + lookup(dis->d86_data, tgt, NULL, 0) == 0) { + (void) strlcat(buf, "\t<", buflen); + curlen = strlen(buf); + lookup(dis->d86_data, tgt, buf + curlen, + buflen - curlen); + (void) strlcat(buf, ">", buflen); + } + + /* + * If we printed a negative immediate above, print the + * positive in case our heuristic was unhelpful + */ + if (save_usv) { + (void) strlcat(buf, "\t<", buflen); + (void) print_imm(dis, save_usv, save_mask, buf, buflen, + IMM, POS); + (void) strlcat(buf, ">", buflen); + } + } + + if (reltgt != 0) { + /* Print symbol or effective address for reltgt */ + + (void) strlcat(buf, "\t<", buflen); + curlen = strlen(buf); + lookup(dis->d86_data, reltgt, buf + curlen, + buflen - curlen); + (void) strlcat(buf, ">", buflen); + } +} + +#endif /* DIS_TEXT */ diff --git a/sys/cddl/dev/dtrace/x86/dis_tables.h b/sys/cddl/dev/dtrace/x86/dis_tables.h new file mode 100644 index 000000000000..87b006cf7845 --- /dev/null +++ b/sys/cddl/dev/dtrace/x86/dis_tables.h @@ -0,0 +1,111 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1988 AT&T */ +/* All Rights Reserved */ + +/* + * $FreeBSD$ + */ + +#ifndef _DIS_TABLES_H +#define _DIS_TABLES_H + +/* + * Constants and prototypes for the IA32 disassembler backend. See dis_tables.c + * for usage information and documentation. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <sys/types.h> +#include <sys/param.h> + +/* + * values for cpu mode + */ +#define SIZE16 1 +#define SIZE32 2 +#define SIZE64 3 + +#define OPLEN 256 +#define PFIXLEN 8 +#define NCPS 20 /* number of chars per symbol */ + +/* + * data structures that must be provided to dtrace_dis86() + */ +typedef struct d86opnd { + char d86_opnd[OPLEN]; /* symbolic rep of operand */ + char d86_prefix[PFIXLEN]; /* any prefix string or "" */ + uint_t d86_mode; /* mode for immediate */ + uint_t d86_value_size; /* size in bytes of d86_value */ + uint64_t d86_value; /* immediate value of opnd */ +} d86opnd_t; + +typedef struct dis86 { + uint_t d86_mode; + uint_t d86_error; + uint_t d86_len; /* instruction length */ + int d86_rmindex; /* index of modrm byte or -1 */ + uint_t d86_memsize; /* size of memory referenced */ + char d86_bytes[16]; /* bytes of instruction */ + char d86_mnem[OPLEN]; + uint_t d86_numopnds; + uint_t d86_rex_prefix; /* value of REX prefix if !0 */ + char *d86_seg_prefix; /* segment prefix, if any */ + uint_t d86_opnd_size; + uint_t d86_addr_size; + uint_t d86_got_modrm; + uint_t d86_vsib; /* Has a VSIB */ + struct d86opnd d86_opnd[4]; /* up to 4 operands */ + int (*d86_check_func)(void *); + int (*d86_get_byte)(void *); +#ifdef DIS_TEXT + int (*d86_sym_lookup)(void *, uint64_t, char *, size_t); + int (*d86_sprintf_func)(char *, size_t, const char *, ...); + int d86_flags; + uint_t d86_imm_bytes; +#endif + void *d86_data; +} dis86_t; + +extern int dtrace_disx86(dis86_t *x, uint_t cpu_mode); + +#define DIS_F_OCTAL 0x1 /* Print all numbers in octal */ +#define DIS_F_NOIMMSYM 0x2 /* Don't print symbols for immediates (.o) */ + +#ifdef DIS_TEXT +extern void dtrace_disx86_str(dis86_t *x, uint_t cpu_mode, uint64_t pc, + char *buf, size_t len); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _DIS_TABLES_H */ diff --git a/sys/cddl/dev/dtrace/x86/instr_size.c b/sys/cddl/dev/dtrace/x86/instr_size.c new file mode 100644 index 000000000000..6eea987244b1 --- /dev/null +++ b/sys/cddl/dev/dtrace/x86/instr_size.c @@ -0,0 +1,140 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1988 AT&T */ +/* All Rights Reserved */ + + +#ifdef illumos +#pragma ident "@(#)instr_size.c 1.14 05/07/08 SMI" +#endif + +#include <sys/types.h> +#include <sys/param.h> +#include <sys/proc.h> +#ifdef illumos +#include <sys/cmn_err.h> +#include <sys/archsystm.h> +#include <sys/copyops.h> +#include <vm/seg_enum.h> +#include <sys/privregs.h> +#else +#include <sys/cred.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> + +typedef u_int model_t; +#define DATAMODEL_NATIVE 0 +int dtrace_instr_size(uchar_t *); +int dtrace_instr_size_isa(uchar_t *, model_t, int *); +#endif + +#include <dis_tables.h> + +/* + * This subsystem (with the minor exception of the instr_size() function) is + * is called from DTrace probe context. This imposes several requirements on + * the implementation: + * + * 1. External subsystems and functions may not be referenced. The one current + * exception is for cmn_err, but only to signal the detection of table + * errors. Assuming the tables are correct, no combination of input is to + * trigger a cmn_err call. + * + * 2. These functions can't be allowed to be traced. To prevent this, + * all functions in the probe path (everything except instr_size()) must + * have names that begin with "dtrace_". + */ + +typedef enum dis_isize { + DIS_ISIZE_INSTR, + DIS_ISIZE_OPERAND +} dis_isize_t; + + +/* + * get a byte from instruction stream + */ +static int +dtrace_dis_get_byte(void *p) +{ + int ret; + uchar_t **instr = p; + + ret = **instr; + *instr += 1; + + return (ret); +} + +/* + * Returns either the size of a given instruction, in bytes, or the size of that + * instruction's memory access (if any), depending on the value of `which'. + * If a programming error in the tables is detected, the system will panic to + * ease diagnosis. Invalid instructions will not be flagged. They will appear + * to have an instruction size between 1 and the actual size, and will be + * reported as having no memory impact. + */ +/* ARGSUSED2 */ +static int +dtrace_dis_isize(uchar_t *instr, dis_isize_t which, model_t model, int *rmindex) +{ + int sz; + dis86_t x; + uint_t mode = SIZE32; + + mode = (model == DATAMODEL_LP64) ? SIZE64 : SIZE32; + + x.d86_data = (void **)&instr; + x.d86_get_byte = dtrace_dis_get_byte; + x.d86_check_func = NULL; + + if (dtrace_disx86(&x, mode) != 0) + return (-1); + + if (which == DIS_ISIZE_INSTR) + sz = x.d86_len; /* length of the instruction */ + else + sz = x.d86_memsize; /* length of memory operand */ + + if (rmindex != NULL) + *rmindex = x.d86_rmindex; + return (sz); +} + +int +dtrace_instr_size_isa(uchar_t *instr, model_t model, int *rmindex) +{ + return (dtrace_dis_isize(instr, DIS_ISIZE_INSTR, model, rmindex)); +} + +int +dtrace_instr_size(uchar_t *instr) +{ + return (dtrace_dis_isize(instr, DIS_ISIZE_INSTR, DATAMODEL_NATIVE, + NULL)); +} diff --git a/sys/cddl/dev/dtrace/x86/regset.h b/sys/cddl/dev/dtrace/x86/regset.h new file mode 100644 index 000000000000..ad12e26a5aec --- /dev/null +++ b/sys/cddl/dev/dtrace/x86/regset.h @@ -0,0 +1,178 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +#ifndef _REGSET_H +#define _REGSET_H + +/* + * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * The names and offsets defined here should be specified by the + * AMD64 ABI suppl. + * + * We make fsbase and gsbase part of the lwp context (since they're + * the only way to access the full 64-bit address range via the segment + * registers) and thus belong here too. However we treat them as + * read-only; if %fs or %gs are updated, the results of the descriptor + * table lookup that those updates implicitly cause will be reflected + * in the corresponding fsbase and/or gsbase values the next time the + * context can be inspected. However it is NOT possible to override + * the fsbase/gsbase settings via this interface. + * + * Direct modification of the base registers (thus overriding the + * descriptor table base address) can be achieved with _lwp_setprivate. + */ + +#define REG_GSBASE 27 +#define REG_FSBASE 26 +#ifdef illumos +#define REG_DS 25 +#define REG_ES 24 + +#define REG_GS 23 +#define REG_FS 22 +#define REG_SS 21 +#define REG_RSP 20 +#define REG_RFL 19 +#define REG_CS 18 +#define REG_RIP 17 +#define REG_ERR 16 +#define REG_TRAPNO 15 +#define REG_RAX 14 +#define REG_RCX 13 +#define REG_RDX 12 +#define REG_RBX 11 +#define REG_RBP 10 +#define REG_RSI 9 +#define REG_RDI 8 +#define REG_R8 7 +#define REG_R9 6 +#define REG_R10 5 +#define REG_R11 4 +#define REG_R12 3 +#define REG_R13 2 +#define REG_R14 1 +#define REG_R15 0 +#else /* !illumos */ +#define REG_SS 25 +#define REG_RSP 24 +#define REG_RFL 23 +#define REG_CS 22 +#define REG_RIP 21 +#define REG_DS 20 +#define REG_ES 19 +#define REG_ERR 18 +#define REG_GS 17 +#define REG_FS 16 +#define REG_TRAPNO 15 +#define REG_RAX 14 +#define REG_RCX 13 +#define REG_RDX 12 +#define REG_RBX 11 +#define REG_RBP 10 +#define REG_RSI 9 +#define REG_RDI 8 +#define REG_R8 7 +#define REG_R9 6 +#define REG_R10 5 +#define REG_R11 4 +#define REG_R12 3 +#define REG_R13 2 +#define REG_R14 1 +#define REG_R15 0 +#endif /* illumos */ + +/* + * The names and offsets defined here are specified by i386 ABI suppl. + */ + +#ifdef illumos +#define SS 18 /* only stored on a privilege transition */ +#define UESP 17 /* only stored on a privilege transition */ +#define EFL 16 +#define CS 15 +#define EIP 14 +#define ERR 13 +#define TRAPNO 12 +#define EAX 11 +#define ECX 10 +#define EDX 9 +#define EBX 8 +#define ESP 7 +#define EBP 6 +#define ESI 5 +#define EDI 4 +#define DS 3 +#define ES 2 +#define FS 1 +#define GS 0 +#else /* !illumos */ +#define GS 18 +#define SS 17 /* only stored on a privilege transition */ +#define UESP 16 /* only stored on a privilege transition */ +#define EFL 15 +#define CS 14 +#define EIP 13 +#define ERR 12 +#define TRAPNO 11 +#define EAX 10 +#define ECX 9 +#define EDX 8 +#define EBX 7 +#define ESP 6 +#define EBP 5 +#define ESI 4 +#define EDI 3 +#define DS 2 +#define ES 1 +#define FS 0 +#endif /* illumos */ + +#define REG_PC EIP +#define REG_FP EBP +#define REG_SP UESP +#define REG_PS EFL +#define REG_R0 EAX +#define REG_R1 EDX + +#ifdef __cplusplus +} +#endif + +#endif /* _REGSET_H */ |