Index: sys/amd64/include/vmm.h =================================================================== --- sys/amd64/include/vmm.h +++ sys/amd64/include/vmm.h @@ -95,6 +95,7 @@ VM_REG_GUEST_DR2, VM_REG_GUEST_DR3, VM_REG_GUEST_DR6, + VM_REG_GUEST_ENTRY_INST_LENGTH, VM_REG_LAST }; @@ -434,6 +435,7 @@ VM_CAP_PAUSE_EXIT, VM_CAP_UNRESTRICTED_GUEST, VM_CAP_ENABLE_INVPCID, + VM_CAP_BPT_EXIT, VM_CAP_MAX }; @@ -559,6 +561,7 @@ VM_EXITCODE_REQIDLE, VM_EXITCODE_DEBUG, VM_EXITCODE_VMINSN, + VM_EXITCODE_BPT, VM_EXITCODE_MAX }; @@ -645,6 +648,9 @@ uint64_t exitinfo1; uint64_t exitinfo2; } svm; + struct { + int inst_length; + } bpt; struct { uint32_t code; /* ecx value */ uint64_t wval; Index: sys/amd64/vmm/amd/svm.c =================================================================== --- sys/amd64/vmm/amd/svm.c +++ sys/amd64/vmm/amd/svm.c @@ -2190,6 +2190,11 @@ return (0); } + if (ident == VM_REG_GUEST_ENTRY_INST_LENGTH) { + /* Ignore. */ + return (0); + } + /* * XXX deal with CR3 and invalidate TLB entries tagged with the * vcpu's ASID. This needs to be treated differently depending on Index: sys/amd64/vmm/intel/vmcs.c =================================================================== --- sys/amd64/vmm/intel/vmcs.c +++ sys/amd64/vmm/intel/vmcs.c @@ -120,6 +120,8 @@ return (VMCS_GUEST_PDPTE2); case VM_REG_GUEST_PDPTE3: return (VMCS_GUEST_PDPTE3); + case VM_REG_GUEST_ENTRY_INST_LENGTH: + return (VMCS_ENTRY_INST_LENGTH); default: return (-1); } Index: sys/amd64/vmm/intel/vmx.h =================================================================== --- sys/amd64/vmm/intel/vmx.h +++ sys/amd64/vmm/intel/vmx.h @@ -87,6 +87,7 @@ int set; uint32_t proc_ctls; uint32_t proc_ctls2; + uint32_t exc_bitmap; }; struct vmxstate { Index: sys/amd64/vmm/intel/vmx.c =================================================================== --- sys/amd64/vmm/intel/vmx.c +++ sys/amd64/vmm/intel/vmx.c @@ -1071,6 +1071,7 @@ vmx->cap[i].set = 0; vmx->cap[i].proc_ctls = procbased_ctls; vmx->cap[i].proc_ctls2 = procbased_ctls2; + vmx->cap[i].exc_bitmap = exc_bitmap; vmx->state[i].nextrip = ~0; vmx->state[i].lastcpu = NOCPU; @@ -2547,6 +2548,18 @@ return (1); } + /* + * If the hypervisor has requested user exits for + * debug exceptions, bounce them out to userland. + */ + if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP && + (vmx->cap[vcpu].set & (1 << VM_CAP_BPT_EXIT))) { + vmexit->exitcode = VM_EXITCODE_BPT; + vmexit->u.bpt.inst_length = vmexit->inst_length; + vmexit->inst_length = 0; + break; + } + if (intr_vec == IDT_PF) { error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual); KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d", @@ -3296,6 +3309,9 @@ if (cap_invpcid) ret = 0; break; + case VM_CAP_BPT_EXIT: + ret = 0; + break; default: break; } @@ -3367,11 +3383,25 @@ reg = VMCS_SEC_PROC_BASED_CTLS; } break; + case VM_CAP_BPT_EXIT: + retval = 0; + + /* Don't change the bitmap if we are tracing all exceptions. */ + if (vmx->cap[vcpu].exc_bitmap != 0xffffffff) { + pptr = &vmx->cap[vcpu].exc_bitmap; + baseval = *pptr; + flag = (1 << IDT_BP); + reg = VMCS_EXCEPTION_BITMAP; + } + break; default: break; } - if (retval == 0) { + if (retval) + return (retval); + + if (pptr != NULL) { if (val) { baseval |= flag; } else { @@ -3381,26 +3411,23 @@ error = vmwrite(reg, baseval); VMCLEAR(vmcs); - if (error) { - retval = error; - } else { - /* - * Update optional stored flags, and record - * setting - */ - if (pptr != NULL) { - *pptr = baseval; - } + if (error) + return (error); - if (val) { - vmx->cap[vcpu].set |= (1 << type); - } else { - vmx->cap[vcpu].set &= ~(1 << type); - } - } + /* + * Update optional stored flags, and record + * setting + */ + *pptr = baseval; } - return (retval); + if (val) { + vmx->cap[vcpu].set |= (1 << type); + } else { + vmx->cap[vcpu].set &= ~(1 << type); + } + + return (0); } struct vlapic_vtx { Index: usr.sbin/bhyve/bhyverun.c =================================================================== --- usr.sbin/bhyve/bhyverun.c +++ usr.sbin/bhyve/bhyverun.c @@ -167,6 +167,7 @@ char *guest_uuid_str; +static int gdb_port = 0; static int guest_vmexit_on_hlt, guest_vmexit_on_pause; static int virtio_msix = 1; static int x2apic_mode = 0; /* default is xAPIC */ @@ -416,7 +417,8 @@ snprintf(tname, sizeof(tname), "vcpu %d", vcpu); pthread_set_name_np(mtp->mt_thr, tname); - gdb_cpu_add(vcpu); + if (gdb_port != 0) + gdb_cpu_add(vcpu); vm_loop(mtp->mt_ctx, vcpu, vmexit[vcpu].rip); @@ -690,8 +692,12 @@ stats.vmexit_mtrap++; + if (gdb_port == 0) { + fprintf(stderr, "vm_loop: unexpected VMEXIT_MTRAP\n"); + exit(4); + } + gdb_cpu_mtrap(*pvcpu); - return (VMEXIT_CONTINUE); } @@ -770,10 +776,26 @@ vmexit_debug(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) { + if (gdb_port == 0) { + fprintf(stderr, "vm_loop: unexpected VMEXIT_DEBUG\n"); + exit(4); + } gdb_cpu_suspend(*pvcpu); return (VMEXIT_CONTINUE); } +static int +vmexit_breakpoint(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu) +{ + + if (gdb_port == 0) { + fprintf(stderr, "vm_loop: unexpected VMEXIT_DEBUG\n"); + exit(4); + } + gdb_cpu_breakpoint(*pvcpu, vmexit); + return (VMEXIT_CONTINUE); +} + static vmexit_handler_t handler[VM_EXITCODE_MAX] = { [VM_EXITCODE_INOUT] = vmexit_inout, [VM_EXITCODE_INOUT_STR] = vmexit_inout, @@ -789,6 +811,7 @@ [VM_EXITCODE_SUSPENDED] = vmexit_suspend, [VM_EXITCODE_TASK_SWITCH] = vmexit_task_switch, [VM_EXITCODE_DEBUG] = vmexit_debug, + [VM_EXITCODE_BPT] = vmexit_breakpoint, }; static void @@ -975,7 +998,7 @@ int main(int argc, char *argv[]) { - int c, error, dbg_port, gdb_port, err, bvmcons; + int c, error, dbg_port, err, bvmcons; int max_vcpus, mptgen, memflags; int rtc_localtime; bool gdb_stop; @@ -987,7 +1010,6 @@ bvmcons = 0; progname = basename(argv[0]); dbg_port = 0; - gdb_port = 0; gdb_stop = false; guest_ncpus = 1; sockets = cores = threads = 1; Index: usr.sbin/bhyve/gdb.h =================================================================== --- usr.sbin/bhyve/gdb.h +++ usr.sbin/bhyve/gdb.h @@ -31,6 +31,7 @@ #define __GDB_H__ void gdb_cpu_add(int vcpu); +void gdb_cpu_breakpoint(int vcpu, struct vm_exit *vmexit); void gdb_cpu_mtrap(int vcpu); void gdb_cpu_suspend(int vcpu); void init_gdb(struct vmctx *ctx, int sport, bool wait); Index: usr.sbin/bhyve/gdb.c =================================================================== --- usr.sbin/bhyve/gdb.c +++ usr.sbin/bhyve/gdb.c @@ -34,6 +34,7 @@ #endif #include #include +#include #include #include #include @@ -57,6 +58,7 @@ #include #include "bhyverun.h" +#include "gdb.h" #include "mem.h" #include "mevent.h" @@ -74,8 +76,7 @@ static cpuset_t vcpus_active, vcpus_suspended, vcpus_waiting; static pthread_mutex_t gdb_lock; static pthread_cond_t idle_vcpus; -static bool stop_pending, first_stop; -static int stepping_vcpu, stopped_vcpu; +static bool first_stop, swbreak_enabled, report_next_stop; /* * An I/O buffer contains 'capacity' bytes of room at 'data'. For a @@ -91,11 +92,49 @@ size_t len; }; +struct breakpoint { + uint64_t gpa; + int refs; + uint8_t shadow_inst; + TAILQ_ENTRY(breakpoint) link; +}; + +/* + * When a vCPU stops to due to an event that should be reported to the + * debugger, information about the event is stored in this structure. + * 'stopped_vcpus' is a linked list of vCPUs with pending events that + * need to be reported. The report_stop() function reports the event + * from the first pending vCPU in this list. When the debugger + * resumes execution via continue or step, the first pending event is + * discarded. + * + * An idle vCPU will have all of the boolean fields set to false. + * + * When a vCPU is stepped, 'stepping' is set to true when the vCPU is + * released to execute the stepped instruction. When the vCPU reports + * the stepping trap, 'stepped' is set and the vCPU is then queued in + * 'stopped_vcpus'. + * + * When a vCPU hits a breakpoint set by the debug server, + * 'hit_swbreak' is set to true and the vCPU is then queued in + * 'stopped_vcpus'. + */ +struct vcpu_state { + int vcpu; + bool stepping; + bool stepped; + bool hit_swbreak; + TAILQ_ENTRY(vcpu_state) link; +}; + static struct io_buffer cur_comm, cur_resp; static uint8_t cur_csum; static int cur_vcpu; static struct vmctx *ctx; static int cur_fd = -1; +static TAILQ_HEAD(, breakpoint) breakpoints; +static TAILQ_HEAD(, vcpu_state) stopped_vcpus; +static struct vcpu_state *vcpu_state; const int gdb_regset[] = { VM_REG_GUEST_RAX, @@ -555,7 +594,7 @@ if (value == 0) append_char('0'); else - append_unsigned_be(value, fls(value) + 7 / 8); + append_unsigned_be(value, (fls(value) + 7) / 8); } static void @@ -609,22 +648,48 @@ } static void -report_stop(void) +report_stop(bool set_cur_vcpu) { + struct vcpu_state *vs; + vs = TAILQ_FIRST(&stopped_vcpus); start_packet(); - if (stopped_vcpu == -1) + if (vs == NULL) append_char('S'); else append_char('T'); append_byte(GDB_SIGNAL_TRAP); - if (stopped_vcpu != -1) { + if (vs != NULL) { + if (set_cur_vcpu) + cur_vcpu = vs->vcpu; append_string("thread:"); - append_integer(stopped_vcpu + 1); + append_integer(vs->vcpu + 1); append_char(';'); + if (vs->hit_swbreak) { + debug("$vCPU %d reporting swbreak\n", vs->vcpu); + if (swbreak_enabled) + append_string("swbreak:;"); + } else if (vs->stepped) + debug("$vCPU %d reporting step\n", vs->vcpu); + else + debug("$vCPU %d reporting ???\n", vs->vcpu); } - stopped_vcpu = -1; finish_packet(); + report_next_stop = false; +} + +static void +discard_stop(void) +{ + struct vcpu_state *vs; + + vs = TAILQ_FIRST(&stopped_vcpus); + if (vs != NULL) { + vs->hit_swbreak = false; + vs->stepped = false; + TAILQ_REMOVE(&stopped_vcpus, vs, link); + } + report_next_stop = true; } static void @@ -633,11 +698,10 @@ if (first_stop) { first_stop = false; - stopped_vcpu = -1; - } else if (response_pending()) - stop_pending = true; - else { - report_stop(); + TAILQ_INIT(&stopped_vcpus); + } else if (report_next_stop) { + assert(!response_pending()); + report_stop(true); send_pending_data(cur_fd); } } @@ -650,7 +714,7 @@ CPU_SET(vcpu, &vcpus_waiting); if (report_stop && CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0) gdb_finish_suspend_vcpus(); - while (CPU_ISSET(vcpu, &vcpus_suspended) && vcpu != stepping_vcpu) + while (CPU_ISSET(vcpu, &vcpus_suspended)) pthread_cond_wait(&idle_vcpus, &gdb_lock); CPU_CLR(vcpu, &vcpus_waiting); debug("$vCPU %d resuming\n", vcpu); @@ -662,7 +726,13 @@ debug("$vCPU %d starting\n", vcpu); pthread_mutex_lock(&gdb_lock); + assert(vcpu < guest_ncpus); CPU_SET(vcpu, &vcpus_active); + vcpu_state[vcpu].vcpu = vcpu; + if (!TAILQ_EMPTY(&breakpoints)) { + vm_set_capability(ctx, vcpu, VM_CAP_BPT_EXIT, 1); + debug("$vCPU %d enabled breakpoint exits\n", vcpu); + } /* * If a vcpu is added while vcpus are stopped, suspend the new @@ -685,23 +755,6 @@ pthread_mutex_unlock(&gdb_lock); } -void -gdb_cpu_mtrap(int vcpu) -{ - - debug("$vCPU %d MTRAP\n", vcpu); - pthread_mutex_lock(&gdb_lock); - if (vcpu == stepping_vcpu) { - stepping_vcpu = -1; - vm_set_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, 0); - vm_suspend_cpu(ctx, vcpu); - assert(stopped_vcpu == -1); - stopped_vcpu = vcpu; - _gdb_cpu_suspend(vcpu, true); - } - pthread_mutex_unlock(&gdb_lock); -} - static void gdb_suspend_vcpus(void) { @@ -714,18 +767,99 @@ gdb_finish_suspend_vcpus(); } +void +gdb_cpu_mtrap(int vcpu) +{ + struct vcpu_state *vs; + + debug("$vCPU %d MTRAP\n", vcpu); + pthread_mutex_lock(&gdb_lock); + vs = &vcpu_state[vcpu]; + if (vs->stepping) { + vs->stepping = false; + vs->stepped = true; + vm_set_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, 0); + vm_suspend_cpu(ctx, vcpu); + CPU_SET(vcpu, &vcpus_suspended); + debug("$vCPU %d reporting step\n", vs->vcpu); + TAILQ_INSERT_HEAD(&stopped_vcpus, vs, link); + _gdb_cpu_suspend(vcpu, true); + } + pthread_mutex_unlock(&gdb_lock); +} + +static struct breakpoint * +find_breakpoint(uint64_t gpa) +{ + struct breakpoint *bp; + + TAILQ_FOREACH(bp, &breakpoints, link) { + if (bp->gpa == gpa) + return (bp); + } + return (NULL); +} + +void +gdb_cpu_breakpoint(int vcpu, struct vm_exit *vmexit) +{ + struct breakpoint *bp; + struct vcpu_state *vs; + uint64_t gpa; + int error; + + pthread_mutex_lock(&gdb_lock); + error = guest_vaddr2paddr(vcpu, vmexit->rip, &gpa); + assert(error == 1); + bp = find_breakpoint(gpa); + if (bp != NULL) { + vs = &vcpu_state[vcpu]; + assert(vs->stepping == false); + assert(vs->stepped == false); + assert(vs->hit_swbreak == false); + vs->hit_swbreak = true; + vm_set_register(ctx, vcpu, VM_REG_GUEST_RIP, vmexit->rip); + debug("$vCPU %d reporting breakpoint at rip %#lx\n", vcpu, + vmexit->rip); + TAILQ_INSERT_TAIL(&stopped_vcpus, vs, link); + gdb_suspend_vcpus(); + _gdb_cpu_suspend(vcpu, true); + } else { + debug("$vCPU %d injecting breakpoint at rip %#lx\n", vcpu, + vmexit->rip); + error = vm_set_register(ctx, vcpu, + VM_REG_GUEST_ENTRY_INST_LENGTH, vmexit->u.bpt.inst_length); + assert(error == 0); + error = vm_inject_exception(ctx, vcpu, IDT_BP, 0, 0, 0); + assert(error == 0); + } + pthread_mutex_unlock(&gdb_lock); +} + static bool gdb_step_vcpu(int vcpu) { + struct vcpu_state *vs; int error, val; debug("$vCPU %d step\n", vcpu); error = vm_get_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, &val); if (error < 0) return (false); + + /* + * XXX: What if this vCPU has a pending event not cleared by + * discard_stop? + */ + discard_stop(); + vs = &vcpu_state[vcpu]; + assert(vs->stepping == false); + assert(vs->stepped == false); error = vm_set_capability(ctx, vcpu, VM_CAP_MTRAP_EXIT, 1); + assert(error == 0); vm_resume_cpu(ctx, vcpu); - stepping_vcpu = vcpu; + vs->stepping = true; + CPU_CLR(vcpu, &vcpus_suspended); pthread_cond_broadcast(&idle_vcpus); return (true); } @@ -983,6 +1117,159 @@ send_ok(); } +static bool +set_breakpoint_caps(bool enable) +{ + cpuset_t mask; + int vcpu; + + mask = vcpus_active; + while (!CPU_EMPTY(&mask)) { + vcpu = CPU_FFS(&mask) - 1; + CPU_CLR(vcpu, &mask); + if (vm_set_capability(ctx, vcpu, VM_CAP_BPT_EXIT, + enable ? 1 : 0) < 0) + return (false); + debug("$vCPU %d %sabled breakpoint exits\n", vcpu, + enable ? "en" : "dis"); + } + return (true); +} + +static void +update_sw_breakpoint(uint64_t gva, int kind, bool insert) +{ + struct breakpoint *bp; + uint64_t gpa; + uint8_t *cp; + int error; + + if (kind != 1) { + send_error(EINVAL); + return; + } + + error = guest_vaddr2paddr(cur_vcpu, gva, &gpa); + if (error == -1) { + send_error(errno); + return; + } + if (error == 0) { + send_error(EFAULT); + return; + } + + cp = paddr_guest2host(ctx, gpa, 1); + + /* Only permit breakpoints in guest RAM. */ + if (cp == NULL) { + send_error(EFAULT); + return; + } + + /* Find any existing breakpoint. */ + bp = find_breakpoint(gpa); + + if (insert) { + if (bp == NULL) { + if (TAILQ_EMPTY(&breakpoints) && + !set_breakpoint_caps(true)) { + send_empty_response(); + return; + } + bp = malloc(sizeof(*bp)); + bp->gpa = gpa; + bp->refs = 1; + bp->shadow_inst = *cp; + *cp = 0xcc; /* INT 3 */ + TAILQ_INSERT_TAIL(&breakpoints, bp, link); + debug("new breakpoint at %#lx\n", gpa); + } else { + bp->refs++; + assert(bp->refs != 0); + } + } else { + if (bp == NULL) { + send_error(ENOENT); + return; + } + if (bp->refs == 1) { + debug("remove breakpoint at %#lx\n", gpa); + *cp = bp->shadow_inst; + TAILQ_REMOVE(&breakpoints, bp, link); + free(bp); + if (TAILQ_EMPTY(&breakpoints)) + set_breakpoint_caps(false); + } else + bp->refs--; + } + send_ok(); +} + +static void +parse_breakpoint(const uint8_t *data, size_t len) +{ + uint64_t gva; + uint8_t *cp; + bool insert; + int kind, type; + + insert = data[0] == 'Z'; + + /* Skip 'Z/z' */ + data += 1; + len -= 1; + + /* Parse and consume type. */ + cp = memchr(data, ',', len); + if (cp == NULL || cp == data) { + send_error(EINVAL); + return; + } + type = parse_integer(data, cp - data); + len -= (cp - data) + 1; + data += (cp - data) + 1; + + /* Parse and consume address. */ + cp = memchr(data, ',', len); + if (cp == NULL || cp == data) { + send_error(EINVAL); + return; + } + gva = parse_integer(data, cp - data); + len -= (cp - data) + 1; + data += (cp - data) + 1; + + /* Parse and consume kind. */ + cp = memchr(data, ';', len); + if (cp == data) { + send_error(EINVAL); + return; + } + if (cp != NULL) { + /* + * We do not advertise support for either the + * ConditionalBreakpoints or BreakpointCommands + * features, so we should not be getting conditions or + * commands from the remote end. + */ + send_empty_response(); + return; + } + kind = parse_integer(data, len); + data += len; + len = 0; + + switch (type) { + case 0: + update_sw_breakpoint(gva, kind, insert); + break; + default: + send_empty_response(); + break; + } +} + static bool command_equals(const uint8_t *data, size_t len, const char *cmd) { @@ -1041,7 +1328,8 @@ value = NULL; } - /* No currently supported features. */ + if (strcmp(feature, "swbreak") == 0) + swbreak_enabled = supported; } free(str); @@ -1049,6 +1337,7 @@ /* This is an arbitrary limit. */ append_string("PacketSize=4096"); + append_string(";swbreak+"); finish_packet(); } @@ -1142,8 +1431,12 @@ break; } - /* Don't send a reply until a stop occurs. */ - gdb_resume_vcpus(); + discard_stop(); + if (TAILQ_EMPTY(&stopped_vcpus)) { + /* Don't send a reply until a stop occurs. */ + gdb_resume_vcpus(); + } else + report_stop(true); break; case 'D': send_ok(); @@ -1214,13 +1507,12 @@ break; } break; + case 'z': + case 'Z': + parse_breakpoint(data, len); + break; case '?': - /* XXX: Only if stopped? */ - /* For now, just report that we are always stopped. */ - start_packet(); - append_char('S'); - append_byte(GDB_SIGNAL_TRAP); - finish_packet(); + report_stop(false); break; case 'G': /* TODO */ case 'v': @@ -1231,8 +1523,6 @@ case 'Q': /* TODO */ case 't': /* TODO */ case 'X': /* TODO */ - case 'z': /* TODO */ - case 'Z': /* TODO */ default: send_empty_response(); } @@ -1263,9 +1553,8 @@ if (response_pending()) io_buffer_reset(&cur_resp); io_buffer_consume(&cur_comm, 1); - if (stop_pending) { - stop_pending = false; - report_stop(); + if (!TAILQ_EMPTY(&stopped_vcpus) && report_next_stop) { + report_stop(true); send_pending_data(fd); } break; @@ -1377,7 +1666,7 @@ static void new_connection(int fd, enum ev_type event, void *arg) { - int optval, s; + int optval, s, vcpu; s = accept4(fd, NULL, NULL, SOCK_NONBLOCK); if (s == -1) { @@ -1419,12 +1708,16 @@ cur_fd = s; cur_vcpu = 0; - stepping_vcpu = -1; - stopped_vcpu = -1; - stop_pending = false; + for (vcpu = 0; vcpu < guest_ncpus; vcpu++) { + vcpu_state[vcpu].stepping = false; + vcpu_state[vcpu].stepped = false; + vcpu_state[vcpu].hit_swbreak = false; + } + TAILQ_INIT(&stopped_vcpus); /* Break on attach. */ first_stop = true; + report_next_stop = false; gdb_suspend_vcpus(); pthread_mutex_unlock(&gdb_lock); } @@ -1476,6 +1769,9 @@ if (listen(s, 1) < 0) err(1, "gdb socket listen"); + TAILQ_INIT(&stopped_vcpus); + TAILQ_INIT(&breakpoints); + vcpu_state = calloc(guest_ncpus, sizeof(*vcpu_state)); if (wait) { /* * Set vcpu 0 in vcpus_suspended. This will trigger the @@ -1483,9 +1779,8 @@ * it starts execution. The vcpu will remain suspended * until a debugger connects. */ - stepping_vcpu = -1; - stopped_vcpu = -1; CPU_SET(0, &vcpus_suspended); + TAILQ_INSERT_TAIL(&stopped_vcpus, &vcpu_state[0], link); } flags = fcntl(s, F_GETFL);