Page MenuHomeFreeBSD
Files F177525277

D28199.1794627610.diff
No OneTemporary
Actions

Authored By
Unknown
Size
18 KB
Referenced Files
None
Subscribers
None

D28199.1794627610.diff
View Options

Index: etc/mtree/BSD.var.dist
===================================================================
--- etc/mtree/BSD.var.dist
+++ etc/mtree/BSD.var.dist
@@ -76,6 +76,8 @@
run
dhclient
..
+ bhyve
+ ..
ppp gname=network mode=0770
..
wpa_supplicant
Index: lib/libvmmapi/vmmapi.h
===================================================================
--- lib/libvmmapi/vmmapi.h
+++ lib/libvmmapi/vmmapi.h
@@ -43,6 +43,14 @@
*/
#define VMMAPI_VERSION 0103 /* 2 digit major followed by 2 digit minor */
+/*
+ * Note, a bhyve uses its VM name in the path of its unix domain socket.
+ * The socket path must be less than SUNPATHLEN (104).
+ * The default prefix for BHYVE_RUN_DIR is '/var/run/bhyve/' (15).
+ * So, 104-15 = 89. This allows for a VM name of 88 characters (+1 for terminator).
+ */
+#define MAX_VMNAME 89
+
struct iovec;
struct vmctx;
struct vm_snapshot_meta;
@@ -255,19 +263,6 @@
/*
* Save and restore
*/
-
-#define MAX_SNAPSHOT_VMNAME 100
-
-enum checkpoint_opcodes {
- START_CHECKPOINT = 0,
- START_SUSPEND = 1,
-};
-
-struct checkpoint_op {
- unsigned int op;
- char snapshot_filename[MAX_SNAPSHOT_VMNAME];
-};
-
int vm_snapshot_req(struct vm_snapshot_meta *meta);
int vm_restore_time(struct vmctx *ctx);
Index: share/man/man7/hier.7
===================================================================
--- share/man/man7/hier.7
+++ share/man/man7/hier.7
@@ -28,7 +28,7 @@
.\" @(#)hier.7 8.1 (Berkeley) 6/5/93
.\" $FreeBSD$
.\"
-.Dd January 9, 2021
+.Dd January 30, 2021
.Dt HIER 7
.Os
.Sh NAME
@@ -832,6 +832,8 @@
system since it was booted
.Pp
.Bl -tag -width Fl -compact
+.It Pa bhyve/
+bhyve vm unix domain socket.
.It Pa ppp/
writable by the
.Dq network
Index: usr.sbin/bhyve/Makefile
===================================================================
--- usr.sbin/bhyve/Makefile
+++ usr.sbin/bhyve/Makefile
@@ -75,7 +75,7 @@
iov.c
.if ${MK_BHYVE_SNAPSHOT} != "no"
-SRCS+= snapshot.c
+SRCS+= ipc.c snapshot.c
.endif
CFLAGS.kernemu_dev.c+= -I${SRCTOP}/sys/amd64
Index: usr.sbin/bhyve/bhyverun.c
===================================================================
--- usr.sbin/bhyve/bhyverun.c
+++ usr.sbin/bhyve/bhyverun.c
@@ -101,6 +101,7 @@
#include "pci_lpc.h"
#include "smbiostbl.h"
#ifdef BHYVE_SNAPSHOT
+#include "ipc.h"
#include "snapshot.h"
#endif
#include "xmsr.h"
@@ -1402,11 +1403,12 @@
if (restore_file != NULL)
destroy_restore_state(&rstate);
- /*
- * checkpointing thread for communication with bhyvectl
- */
- if (init_checkpoint_thread(ctx) < 0)
- printf("Failed to start checkpoint thread!\r\n");
+ /* initialize mutex/cond variables */
+ init_snapshot();
+
+ /* open up IPC to bhyve */
+ if (init_ipc(ctx) != 0)
+ fprintf(stderr, "Unable to open IPC to %s\n", vmname);
if (restore_file != NULL)
vm_restore_time(ctx);
Index: usr.sbin/bhyve/ipc.h
===================================================================
--- /dev/null
+++ usr.sbin/bhyve/ipc.h
@@ -0,0 +1,63 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021 Robert Wing <rew@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef _IPC_H_
+#define _IPC_H_
+
+#define BHYVE_RUN_DIR "/var/run/bhyve/"
+
+#define MAX_SNAPSHOT_FILENAME 100
+
+int init_ipc(struct vmctx *);
+
+/* Filename that will be used for save/restore */
+struct checkpoint_op {
+ char snapshot_filename[MAX_SNAPSHOT_FILENAME];
+};
+
+/* Messages that a bhyve process understands. */
+enum ipc_opcode {
+ START_CHECKPOINT,
+ START_SUSPEND,
+};
+
+/*
+ * The type of message and associated data to
+ * send to a bhyve process.
+ */
+struct ipc_message {
+ enum ipc_opcode code;
+ union {
+ /*
+ * message specific structures
+ */
+ struct checkpoint_op op;
+ } data;
+};
+
+#endif /* _IPC_H_ */
Index: usr.sbin/bhyve/ipc.c
===================================================================
--- /dev/null
+++ usr.sbin/bhyve/ipc.c
@@ -0,0 +1,137 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2016 Flavius Anton
+ * Copyright (c) 2016 Mihai Tiganus
+ * Copyright (c) 2016-2019 Mihai Carabas
+ * Copyright (c) 2017-2019 Darius Mihai
+ * Copyright (c) 2017-2019 Elena Mihailescu
+ * Copyright (c) 2018-2019 Sergiu Weisz
+ * All rights reserved.
+ * The bhyve-snapshot feature was developed under sponsorships
+ * from Matthew Grooms.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+
+#include <errno.h>
+#include <pthread.h>
+#include <pthread_np.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <machine/vmm.h>
+#include <vmmapi.h>
+
+#include "ipc.h"
+#include "snapshot.h"
+
+static int
+handle_message(struct ipc_message *imsg, struct vmctx *ctx)
+{
+ int err;
+
+ switch (imsg->code) {
+ case START_CHECKPOINT:
+ err = vm_checkpoint(ctx, imsg->data.op.snapshot_filename, false);
+ break;
+ case START_SUSPEND:
+ err = vm_checkpoint(ctx, imsg->data.op.snapshot_filename, true);
+ break;
+ default:
+ fprintf(stderr, "Unrecognized message operation.\n");
+ err = -1;
+ }
+
+ if (err != 0)
+ perror("Unable to perform requested operation");
+
+ return (err);
+}
+
+static void *
+ipc_thread(void *arg)
+{
+ struct vmctx *ctx;
+ struct ipc_message imsg;
+ struct sockaddr_un addr;
+ int bytes, socket_fd;
+ char vmname[MAX_VMNAME];
+
+ pthread_set_name_np(pthread_self(), "ipc thread");
+ ctx = (struct vmctx *)arg;
+
+ if (vm_get_name(ctx, vmname, MAX_VMNAME - 1) != 0) {
+ perror("Failed to get VM name");
+ return (NULL);
+ }
+
+ socket_fd = socket(PF_UNIX, SOCK_DGRAM, 0);
+ if (socket_fd < 0) {
+ perror("Socket creation failed (IPC with bhyvectl");
+ return (NULL);
+ }
+
+ /*
+ * Note, BHYVE_RUN_DIR + vmname should be < SUNPATHLEN (104)
+ * which allows for a vmname of 88 characters.
+ */
+ memset(&addr, 0, sizeof(addr));
+ addr.sun_family = AF_UNIX;
+ snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s",
+ BHYVE_RUN_DIR, vmname);
+ addr.sun_len = SUN_LEN(&addr);
+ unlink(addr.sun_path);
+
+ if (bind(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
+ perror("Failed to bind socket (IPC with bhyvectl)");
+ close(socket_fd);
+ return (NULL);
+ }
+
+ for (;;) {
+ bytes = recvfrom(socket_fd, &imsg, sizeof(imsg), 0, NULL, 0);
+ if (bytes >= sizeof(imsg.code))
+ handle_message(&imsg, ctx);
+ }
+
+ unlink(addr.sun_path);
+ close(socket_fd);
+ return (NULL);
+}
+
+/*
+ * Start a thread to handle IPC messages.
+ */
+int
+init_ipc(struct vmctx *ctx)
+{
+ pthread_t ipc_pthread;
+
+ return (pthread_create(&ipc_pthread, NULL, ipc_thread, ctx));
+}
Index: usr.sbin/bhyve/snapshot.h
===================================================================
--- usr.sbin/bhyve/snapshot.h
+++ usr.sbin/bhyve/snapshot.h
@@ -57,11 +57,6 @@
ucl_object_t *meta_root_obj;
};
-struct checkpoint_thread_info {
- struct vmctx *ctx;
- int socket_fd;
-};
-
typedef int (*vm_snapshot_dev_cb)(struct vm_snapshot_meta *);
typedef int (*vm_pause_dev_cb) (struct vmctx *, const char *);
typedef int (*vm_resume_dev_cb) (struct vmctx *, const char *);
@@ -96,9 +91,8 @@
int vm_pause_user_devs(struct vmctx *ctx);
int vm_resume_user_devs(struct vmctx *ctx);
-int get_checkpoint_msg(int conn_fd, struct vmctx *ctx);
-void *checkpoint_thread(void *param);
-int init_checkpoint_thread(struct vmctx *ctx);
+int vm_checkpoint(struct vmctx *ctx, char *checkpoint_file, bool stop_vm);
+int init_snapshot(void);
int load_restore_file(const char *filename, struct restore_state *rstate);
Index: usr.sbin/bhyve/snapshot.c
===================================================================
--- usr.sbin/bhyve/snapshot.c
+++ usr.sbin/bhyve/snapshot.c
@@ -115,12 +115,6 @@
#define SNAPSHOT_CHUNK (4 * MB)
#define PROG_BUF_SZ (8192)
-#define BHYVE_RUN_DIR "/var/run/bhyve"
-#define CHECKPOINT_RUN_DIR BHYVE_RUN_DIR "/checkpoint"
-#define MAX_VMNAME 100
-
-#define MAX_MSG_SIZE 1024
-
#define SNAPSHOT_BUFFER_SIZE (20 * MB)
#define JSON_STRUCT_ARR_KEY "structs"
@@ -1327,7 +1321,7 @@
pthread_cond_broadcast(&vcpus_can_run);
}
-static int
+int
vm_checkpoint(struct vmctx *ctx, char *checkpoint_file, bool stop_vm)
{
int fd_checkpoint = 0, kdata_fd = 0;
@@ -1444,102 +1438,10 @@
}
int
-get_checkpoint_msg(int conn_fd, struct vmctx *ctx)
-{
- unsigned char buf[MAX_MSG_SIZE];
- struct checkpoint_op *checkpoint_op;
- int len, recv_len, total_recv = 0;
- int err = 0;
-
- len = sizeof(struct checkpoint_op); /* expected length */
- while ((recv_len = recv(conn_fd, buf + total_recv, len - total_recv, 0)) > 0) {
- total_recv += recv_len;
- }
- if (recv_len < 0) {
- perror("Error while receiving data from bhyvectl");
- err = -1;
- goto done;
- }
-
- checkpoint_op = (struct checkpoint_op *)buf;
- switch (checkpoint_op->op) {
- case START_CHECKPOINT:
- err = vm_checkpoint(ctx, checkpoint_op->snapshot_filename, false);
- break;
- case START_SUSPEND:
- err = vm_checkpoint(ctx, checkpoint_op->snapshot_filename, true);
- break;
- default:
- fprintf(stderr, "Unrecognized checkpoint operation.\n");
- err = -1;
- }
-
-done:
- close(conn_fd);
- return (err);
-}
-
-/*
- * Listen for commands from bhyvectl
- */
-void *
-checkpoint_thread(void *param)
-{
- struct checkpoint_thread_info *thread_info;
- int conn_fd, ret;
-
- pthread_set_name_np(pthread_self(), "checkpoint thread");
- thread_info = (struct checkpoint_thread_info *)param;
-
- while ((conn_fd = accept(thread_info->socket_fd, NULL, NULL)) > -1) {
- ret = get_checkpoint_msg(conn_fd, thread_info->ctx);
- if (ret != 0) {
- fprintf(stderr, "Failed to read message on checkpoint "
- "socket. Retrying.\n");
- }
- }
- if (conn_fd < -1) {
- perror("Failed to accept connection");
- }
-
- return (NULL);
-}
-
-/*
- * Create directory tree to store runtime specific information:
- * i.e. UNIX sockets for IPC with bhyvectl.
- */
-static int
-make_checkpoint_dir(void)
+init_snapshot(void)
{
int err;
- err = mkdir(BHYVE_RUN_DIR, 0755);
- if (err < 0 && errno != EEXIST)
- return (err);
-
- err = mkdir(CHECKPOINT_RUN_DIR, 0755);
- if (err < 0 && errno != EEXIST)
- return (err);
-
- return 0;
-}
-
-/*
- * Create the listening socket for IPC with bhyvectl
- */
-int
-init_checkpoint_thread(struct vmctx *ctx)
-{
- struct checkpoint_thread_info *checkpoint_info = NULL;
- struct sockaddr_un addr;
- int socket_fd;
- pthread_t checkpoint_pthread;
- char vmname_buf[MAX_VMNAME];
- int ret, err = 0;
-
- memset(&addr, 0, sizeof(addr));
-
err = pthread_mutex_init(&vcpu_lock, NULL);
if (err != 0)
errc(1, err, "checkpoint mutex init");
@@ -1549,63 +1451,7 @@
if (err != 0)
errc(1, err, "checkpoint cv init");
- socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
- if (socket_fd < 0) {
- perror("Socket creation failed (IPC with bhyvectl");
- err = -1;
- goto fail;
- }
-
- err = make_checkpoint_dir();
- if (err < 0) {
- perror("Failed to create checkpoint runtime directory");
- goto fail;
- }
-
- addr.sun_family = AF_UNIX;
-
- err = vm_get_name(ctx, vmname_buf, MAX_VMNAME - 1);
- if (err != 0) {
- perror("Failed to get VM name");
- goto fail;
- }
-
- snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/%s",
- CHECKPOINT_RUN_DIR, vmname_buf);
- addr.sun_len = SUN_LEN(&addr);
- unlink(addr.sun_path);
-
- if (bind(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
- perror("Failed to bind socket (IPC with bhyvectl)");
- err = -1;
- goto fail;
- }
-
- if (listen(socket_fd, 10) < 0) {
- perror("Failed to listen on socket (IPC with bhyvectl)");
- err = -1;
- goto fail;
- }
-
- checkpoint_info = calloc(1, sizeof(*checkpoint_info));
- checkpoint_info->ctx = ctx;
- checkpoint_info->socket_fd = socket_fd;
-
- ret = pthread_create(&checkpoint_pthread, NULL, checkpoint_thread,
- checkpoint_info);
- if (ret < 0) {
- err = ret;
- goto fail;
- }
-
return (0);
-fail:
- free(checkpoint_info);
- if (socket_fd > 0)
- close(socket_fd);
- unlink(addr.sun_path);
-
- return (err);
}
void
Index: usr.sbin/bhyvectl/Makefile
===================================================================
--- usr.sbin/bhyvectl/Makefile
+++ usr.sbin/bhyvectl/Makefile
@@ -18,6 +18,9 @@
.if ${MK_BHYVE_SNAPSHOT} != "no"
CFLAGS+= -DBHYVE_SNAPSHOT
+
+BHYVE= ${SRCTOP}/usr.sbin/bhyve
+CFLAGS+= -I${BHYVE}
.endif
.include <bsd.prog.mk>
Index: usr.sbin/bhyvectl/bhyvectl.c
===================================================================
--- usr.sbin/bhyvectl/bhyvectl.c
+++ usr.sbin/bhyvectl/bhyvectl.c
@@ -38,6 +38,7 @@
#include <sys/mman.h>
#include <sys/cpuset.h>
+#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
@@ -63,6 +64,10 @@
#include "amd/vmcb.h"
#include "intel/vmcs.h"
+#ifdef BHYVE_SNAPSHOT
+#include "ipc.h"
+#endif
+
#define MB (1UL << 20)
#define GB (1UL << 30)
@@ -70,9 +75,6 @@
#define NO_ARG no_argument
#define OPT_ARG optional_argument
-#define CHECKPOINT_RUN_DIR "/var/run/bhyve/checkpoint"
-#define MAX_VMNAME 100
-
static const char *progname;
static void
@@ -1681,77 +1683,53 @@
#ifdef BHYVE_SNAPSHOT
static int
-send_checkpoint_op_req(struct vmctx *ctx, struct checkpoint_op *op)
+send_message(struct vmctx *ctx, void *data, size_t len)
{
struct sockaddr_un addr;
- int socket_fd, len, len_sent, total_sent;
- int err = 0;
- char vmname_buf[MAX_VMNAME];
+ int socket_fd, bytes, err;
+ char vmname[MAX_VMNAME];
- socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
+ err = vm_get_name(ctx, vmname, MAX_VMNAME - 1);
+ if (err != 0) {
+ perror("Failed to get VM name");
+ return (err);
+ }
+
+ socket_fd = socket(PF_UNIX, SOCK_DGRAM, 0);
if (socket_fd < 0) {
perror("Error creating bhyvectl socket");
- err = -1;
- goto done;
+ return (-1);
}
memset(&addr, 0, sizeof(struct sockaddr_un));
addr.sun_family = AF_UNIX;
+ snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s", BHYVE_RUN_DIR, vmname);
- err = vm_get_name(ctx, vmname_buf, MAX_VMNAME - 1);
- if (err != 0) {
- perror("Failed to get VM name");
- goto done;
- }
-
- snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/%s", CHECKPOINT_RUN_DIR, vmname_buf);
+ bytes = sendto(socket_fd, (char *)data, len, 0,
+ (struct sockaddr *)&addr, sizeof(struct sockaddr_un));
- if (connect(socket_fd, (struct sockaddr *)&addr,
- sizeof(struct sockaddr_un)) != 0) {
- perror("Connect to VM socket failed");
+ if (bytes < 0) {
+ perror("Failed to send message to bhyve vm");
err = -1;
- goto done;
}
+ close(socket_fd);
- len = sizeof(*op);
- total_sent = 0;
- while ((len_sent = send(socket_fd, (char *)op + total_sent, len - total_sent, 0)) > 0) {
- total_sent += len_sent;
- }
-
- if (len_sent < 0) {
- perror("Failed to send checkpoint operation request");
- err = -1;
- }
-
-done:
- if (socket_fd > 0)
- close(socket_fd);
return (err);
}
static int
-send_start_checkpoint(struct vmctx *ctx, const char *checkpoint_file)
+snapshot_request(struct vmctx *ctx, const char *file, enum ipc_opcode code)
{
- struct checkpoint_op op;
-
- op.op = START_CHECKPOINT;
- strncpy(op.snapshot_filename, checkpoint_file, MAX_SNAPSHOT_VMNAME);
- op.snapshot_filename[MAX_SNAPSHOT_VMNAME - 1] = 0;
+ struct ipc_message imsg;
+ size_t length;
- return (send_checkpoint_op_req(ctx, &op));
-}
+ imsg.code = code;
-static int
-send_start_suspend(struct vmctx *ctx, const char *suspend_file)
-{
- struct checkpoint_op op;
+ strlcpy(imsg.data.op.snapshot_filename, file, MAX_SNAPSHOT_FILENAME);
- op.op = START_SUSPEND;
- strncpy(op.snapshot_filename, suspend_file, MAX_SNAPSHOT_VMNAME);
- op.snapshot_filename[MAX_SNAPSHOT_VMNAME - 1] = 0;
+ length = offsetof(struct ipc_message, data) + sizeof(imsg.data.op);
- return (send_checkpoint_op_req(ctx, &op));
+ return (send_message(ctx, (void *)&imsg, length));
}
#endif
@@ -2413,10 +2391,10 @@
#ifdef BHYVE_SNAPSHOT
if (!error && vm_checkpoint_opt)
- error = send_start_checkpoint(ctx, checkpoint_file);
+ error = snapshot_request(ctx, checkpoint_file, START_CHECKPOINT);
if (!error && vm_suspend_opt)
- error = send_start_suspend(ctx, suspend_file);
+ error = snapshot_request(ctx, suspend_file, START_SUSPEND);
#endif
free (opts);

File Metadata

Mime Type
text/plain
Expires
Sat, Nov 14, 3:40 AM (15 h, 15 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41715030
Default Alt Text
D28199.1794627610.diff (18 KB)

Event Timeline