Changeset View
Changeset View
Standalone View
Standalone View
lib/libedit/tokenizer.c
/* $NetBSD: tokenizer.c,v 1.24 2016/02/17 19:47:49 christos Exp $ */ | /* $NetBSD: tokenizer.c,v 1.28 2016/04/11 18:56:31 christos Exp $ */ | ||||
/*- | /*- | ||||
* Copyright (c) 1992, 1993 | * Copyright (c) 1992, 1993 | ||||
* The Regents of the University of California. All rights reserved. | * The Regents of the University of California. All rights reserved. | ||||
* | * | ||||
* This code is derived from software contributed to Berkeley by | * This code is derived from software contributed to Berkeley by | ||||
* Christos Zoulas of Cornell University. | * Christos Zoulas of Cornell University. | ||||
* | * | ||||
Show All 22 Lines | |||||
* SUCH DAMAGE. | * SUCH DAMAGE. | ||||
*/ | */ | ||||
#include "config.h" | #include "config.h" | ||||
#if !defined(lint) && !defined(SCCSID) | #if !defined(lint) && !defined(SCCSID) | ||||
#if 0 | #if 0 | ||||
static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93"; | static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93"; | ||||
#else | #else | ||||
__RCSID("$NetBSD: tokenizer.c,v 1.24 2016/02/17 19:47:49 christos Exp $"); | __RCSID("$NetBSD: tokenizer.c,v 1.28 2016/04/11 18:56:31 christos Exp $"); | ||||
#endif | #endif | ||||
#endif /* not lint && not SCCSID */ | #endif /* not lint && not SCCSID */ | ||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
/* We build this file twice, once as NARROW, once as WIDE. */ | /* We build this file twice, once as NARROW, once as WIDE. */ | ||||
/* | /* | ||||
* tokenize.c: Bourne shell like tokenizer | * tokenize.c: Bourne shell like tokenizer | ||||
*/ | */ | ||||
#include <stdlib.h> | #include <stdlib.h> | ||||
#include <string.h> | #include <string.h> | ||||
#include "histedit.h" | #include "histedit.h" | ||||
#include "chartype.h" | |||||
typedef enum { | typedef enum { | ||||
Q_none, Q_single, Q_double, Q_one, Q_doubleone | Q_none, Q_single, Q_double, Q_one, Q_doubleone | ||||
} quote_t; | } quote_t; | ||||
#define TOK_KEEP 1 | #define TOK_KEEP 1 | ||||
#define TOK_EAT 2 | #define TOK_EAT 2 | ||||
#define WINCR 20 | #define WINCR 20 | ||||
#define AINCR 10 | #define AINCR 10 | ||||
#define IFS STR("\t \n") | #define IFS STR("\t \n") | ||||
#define tok_malloc(a) malloc(a) | #define tok_malloc(a) malloc(a) | ||||
#define tok_free(a) free(a) | #define tok_free(a) free(a) | ||||
#define tok_realloc(a, b) realloc(a, b) | #define tok_realloc(a, b) realloc(a, b) | ||||
#define tok_strdup(a) Strdup(a) | |||||
#ifdef NARROWCHAR | |||||
#define Char char | |||||
#define FUN(prefix, rest) prefix ## _ ## rest | |||||
#define TYPE(type) type | |||||
#define STR(x) x | |||||
#define Strchr(s, c) strchr(s, c) | |||||
#define tok_strdup(s) strdup(s) | |||||
#else | |||||
#define Char wchar_t | |||||
#define FUN(prefix, rest) prefix ## _w ## rest | |||||
#define TYPE(type) type ## W | |||||
#define STR(x) L ## x | |||||
#define Strchr(s, c) wcschr(s, c) | |||||
#define tok_strdup(s) wcsdup(s) | |||||
#endif | |||||
struct TYPE(tokenizer) { | struct TYPE(tokenizer) { | ||||
Char *ifs; /* In field separator */ | Char *ifs; /* In field separator */ | ||||
size_t argc, amax; /* Current and maximum number of args */ | size_t argc, amax; /* Current and maximum number of args */ | ||||
Char **argv; /* Argument list */ | Char **argv; /* Argument list */ | ||||
Char *wptr, *wmax; /* Space and limit on the word buffer */ | Char *wptr, *wmax; /* Space and limit on the word buffer */ | ||||
Char *wstart; /* Beginning of next word */ | Char *wstart; /* Beginning of next word */ | ||||
Char *wspace; /* Space of word buffer */ | Char *wspace; /* Space of word buffer */ | ||||
quote_t quote; /* Quoting state */ | quote_t quote; /* Quoting state */ | ||||
int flags; /* flags; */ | int flags; /* flags; */ | ||||
}; | }; | ||||
private void FUN(tok,finish)(TYPE(Tokenizer) *); | static void FUN(tok,finish)(TYPE(Tokenizer) *); | ||||
/* FUN(tok,finish)(): | /* FUN(tok,finish)(): | ||||
* Finish a word in the tokenizer. | * Finish a word in the tokenizer. | ||||
*/ | */ | ||||
private void | static void | ||||
FUN(tok,finish)(TYPE(Tokenizer) *tok) | FUN(tok,finish)(TYPE(Tokenizer) *tok) | ||||
{ | { | ||||
*tok->wptr = '\0'; | *tok->wptr = '\0'; | ||||
if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) { | if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) { | ||||
tok->argv[tok->argc++] = tok->wstart; | tok->argv[tok->argc++] = tok->wstart; | ||||
tok->argv[tok->argc] = NULL; | tok->argv[tok->argc] = NULL; | ||||
tok->wstart = ++tok->wptr; | tok->wstart = ++tok->wptr; | ||||
} | } | ||||
tok->flags &= ~TOK_KEEP; | tok->flags &= ~TOK_KEEP; | ||||
} | } | ||||
/* FUN(tok,init)(): | /* FUN(tok,init)(): | ||||
* Initialize the tokenizer | * Initialize the tokenizer | ||||
*/ | */ | ||||
public TYPE(Tokenizer) * | TYPE(Tokenizer) * | ||||
FUN(tok,init)(const Char *ifs) | FUN(tok,init)(const Char *ifs) | ||||
{ | { | ||||
TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok)); | TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok)); | ||||
if (tok == NULL) | if (tok == NULL) | ||||
return NULL; | return NULL; | ||||
tok->ifs = tok_strdup(ifs ? ifs : IFS); | tok->ifs = tok_strdup(ifs ? ifs : IFS); | ||||
if (tok->ifs == NULL) { | if (tok->ifs == NULL) { | ||||
Show All 24 Lines | FUN(tok,init)(const Char *ifs) | ||||
return tok; | return tok; | ||||
} | } | ||||
/* FUN(tok,reset)(): | /* FUN(tok,reset)(): | ||||
* Reset the tokenizer | * Reset the tokenizer | ||||
*/ | */ | ||||
public void | void | ||||
FUN(tok,reset)(TYPE(Tokenizer) *tok) | FUN(tok,reset)(TYPE(Tokenizer) *tok) | ||||
{ | { | ||||
tok->argc = 0; | tok->argc = 0; | ||||
tok->wstart = tok->wspace; | tok->wstart = tok->wspace; | ||||
tok->wptr = tok->wspace; | tok->wptr = tok->wspace; | ||||
tok->flags = 0; | tok->flags = 0; | ||||
tok->quote = Q_none; | tok->quote = Q_none; | ||||
} | } | ||||
/* FUN(tok,end)(): | /* FUN(tok,end)(): | ||||
* Clean up | * Clean up | ||||
*/ | */ | ||||
public void | void | ||||
FUN(tok,end)(TYPE(Tokenizer) *tok) | FUN(tok,end)(TYPE(Tokenizer) *tok) | ||||
{ | { | ||||
tok_free(tok->ifs); | tok_free(tok->ifs); | ||||
tok_free(tok->wspace); | tok_free(tok->wspace); | ||||
tok_free(tok->argv); | tok_free(tok->argv); | ||||
tok_free(tok); | tok_free(tok); | ||||
} | } | ||||
Show All 12 Lines | |||||
* 1 Unmatched single quote | * 1 Unmatched single quote | ||||
* 0 Ok | * 0 Ok | ||||
* Modifies (if return value is 0): | * Modifies (if return value is 0): | ||||
* argc number of arguments | * argc number of arguments | ||||
* argv argument array | * argv argument array | ||||
* cursorc if !NULL, argv element containing cursor | * cursorc if !NULL, argv element containing cursor | ||||
* cursorv if !NULL, offset in argv[cursorc] of cursor | * cursorv if !NULL, offset in argv[cursorc] of cursor | ||||
*/ | */ | ||||
public int | int | ||||
FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line, | FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line, | ||||
int *argc, const Char ***argv, int *cursorc, int *cursoro) | int *argc, const Char ***argv, int *cursorc, int *cursoro) | ||||
{ | { | ||||
const Char *ptr; | const Char *ptr; | ||||
int cc, co; | int cc, co; | ||||
cc = co = -1; | cc = co = -1; | ||||
ptr = line->buffer; | ptr = line->buffer; | ||||
▲ Show 20 Lines • Show All 234 Lines • ▼ Show 20 Lines | tok_line_outok: | ||||
*argc = (int)tok->argc; | *argc = (int)tok->argc; | ||||
return 0; | return 0; | ||||
} | } | ||||
/* FUN(tok,str)(): | /* FUN(tok,str)(): | ||||
* Simpler version of tok_line, taking a NUL terminated line | * Simpler version of tok_line, taking a NUL terminated line | ||||
* and splitting into words, ignoring cursor state. | * and splitting into words, ignoring cursor state. | ||||
*/ | */ | ||||
public int | int | ||||
FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc, | FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc, | ||||
const Char ***argv) | const Char ***argv) | ||||
{ | { | ||||
TYPE(LineInfo) li; | TYPE(LineInfo) li; | ||||
memset(&li, 0, sizeof(li)); | memset(&li, 0, sizeof(li)); | ||||
li.buffer = line; | li.buffer = line; | ||||
li.cursor = li.lastchar = Strchr(line, '\0'); | li.cursor = li.lastchar = Strchr(line, '\0'); | ||||
return FUN(tok,line)(tok, &li, argc, argv, NULL, NULL); | return FUN(tok,line)(tok, &li, argc, argv, NULL, NULL); | ||||
} | } |