Index: head/contrib/libucl/CMakeLists.txt =================================================================== --- head/contrib/libucl/CMakeLists.txt (revision 306543) +++ head/contrib/libucl/CMakeLists.txt (revision 306544) @@ -1,260 +1,259 @@ PROJECT(libucl C) CMAKE_MINIMUM_REQUIRED(VERSION 2.6.0 FATAL_ERROR) SET(LIBUCL_VERSION_MAJOR 0) SET(LIBUCL_VERSION_MINOR 5) SET(LIBUCL_VERSION_PATCH 0) SET(LIBUCL_VERSION "${LIBUCL_VERSION_MAJOR}.${LIBUCL_VERSION_MINOR}.${LIBUCL_VERSION_PATCH}") INCLUDE(CheckCCompilerFlag) INCLUDE(FindOpenSSL) OPTION(ENABLE_URL_INCLUDE "Enable urls in ucl includes (requires libcurl or libfetch) [default: OFF]" OFF) OPTION(ENABLE_URL_SIGN "Enable signatures check in ucl includes (requires openssl) [default: OFF]" OFF) OPTION(BUILD_SHARED_LIBS "Build Shared Libraries [default: OFF]" OFF) OPTION(ENABLE_LUA "Enable lua support [default: OFF]" OFF) OPTION(ENABLE_LUAJIT "Enable luajit support [default: OFF]" OFF) # Find lua installation MACRO(FindLua) # Find lua libraries UNSET(LUA_INCLUDE_DIR CACHE) UNSET(LUA_LIBRARY CACHE) CMAKE_PARSE_ARGUMENTS(LUA "" "VERSION_MAJOR;VERSION_MINOR;ROOT" "" ${ARGN}) IF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) MESSAGE(FATAL_ERROR "Invalid FindLua invocation: ${ARGN}") ENDIF() IF(ENABLE_LUAJIT MATCHES "ON") MESSAGE(STATUS "Check for luajit ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") FIND_PATH(LUA_INCLUDE_DIR luajit.h HINTS "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" $ENV{LUA_DIR} PATH_SUFFIXES "include/luajit-2.0" "include/luajit${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" "include/luajit${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" "include/luajit-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" "include/luajit" "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" include/lua include PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} ) FIND_LIBRARY(LUA_LIBRARY NAMES luajit "luajit-2.0" "luajit2.0" "luajit${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" "luajit${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" "luajit-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" HINTS "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" $ENV{LUA_DIR} PATH_SUFFIXES lib64 lib PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} DOC "Lua library" ) IF(NOT LUA_LIBRARY OR NOT LUA_INCLUDE_DIR) MESSAGE(STATUS "Fallback from luajit to plain lua") SET(ENABLE_LUAJIT "OFF") MESSAGE(STATUS "Check for lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") FIND_PATH(LUA_INCLUDE_DIR lua.h HINTS "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" $ENV{LUA_DIR} PATH_SUFFIXES "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" include/lua include PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} ) FIND_LIBRARY(LUA_LIBRARY NAMES lua "lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" "lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" "lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" HINTS "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" $ENV{LUA_DIR} PATH_SUFFIXES lib64 lib PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} DOC "Lua library" ) ENDIF() ELSE(ENABLE_LUAJIT MATCHES "ON") MESSAGE(STATUS "Check for lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") FIND_PATH(LUA_INCLUDE_DIR lua.h HINTS "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" $ENV{LUA_DIR} PATH_SUFFIXES "include/lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" "include/lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" "include/lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" include/lua include PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS} ) FIND_LIBRARY(LUA_LIBRARY NAMES lua "lua${LUA_VERSION_MAJOR}${LUA_VERSION_MINOR}" "lua${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" "lua-${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}" HINTS "${RSPAMD_SEARCH_PATH}" "${LUA_ROOT}" $ENV{LUA_DIR} PATH_SUFFIXES lib64 lib PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS} DOC "Lua library" ) ENDIF(ENABLE_LUAJIT MATCHES "ON") IF(LUA_LIBRARY AND LUA_INCLUDE_DIR) SET(LUA_FOUND 1) IF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) SET(LUA_VERSION_MAJOR ${LUA_VERSION_MAJOR}) SET(LUA_VERSION_MINOR ${LUA_VERSION_MINOR}) ENDIF(NOT LUA_VERSION_MAJOR OR NOT LUA_VERSION_MINOR) IF(ENABLE_LUAJIT MATCHES "ON") MESSAGE(STATUS "Found luajit ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") ELSE(ENABLE_LUAJIT MATCHES "ON") MESSAGE(STATUS "Found lua ${LUA_VERSION_MAJOR}.${LUA_VERSION_MINOR}") ENDIF(ENABLE_LUAJIT MATCHES "ON") ENDIF(LUA_LIBRARY AND LUA_INCLUDE_DIR) ENDMACRO() IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt) ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") IF(ENABLE_URL_INCLUDE MATCHES "ON") FIND_LIBRARY(LIBFETCH_LIBRARY NAMES fetch PATHS PATH_SUFFIXES lib64 lib PATHS ~/Library/Frameworks /Library/Frameworks /usr/local /usr /sw /opt/local /opt/csw /opt DOC "Path where the libfetch library can be found") IF(LIBFETCH_LIBRARY) FIND_FILE(HAVE_FETCH_H NAMES fetch.h PATHS /usr/include /opt/include /usr/local/include DOC "Path to libfetch header") ELSE(LIBFETCH_LIBRARY) # Try to find libcurl ProcessPackage(CURL libcurl) IF(NOT CURL_FOUND) MESSAGE(WARNING "Neither libcurl nor libfetch were found, no support of URL includes in configuration") ENDIF(NOT CURL_FOUND) ENDIF(LIBFETCH_LIBRARY) ENDIF(ENABLE_URL_INCLUDE MATCHES "ON") SET(CMAKE_C_WARN_FLAGS "") CHECK_C_COMPILER_FLAG(-Wall SUPPORT_WALL) CHECK_C_COMPILER_FLAG(-W SUPPORT_W) CHECK_C_COMPILER_FLAG(-Wno-unused-parameter SUPPORT_WPARAM) CHECK_C_COMPILER_FLAG(-Wno-pointer-sign SUPPORT_WPOINTER_SIGN) CHECK_C_COMPILER_FLAG(-Wstrict-prototypes SUPPORT_WSTRICT_PROTOTYPES) IF(NOT "${CMAKE_C_COMPILER_ID}" MATCHES SunPro) CHECK_C_COMPILER_FLAG("-std=c99" SUPPORT_STD_FLAG) ENDIF(NOT "${CMAKE_C_COMPILER_ID}" MATCHES SunPro) IF(SUPPORT_W) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -W") ENDIF(SUPPORT_W) IF(SUPPORT_WALL) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wall") ENDIF(SUPPORT_WALL) IF(SUPPORT_WPARAM) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-unused-parameter") ENDIF(SUPPORT_WPARAM) IF(SUPPORT_WPOINTER_SIGN) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-pointer-sign") ENDIF(SUPPORT_WPOINTER_SIGN) IF(SUPPORT_WSTRICT_PROTOTYPES) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wstrict-prototypes") ENDIF(SUPPORT_WSTRICT_PROTOTYPES) IF(SUPPORT_STD_FLAG) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -std=c99") ENDIF(SUPPORT_STD_FLAG) IF(ENABLE_URL_SIGN MATCHES "ON") IF(OPENSSL_FOUND) SET(HAVE_OPENSSL 1) INCLUDE_DIRECTORIES("${OPENSSL_INCLUDE_DIR}") ENDIF(OPENSSL_FOUND) ENDIF(ENABLE_URL_SIGN MATCHES "ON") INCLUDE_DIRECTORIES("src") INCLUDE_DIRECTORIES("include") INCLUDE_DIRECTORIES("uthash") INCLUDE_DIRECTORIES("klib") SET(UCLSRC src/ucl_util.c src/ucl_parser.c src/ucl_emitter.c src/ucl_emitter_streamline.c src/ucl_emitter_utils.c src/ucl_hash.c src/ucl_schema.c src/ucl_msgpack.c - src/ucl_sexp.c - src/xxhash.c) + src/ucl_sexp.c) SET (LIB_TYPE STATIC) IF (BUILD_SHARED_LIBS) SET (LIB_TYPE SHARED) ENDIF (BUILD_SHARED_LIBS) ADD_LIBRARY(ucl ${LIB_TYPE} ${UCLSRC}) SET_TARGET_PROPERTIES(ucl PROPERTIES VERSION ${LIBUCL_VERSION} SOVERSION ${LIBUCL_VERSION_MAJOR}) IF(ENABLE_LUA MATCHES "ON") IF(ENABLE_LUAJIT MATCHES "ON") FindLua(VERSION_MAJOR "5" VERSION_MINOR "1" ROOT "${LUA_ROOT}") IF(NOT LUA_FOUND) MESSAGE(FATAL_ERROR "Lua not found, lua support is required") ELSE(NOT LUA_FOUND) INCLUDE_DIRECTORIES("${LUA_INCLUDE_DIR}") ENDIF(NOT LUA_FOUND) ELSE(ENABLE_LUAJIT MATCHES "ON") FindLua(VERSION_MAJOR "5" VERSION_MINOR "2" ROOT "${LUA_ROOT}") IF(NOT LUA_FOUND) FindLua(VERSION_MAJOR "5" VERSION_MINOR "1" ROOT "${LUA_ROOT}") ENDIF(NOT LUA_FOUND) IF(NOT LUA_FOUND) MESSAGE(FATAL_ERROR "Lua not found, lua support is required") ELSE(NOT LUA_FOUND) INCLUDE_DIRECTORIES("${LUA_INCLUDE_DIR}") ENDIF(NOT LUA_FOUND) ENDIF(ENABLE_LUAJIT MATCHES "ON") SET(UCL_LUA_SRC lua/lua_ucl.c) ADD_LIBRARY(lua-ucl ${LIB_TYPE} ${UCL_LUA_SRC}) IF(ENABLE_LUAJIT MATCHES "ON") TARGET_LINK_LIBRARIES(lua-ucl "${LUAJIT_LIBRARY}") ELSE(ENABLE_LUAJIT MATCHES "ON") TARGET_LINK_LIBRARIES(lua-ucl "${LUA_LIBRARY}") ENDIF(ENABLE_LUAJIT MATCHES "ON") TARGET_LINK_LIBRARIES(lua-ucl ucl) SET_TARGET_PROPERTIES(lua-ucl PROPERTIES VERSION ${LIBUCL_VERSION} SOVERSION ${LIBUCL_VERSION_MAJOR}) ENDIF() IF(HAVE_FETCH_H) TARGET_LINK_LIBRARIES(ucl fetch) ELSE(HAVE_FETCH_H) IF(CURL_FOUND) TARGET_LINK_LIBRARIES(ucl ${CURL_LIBRARIES}) ENDIF(CURL_FOUND) ENDIF(HAVE_FETCH_H) IF(ENABLE_URL_SIGN MATCHES "ON") IF(OPENSSL_FOUND) TARGET_LINK_LIBRARIES(ucl ${OPENSSL_LIBRARIES}) ENDIF(OPENSSL_FOUND) ENDIF(ENABLE_URL_SIGN MATCHES "ON") Index: head/contrib/libucl/Makefile.unix =================================================================== --- head/contrib/libucl/Makefile.unix (revision 306543) +++ head/contrib/libucl/Makefile.unix (revision 306544) @@ -1,92 +1,89 @@ CC ?= gcc DESTDIR ?= /usr/local LD ?= gcc -C_COMMON_FLAGS ?= -fPIC -Wall -W -Wno-unused-parameter -Wno-pointer-sign -I./include -I./uthash -I./src +C_COMMON_FLAGS ?= -fPIC -Wall -W -Wno-unused-parameter -Wno-pointer-sign -I./include -I./uthash -I./src -I./klib MAJOR_VERSION = 0 MINOR_VERSION = 2 PATCH_VERSION = 9 VERSION = "$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)" SONAME = libucl.so SONAME_FULL = $(SONAME).$(MAJOR_VERSION) OBJDIR ?= .obj TESTDIR ?= tests SRCDIR ?= src INCLUDEDIR ?= include MKDIR ?= mkdir INSTALL ?= install RM ?= rm RMDIR ?= rmdir LN ?= ln LD_SHARED_FLAGS ?= -Wl,-soname,$(SONAME) -shared -lm LD_UCL_FLAGS ?= -L$(OBJDIR) -Wl,-rpath,$(OBJDIR) -lucl LD_ADD ?= -lrt COPT_FLAGS ?= -O2 HDEPS = $(SRCDIR)/ucl_hash.h \ $(SRCDIR)/ucl_chartable.h \ $(SRCDIR)/ucl_internal.h \ $(INCLUDEDIR)/ucl.h \ - $(SRCDIR)/xxhash.h + $(SRCDIR)/mum.h OBJECTS = $(OBJDIR)/ucl_hash.o \ $(OBJDIR)/ucl_util.o \ $(OBJDIR)/ucl_parser.o \ $(OBJDIR)/ucl_emitter.o \ - $(OBJDIR)/ucl_schema.o \ - $(OBJDIR)/xxhash.o + $(OBJDIR)/ucl_schema.o all: $(OBJDIR) $(OBJDIR)/$(SONAME) $(OBJDIR)/$(SONAME): $(OBJDIR)/$(SONAME_FULL) $(LN) -sf $(SONAME_FULL) $(OBJDIR)/$(SONAME) $(OBJDIR)/$(SONAME_FULL): $(OBJECTS) $(CC) -o $(OBJDIR)/$(SONAME_FULL) $(OBJECTS) $(LD_SHARED_FLAGS) $(LDFLAGS) $(SSL_LIBS) $(FETCH_LIBS) $(OBJDIR): @$(MKDIR) -p $(OBJDIR) # Compile rules $(OBJDIR)/ucl_util.o: $(SRCDIR)/ucl_util.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_util.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_util.c $(OBJDIR)/ucl_parser.o: $(SRCDIR)/ucl_parser.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_parser.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_parser.c $(OBJDIR)/ucl_emitter.o: $(SRCDIR)/ucl_emitter.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_emitter.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_emitter.c $(OBJDIR)/ucl_hash.o: $(SRCDIR)/ucl_hash.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_hash.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_hash.c $(OBJDIR)/ucl_schema.o: $(SRCDIR)/ucl_schema.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_schema.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_schema.c -$(OBJDIR)/xxhash.o: $(SRCDIR)/xxhash.c $(HDEPS) - $(CC) -o $(OBJDIR)/xxhash.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/xxhash.c clean: $(RM) $(OBJDIR)/*.o $(OBJDIR)/$(SONAME_FULL) $(OBJDIR)/$(SONAME) $(OBJDIR)/chargen $(OBJDIR)/test_basic $(OBJDIR)/test_speed $(OBJDIR)/objdump $(OBJDIR)/test_generate $(OBJDIR)/test_schema || true $(RMDIR) $(OBJDIR) # Utils chargen: utils/chargen.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/chargen $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) utils/chargen.c objdump: utils/objdump.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/objdump $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) utils/objdump.c $(LD_UCL_FLAGS) # Tests test: $(OBJDIR) $(OBJDIR)/$(SONAME) $(OBJDIR)/test_basic $(OBJDIR)/test_speed $(OBJDIR)/test_generate $(OBJDIR)/test_schema run-test: test TEST_DIR=$(TESTDIR) $(TESTDIR)/run_tests.sh $(OBJDIR)/test_basic $(OBJDIR)/test_speed $(OBJDIR)/test_generate $(OBJDIR)/test_schema $(OBJDIR)/test_basic: $(TESTDIR)/test_basic.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/test_basic $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) $(TESTDIR)/test_basic.c $(LD_UCL_FLAGS) $(OBJDIR)/test_schema: $(TESTDIR)/test_schema.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/test_schema $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) $(TESTDIR)/test_schema.c $(LD_UCL_FLAGS) $(OBJDIR)/test_speed: $(TESTDIR)/test_speed.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/test_speed $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) $(TESTDIR)/test_speed.c $(LD_UCL_FLAGS) $(LD_ADD) $(OBJDIR)/test_generate: $(TESTDIR)/test_generate.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/test_generate $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) $(TESTDIR)/test_generate.c $(LD_UCL_FLAGS) $(LD_ADD) install: $(OBJDIR)/$(SONAME) $(INSTALL) -m0755 $(SONAME) $(DESTDIR)/lib/$(SONAME) $(INSTALL) -m0644 include/ucl.h $(DESTDIR)/include/ucl.h .PHONY: clean $(OBJDIR) Index: head/contrib/libucl/Makefile.w32 =================================================================== --- head/contrib/libucl/Makefile.w32 (revision 306543) +++ head/contrib/libucl/Makefile.w32 (revision 306544) @@ -1,93 +1,92 @@ CC ?= gcc DESTDIR ?= /usr/local LD ?= gcc C_COMMON_FLAGS ?= -fPIC -Wall -W -Wno-unused-parameter -Wno-pointer-sign -I./include -I./uthash -I./src MAJOR_VERSION = 0 MINOR_VERSION = 2 PATCH_VERSION = 9 VERSION = "$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)" SONAME = libucl.dll OBJDIR ?= .obj TESTDIR ?= tests SRCDIR ?= src INCLUDEDIR ?= include MKDIR ?= mkdir INSTALL ?= install RM ?= rm RMDIR ?= rmdir ifeq (Windows_NT, $(OS)) LN ?= ln else LN ?= rem ln endif LD_SHARED_FLAGS ?= -Wl,-soname,$(SONAME) -shared -lm LD_UCL_FLAGS ?= -L$(OBJDIR) -Wl,-rpath,$(OBJDIR) -lucl LD_ADD ?= -lrt COPT_FLAGS ?= -O2 HDEPS = $(SRCDIR)/ucl_hash.h \ $(SRCDIR)/ucl_chartable.h \ $(SRCDIR)/ucl_internal.h \ $(INCLUDEDIR)/ucl.h \ - $(SRCDIR)/xxhash.h + $(SRCDIR)/mum.h OBJECTS = $(OBJDIR)/ucl_hash.o \ $(OBJDIR)/ucl_util.o \ $(OBJDIR)/ucl_parser.o \ $(OBJDIR)/ucl_emitter.o \ $(OBJDIR)/ucl_emitter_utils.o \ - $(OBJDIR)/ucl_schema.o \ - $(OBJDIR)/xxhash.o + $(OBJDIR)/ucl_schema.o all: $(OBJDIR) $(OBJDIR)/$(SONAME) $(OBJDIR)/$(SONAME): $(OBJECTS) $(CC) -o $(OBJDIR)/$(SONAME) $(OBJECTS) $(LD_SHARED_FLAGS) $(LDFLAGS) $(SSL_LIBS) $(FETCH_LIBS) $(OBJDIR): @$(MKDIR) -p $(OBJDIR) # Compile rules $(OBJDIR)/ucl_util.o: $(SRCDIR)/ucl_util.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_util.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_util.c $(OBJDIR)/ucl_parser.o: $(SRCDIR)/ucl_parser.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_parser.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_parser.c $(OBJDIR)/ucl_emitter.o: $(SRCDIR)/ucl_emitter.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_emitter.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_emitter.c $(OBJDIR)/ucl_emitter_utils.o: $(SRCDIR)/ucl_emitter_utils.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_emitter_utils.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_emitter_utils.c $(OBJDIR)/ucl_hash.o: $(SRCDIR)/ucl_hash.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_hash.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_hash.c $(OBJDIR)/ucl_schema.o: $(SRCDIR)/ucl_schema.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_schema.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_schema.c $(OBJDIR)/xxhash.o: $(SRCDIR)/xxhash.c $(HDEPS) $(CC) -o $(OBJDIR)/xxhash.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/xxhash.c clean: $(RM) $(OBJDIR)/*.o $(OBJDIR)/$(SONAME) $(OBJDIR)/$(SONAME) $(OBJDIR)/chargen $(OBJDIR)/test_basic $(OBJDIR)/test_speed $(OBJDIR)/objdump $(OBJDIR)/test_generate $(RMDIR) $(OBJDIR) # Utils chargen: utils/chargen.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/chargen $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) utils/chargen.c objdump: utils/objdump.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/objdump $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) utils/objdump.c $(LD_UCL_FLAGS) # Tests test: $(OBJDIR) $(OBJDIR)/$(SONAME) $(OBJDIR)/test_basic $(OBJDIR)/test_speed $(OBJDIR)/test_generate run-test: test TEST_DIR=$(TESTDIR) $(TESTDIR)/run_tests.sh $(OBJDIR)/test_basic $(OBJDIR)/test_speed $(OBJDIR)/test_generate $(OBJDIR)/test_basic: $(TESTDIR)/test_basic.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/test_basic $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) $(TESTDIR)/test_basic.c $(LD_UCL_FLAGS) $(OBJDIR)/test_speed: $(TESTDIR)/test_speed.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/test_speed $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) $(TESTDIR)/test_speed.c $(LD_UCL_FLAGS) $(LD_ADD) $(OBJDIR)/test_generate: $(TESTDIR)/test_generate.c $(OBJDIR)/$(SONAME) $(CC) -o $(OBJDIR)/test_generate $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) $(LDFLAGS) $(TESTDIR)/test_generate.c $(LD_UCL_FLAGS) $(LD_ADD) install: $(OBJDIR)/$(SONAME) $(INSTALL) -m0755 $(SONAME) $(DESTDIR)/lib/$(SONAME) $(INSTALL) -m0644 include/ucl.h $(DESTDIR)/include/ucl.h .PHONY: clean $(OBJDIR) Index: head/contrib/libucl/README.md =================================================================== --- head/contrib/libucl/README.md (revision 306543) +++ head/contrib/libucl/README.md (revision 306544) @@ -1,382 +1,382 @@ # LIBUCL [![Build Status](https://travis-ci.org/vstakhov/libucl.svg?branch=master)](https://travis-ci.org/vstakhov/libucl)[![Coverity](https://scan.coverity.com/projects/4138/badge.svg)](https://scan.coverity.com/projects/4138)[![Coverage Status](https://coveralls.io/repos/github/vstakhov/libucl/badge.svg?branch=master)](https://coveralls.io/github/vstakhov/libucl?branch=master) **Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* - [Introduction](#introduction) - [Basic structure](#basic-structure) - [Improvements to the json notation](#improvements-to-the-json-notation) - [General syntax sugar](#general-syntax-sugar) - [Automatic arrays creation](#automatic-arrays-creation) - [Named keys hierarchy](#named-keys-hierarchy) - [Convenient numbers and booleans](#convenient-numbers-and-booleans) - [General improvements](#general-improvements) - - [Commments](#commments) + - [Comments](#comments) - [Macros support](#macros-support) - [Variables support](#variables-support) - [Multiline strings](#multiline-strings) - [Emitter](#emitter) - [Validation](#validation) - [Performance](#performance) - [Conclusion](#conclusion) -## Introduction +## Introduction This document describes the main features and principles of the configuration language called `UCL` - universal configuration language. If you are looking for the libucl API documentation you can find it at [this page](doc/api.md). ## Basic structure UCL is heavily infused by `nginx` configuration as the example of a convenient configuration system. However, UCL is fully compatible with `JSON` format and is able to parse json files. For example, you can write the same configuration in the following ways: * in nginx like: ```nginx param = value; section { param = value; param1 = value1; flag = true; number = 10k; time = 0.2s; string = "something"; subsection { host = { - host = "hostname"; + host = "hostname"; port = 900; } host = { host = "hostname"; port = 901; } } } ``` * or in JSON: ```json { "param": "value", "param1": "value1", "flag": true, "subsection": { "host": [ { "host": "hostname", "port": 900 }, { "host": "hostname", "port": 901 } ] } } ``` ## Improvements to the json notation. There are various things that make ucl configuration more convenient for editing than strict json: ### General syntax sugar * Braces are not necessary to enclose a top object: it is automatically treated as an object: ```json "key": "value" ``` is equal to: ```json {"key": "value"} ``` * There is no requirement of quotes for strings and keys, moreover, `:` may be replaced `=` or even be skipped for objects: ```nginx key = value; section { key = value; } ``` is equal to: ```json { "key": "value", "section": { "key": "value" } } ``` * No commas mess: you can safely place a comma or semicolon for the last element in an array or an object: ```json { "key1": "value", "key2": "value", } ``` ### Automatic arrays creation * Non-unique keys in an object are allowed and are automatically converted to the arrays internally: ```json { "key": "value1", "key": "value2" } ``` is converted to: ```json { "key": ["value1", "value2"] } ``` ### Named keys hierarchy UCL accepts named keys and organize them into objects hierarchy internally. Here is an example of this process: ```nginx section "blah" { key = value; } section foo { key = value; } ``` is converted to the following object: ```nginx section { blah { key = value; } foo { key = value; } } ``` - + Plain definitions may be more complex and contain more than a single level of nested objects: - + ```nginx section "blah" "foo" { key = value; } ``` is presented as: -```nginx +```nginx section { blah { foo { key = value; } } } ``` ### Convenient numbers and booleans * Numbers can have suffixes to specify standard multipliers: + `[kKmMgG]` - standard 10 base multipliers (so `1k` is translated to 1000) + `[kKmMgG]b` - 2 power multipliers (so `1kb` is translated to 1024) + `[s|min|d|w|y]` - time multipliers, all time values are translated to float number of seconds, for example `10min` is translated to 600.0 and `10ms` is translated to 0.01 * Hexadecimal integers can be used by `0x` prefix, for example `key = 0xff`. However, floating point values can use decimal base only. * Booleans can be specified as `true` or `yes` or `on` and `false` or `no` or `off`. * It is still possible to treat numbers and booleans as strings by enclosing them in double quotes. ## General improvements -### Commments +### Comments UCL supports different style of comments: -* single line: `#` +* single line: `#` * multiline: `/* ... */` Multiline comments may be nested: ```c # Sample single line comment -/* +/* some comment /* nested comment */ end of comment */ ``` ### Macros support UCL supports external macros both multiline and single line ones: ```nginx .macro "sometext"; .macro { Some long text .... }; ``` Moreover, each macro can accept an optional list of arguments in braces. These arguments themselves are the UCL object that is parsed and passed to a macro as options: ```nginx .macro(param=value) "something"; .macro(param={key=value}) "something"; .macro(.include "params.conf") "something"; .macro(#this is multiline macro param = [value1, value2]) "something"; .macro(key="()") "something"; ``` UCL also provide a convenient `include` macro to load content from another files to the current UCL object. This macro accepts either path to file: ```nginx .include "/full/path.conf" .include "./relative/path.conf" .include "${CURDIR}/path.conf" ``` or URL (if ucl is built with url support provided by either `libcurl` or `libfetch`): .include "http://example.com/file.conf" `.include` macro supports a set of options: * `try` (default: **false**) - if this option is `true` than UCL treats errors on loading of this file as non-fatal. For example, such a file can be absent but it won't stop the parsing of the top-level document. * `sign` (default: **false**) - if this option is `true` UCL loads and checks the signature for a file from path named `.sig`. Trusted public keys should be provided for UCL API after parser is created but before any configurations are parsed. * `glob` (default: **false**) - if this option is `true` UCL treats the filename as GLOB pattern and load all files that matches the specified pattern (normally the format of patterns is defined in `glob` manual page for your operating system). This option is meaningless for URL includes. * `url` (default: **true**) - allow URL includes. * `path` (default: empty) - A UCL_ARRAY of directories to search for the include file. -Search ends after the first patch, unless `glob` is true, then all matches are included. +Search ends after the first match, unless `glob` is true, then all matches are included. * `prefix` (default false) - Put included contents inside an object, instead of loading them into the root. If no `key` is provided, one is automatically generated based on each files basename() * `key` (default: ) - Key to load contents of include into. If the key already exists, it must be the correct type * `target` (default: object) - Specify if the `prefix` `key` should be an object or an array. * `priority` (default: 0) - specify priority for the include (see below). * `duplicate` (default: 'append') - specify policy of duplicates resolving: - `append` - default strategy, if we have new object of higher priority then it replaces old one, if we have new object with less priority it is ignored completely, and if we have two duplicate objects with the same priority then we have a multi-value key (implicit array) - - `merge` - if we have object or array, then new keys are merged inside, if we have a plain object then an implicit array is formed (regardeless of priorities) + - `merge` - if we have object or array, then new keys are merged inside, if we have a plain object then an implicit array is formed (regardless of priorities) - `error` - create error on duplicate keys and stop parsing - `rewrite` - always rewrite an old value with new one (ignoring priorities) Priorities are used by UCL parser to manage the policy of objects rewriting during including other files as following: * If we have two objects with the same priority then we form an implicit array * If a new object has bigger priority then we overwrite an old one * If a new object has lower priority then we ignore it By default, the priority of top-level object is set to zero (lowest priority). Currently, you can define up to 16 priorities (from 0 to 15). Includes with bigger priorities will rewrite keys from the objects with lower priorities as specified by the policy. ### Variables support UCL supports variables in input. Variables are registered by a user of the UCL parser and can be presented in the following forms: * `${VARIABLE}` * `$VARIABLE` UCL currently does not support nested variables. To escape variables one could use double dollar signs: * `$${VARIABLE}` is converted to `${VARIABLE}` * `$$VARIABLE` is converted to `$VARIABLE` However, if no valid variables are found in a string, no expansion will be performed (and `$$` thus remains unchanged). This may be a subject to change in future libucl releases. ### Multiline strings UCL can handle multiline strings as well as single line ones. It uses shell/perl like notation for such objects: ``` key = < ParserHandle +foreign import ccall "ucl_parser_add_string" ucl_parser_add_string :: ParserHandle -> CString -> CUInt -> IO Bool +foreign import ccall "ucl_parser_add_file" ucl_parser_add_file :: ParserHandle -> CString -> IO Bool +foreign import ccall "ucl_parser_get_object" ucl_parser_get_object :: ParserHandle -> UCLObjectHandle +foreign import ccall "ucl_parser_get_error" ucl_parser_get_error :: ParserHandle -> CString + +foreign import ccall "ucl_object_iterate_new" ucl_object_iterate_new :: UCLObjectHandle -> UCLIterHandle +foreign import ccall "ucl_object_iterate_safe" ucl_object_iterate_safe :: UCLIterHandle -> Bool -> UCLObjectHandle +foreign import ccall "ucl_object_type" ucl_object_type :: UCLObjectHandle -> CUInt +foreign import ccall "ucl_object_key" ucl_object_key :: UCLObjectHandle -> CString +foreign import ccall "ucl_object_toint" ucl_object_toint :: UCLObjectHandle -> CInt +foreign import ccall "ucl_object_todouble" ucl_object_todouble :: UCLObjectHandle -> CDouble +foreign import ccall "ucl_object_tostring" ucl_object_tostring :: UCLObjectHandle -> CString +foreign import ccall "ucl_object_toboolean" ucl_object_toboolean :: UCLObjectHandle -> Bool + +foreign import ccall "ucl_object_emit" ucl_object_emit :: UCLObjectHandle -> UCLEmitterType -> CString +foreign import ccall "ucl_object_emit_len" ucl_object_emit_len :: UCLObjectHandle -> UCLEmitterType -> Ptr CSize -> IO CString + +type UCL_TYPE = CUInt +ucl_OBJECT :: UCL_TYPE +ucl_OBJECT = 0 +ucl_ARRAY :: UCL_TYPE +ucl_ARRAY = 1 +ucl_INT :: UCL_TYPE +ucl_INT = 2 +ucl_FLOAT :: UCL_TYPE +ucl_FLOAT = 3 +ucl_STRING :: UCL_TYPE +ucl_STRING = 4 +ucl_BOOLEAN :: UCL_TYPE +ucl_BOOLEAN = 5 +ucl_TIME :: UCL_TYPE +ucl_TIME = 6 +ucl_USERDATA :: UCL_TYPE +ucl_USERDATA = 7 +ucl_NULL :: UCL_TYPE +ucl_NULL = 8 + +ucl_emit_json :: UCLEmitterType +ucl_emit_json = 0 +ucl_emit_json_compact :: UCLEmitterType +ucl_emit_json_compact = 1 :: UCLEmitterType +ucl_emit_msgpack :: UCLEmitterType +ucl_emit_msgpack = 4 :: UCLEmitterType + +ucl_parser_parse_string_pure :: String -> Either UCLObjectHandle ErrorString +ucl_parser_parse_string_pure s = unsafePerformIO $ do + cs <- newCString s + let p = ucl_parser_new 0x4 + didParse <- ucl_parser_add_string p cs (toEnum $ length s) + if didParse + then return $ Left $ ucl_parser_get_object p + else Right <$> peekCString ( ucl_parser_get_error p) + +ucl_parser_add_file_pure :: String -> Either UCLObjectHandle ErrorString +ucl_parser_add_file_pure s = unsafePerformIO $ do + cs <- newCString s + let p = ucl_parser_new 0x4 + didParse <- ucl_parser_add_file p cs + if didParse + then return $ Left $ ucl_parser_get_object p + else Right <$> peekCString ( ucl_parser_get_error p) + +unpack :: MSG.MessagePack a => String -> Either a ErrorString +unpack s = case ucl_parser_parse_string_pure s of + (Right err) -> Right err + (Left obj) -> case MSG.fromObject (ucl_to_msgpack_object obj) of + Nothing -> Right "MessagePack fromObject Error" + (Just a) -> Left a + +ucl_to_msgpack_object :: UCLObjectHandle -> MSG.Object +ucl_to_msgpack_object o = toMsgPackObj (ucl_object_type o) o + where + toMsgPackObj n obj + |n==ucl_OBJECT = MSG.ObjectMap $ uclObjectToVector obj + |n==ucl_ARRAY = MSG.ObjectArray undefined + |n==ucl_INT = MSG.ObjectInt $ fromEnum $ ucl_object_toint obj + |n==ucl_FLOAT = MSG.ObjectDouble $ realToFrac $ ucl_object_todouble obj + |n==ucl_STRING = MSG.ObjectStr $ T.pack $ unsafePerformIO $ peekCString $ ucl_object_tostring obj + |n==ucl_BOOLEAN = MSG.ObjectBool $ ucl_object_toboolean obj + |n==ucl_TIME = error "time undefined" + |n==ucl_USERDATA = error "userdata undefined" + |n==ucl_NULL = error "null undefined" + |otherwise = error "\"Unknown Type\" Error" + +uclObjectToVector :: UCLObjectHandle -> V.Vector (MSG.Object,MSG.Object) +uclObjectToVector o = iterateObject (ucl_object_iterate_safe iter True ) iter V.empty + where + iter = ucl_object_iterate_new o + iterateObject obj it vec = if ucl_object_type obj == ucl_NULL + then vec + else iterateObject (ucl_object_iterate_safe it True) it (V.snoc vec ( getUclKey obj , ucl_to_msgpack_object obj)) + getUclKey obj = MSG.ObjectStr $ T.pack $ unsafePerformIO $ peekCString $ ucl_object_key obj + +uclArrayToVector :: UCLObjectHandle -> V.Vector MSG.Object +uclArrayToVector o = iterateArray (ucl_object_iterate_safe iter True ) iter V.empty + where + iter = ucl_object_iterate_new o + iterateArray obj it vec = if ucl_object_type obj == ucl_NULL + then vec + else iterateArray (ucl_object_iterate_safe it True) it (V.snoc vec (ucl_to_msgpack_object obj)) + Index: head/contrib/libucl/include/ucl++.h =================================================================== --- head/contrib/libucl/include/ucl++.h (revision 306543) +++ head/contrib/libucl/include/ucl++.h (revision 306544) @@ -1,444 +1,612 @@ /* * Copyright (c) 2015, Vsevolod Stakhov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #pragma once #include +#include +#include +#include #include #include #include "ucl.h" // C++11 API inspired by json11: https://github.com/dropbox/json11/ namespace ucl { struct ucl_map_construct_t { }; constexpr ucl_map_construct_t ucl_map_construct = ucl_map_construct_t(); struct ucl_array_construct_t { }; constexpr ucl_array_construct_t ucl_array_construct = ucl_array_construct_t(); class Ucl final { private: struct ucl_deleter { void operator() (ucl_object_t *obj) { ucl_object_unref (obj); } }; static int append_char (unsigned char c, size_t nchars, void *ud) { std::string *out = reinterpret_cast(ud); out->append (nchars, (char)c); return nchars; } static int append_len (unsigned const char *str, size_t len, void *ud) { std::string *out = reinterpret_cast(ud); out->append ((const char *)str, len); return len; } static int append_int (int64_t elt, void *ud) { std::string *out = reinterpret_cast(ud); auto nstr = std::to_string (elt); out->append (nstr); return nstr.size (); } static int append_double (double elt, void *ud) { std::string *out = reinterpret_cast(ud); auto nstr = std::to_string (elt); out->append (nstr); return nstr.size (); } static struct ucl_emitter_functions default_emit_funcs() { struct ucl_emitter_functions func = { Ucl::append_char, Ucl::append_len, Ucl::append_int, Ucl::append_double, nullptr, nullptr }; return func; }; + static bool ucl_variable_getter(const unsigned char *data, size_t len, + unsigned char ** /*replace*/, size_t * /*replace_len*/, bool *need_free, void* ud) + { + *need_free = false; + + auto vars = reinterpret_cast *>(ud); + if (vars && data && len != 0) { + vars->emplace (data, data + len); + } + return false; + } + + static bool ucl_variable_replacer (const unsigned char *data, size_t len, + unsigned char **replace, size_t *replace_len, bool *need_free, void* ud) + { + *need_free = false; + + auto replacer = reinterpret_cast(ud); + if (!replacer) { + return false; + } + + std::string var_name (data, data + len); + if (!replacer->is_variable (var_name)) { + return false; + } + + std::string var_value = replacer->replace (var_name); + if (var_value.empty ()) { + return false; + } + + *replace = (unsigned char *)UCL_ALLOC (var_value.size ()); + memcpy (*replace, var_value.data (), var_value.size ()); + + *replace_len = var_value.size (); + *need_free = true; + + return true; + } + + template + static Ucl parse_with_strategy_function (C config_func, P parse_func, std::string &err) + { + auto parser = ucl_parser_new (UCL_PARSER_DEFAULT); + + config_func (parser); + + if (!parse_func (parser)) { + err.assign (ucl_parser_get_error (parser)); + ucl_parser_free (parser); + + return nullptr; + } + + auto obj = ucl_parser_get_object (parser); + ucl_parser_free (parser); + + // Obj will handle ownership + return Ucl (obj); + } + std::unique_ptr obj; public: class const_iterator { private: struct ucl_iter_deleter { void operator() (ucl_object_iter_t it) { ucl_object_iterate_free (it); } }; std::shared_ptr it; std::unique_ptr cur; public: typedef std::forward_iterator_tag iterator_category; const_iterator(const Ucl &obj) { it = std::shared_ptr(ucl_object_iterate_new (obj.obj.get()), - ucl_iter_deleter()); + ucl_iter_deleter()); cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true))); - if (!*cur) { + if (cur->type() == UCL_NULL) { it.reset (); cur.reset (); } } const_iterator() {} const_iterator(const const_iterator &other) = delete; const_iterator(const_iterator &&other) = default; ~const_iterator() {} const_iterator& operator=(const const_iterator &other) = delete; const_iterator& operator=(const_iterator &&other) = default; bool operator==(const const_iterator &other) const { if (cur && other.cur) { return cur->obj.get() == other.cur->obj.get(); } return !cur && !other.cur; } bool operator!=(const const_iterator &other) const { return !(*this == other); } const_iterator& operator++() { if (it) { cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true))); } - if (!*cur) { + if (cur && cur->type() == UCL_NULL) { it.reset (); cur.reset (); } return *this; } const Ucl& operator*() const { return *cur; } const Ucl* operator->() const { return cur.get(); } }; + struct variable_replacer { + virtual ~variable_replacer() {} + + virtual bool is_variable (const std::string &str) const + { + return !str.empty (); + } + + virtual std::string replace (const std::string &var) const = 0; + }; + // We grab ownership if get non-const ucl_object_t Ucl(ucl_object_t *other) { obj.reset (other); } // Shared ownership Ucl(const ucl_object_t *other) { obj.reset (ucl_object_ref (other)); } Ucl(const Ucl &other) { obj.reset (ucl_object_ref (other.obj.get())); } Ucl(Ucl &&other) { obj.swap (other.obj); } Ucl() noexcept { obj.reset (ucl_object_typed_new (UCL_NULL)); } Ucl(std::nullptr_t) noexcept { obj.reset (ucl_object_typed_new (UCL_NULL)); } Ucl(double value) { obj.reset (ucl_object_typed_new (UCL_FLOAT)); obj->value.dv = value; } Ucl(int64_t value) { obj.reset (ucl_object_typed_new (UCL_INT)); obj->value.iv = value; } Ucl(bool value) { obj.reset (ucl_object_typed_new (UCL_BOOLEAN)); obj->value.iv = static_cast(value); } Ucl(const std::string &value) { obj.reset (ucl_object_fromstring_common (value.data (), value.size (), UCL_STRING_RAW)); } - Ucl(const char * value) { + Ucl(const char *value) { obj.reset (ucl_object_fromstring_common (value, 0, UCL_STRING_RAW)); } // Implicit constructor: anything with a to_json() function. template - Ucl(const T & t) : Ucl(t.to_ucl()) {} + Ucl(const T &t) : Ucl(t.to_ucl()) {} // Implicit constructor: map-like objects (std::map, std::unordered_map, etc) template ::value && std::is_constructible::value, int>::type = 0> - Ucl(const M & m) { + Ucl(const M &m) { obj.reset (ucl_object_typed_new (UCL_OBJECT)); auto cobj = obj.get (); for (const auto &e : m) { ucl_object_insert_key (cobj, ucl_object_ref (e.second.obj.get()), e.first.data (), e.first.size (), true); } } // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc) template ::value, int>::type = 0> - Ucl(const V & v) { + Ucl(const V &v) { obj.reset (ucl_object_typed_new (UCL_ARRAY)); auto cobj = obj.get (); for (const auto &e : v) { ucl_array_append (cobj, ucl_object_ref (e.obj.get())); } } ucl_type_t type () const { if (obj) { return ucl_object_type (obj.get ()); } return UCL_NULL; } const std::string key () const { std::string res; if (obj->key) { res.assign (obj->key, obj->keylen); } return res; } double number_value (const double default_val = 0.0) const { double res; if (ucl_object_todouble_safe(obj.get(), &res)) { return res; } return default_val; } int64_t int_value (const int64_t default_val = 0) const { int64_t res; if (ucl_object_toint_safe(obj.get(), &res)) { return res; } return default_val; } bool bool_value (const bool default_val = false) const { bool res; if (ucl_object_toboolean_safe(obj.get(), &res)) { return res; } return default_val; } const std::string string_value (const std::string& default_val = "") const { const char* res = nullptr; if (ucl_object_tostring_safe(obj.get(), &res)) { return res; } return default_val; } const Ucl at (size_t i) const { if (type () == UCL_ARRAY) { return Ucl (ucl_array_find_index (obj.get(), i)); } return Ucl (nullptr); } const Ucl lookup (const std::string &key) const { if (type () == UCL_OBJECT) { return Ucl (ucl_object_lookup_len (obj.get(), key.data (), key.size ())); } return Ucl (nullptr); } inline const Ucl operator[] (size_t i) const { return at(i); } inline const Ucl operator[](const std::string &key) const { return lookup(key); } // Serialize. void dump (std::string &out, ucl_emitter_t type = UCL_EMIT_JSON) const { struct ucl_emitter_functions cbdata; cbdata = Ucl::default_emit_funcs(); cbdata.ud = reinterpret_cast(&out); ucl_object_emit_full (obj.get(), type, &cbdata, nullptr); } std::string dump (ucl_emitter_t type = UCL_EMIT_JSON) const { std::string out; dump (out, type); return out; } - static Ucl parse (const std::string & in, std::string & err) + static Ucl parse (const std::string &in, std::string &err) { - auto parser = ucl_parser_new (UCL_PARSER_DEFAULT); + return parse (in, std::map(), err); + } - if (!ucl_parser_add_chunk (parser, (const unsigned char *)in.data (), - in.size ())) { - err.assign (ucl_parser_get_error (parser)); - ucl_parser_free (parser); + static Ucl parse (const std::string &in, const std::map &vars, std::string &err) + { + auto config_func = [&vars] (ucl_parser *parser) { + for (const auto & item : vars) { + ucl_parser_register_variable (parser, item.first.c_str (), item.second.c_str ()); + } + }; + auto parse_func = [&in] (ucl_parser *parser) { + return ucl_parser_add_chunk (parser, (unsigned char *)in.data (), in.size ()); + }; + + return parse_with_strategy_function (config_func, parse_func, err); + } + + static Ucl parse (const std::string &in, const variable_replacer &replacer, std::string &err) + { + auto config_func = [&replacer] (ucl_parser *parser) { + ucl_parser_set_variables_handler (parser, ucl_variable_replacer, + &const_cast(replacer)); + }; + + auto parse_func = [&in] (ucl_parser *parser) { + return ucl_parser_add_chunk (parser, (unsigned char *) in.data (), in.size ()); + }; + + return parse_with_strategy_function (config_func, parse_func, err); + } + + static Ucl parse (const char *in, std::string &err) + { + return parse (in, std::map(), err); + } + + static Ucl parse (const char *in, const std::map &vars, std::string &err) + { + if (!in) { + err = "null input"; return nullptr; } + return parse (std::string (in), vars, err); + } - auto obj = ucl_parser_get_object (parser); + static Ucl parse (const char *in, const variable_replacer &replacer, std::string &err) + { + if (!in) { + err = "null input"; + return nullptr; + } + return parse (std::string(in), replacer, err); + } + + static Ucl parse_from_file (const std::string &filename, std::string &err) + { + return parse_from_file (filename, std::map(), err); + } + + static Ucl parse_from_file (const std::string &filename, const std::map &vars, std::string &err) + { + auto config_func = [&vars] (ucl_parser *parser) { + for (const auto & item : vars) { + ucl_parser_register_variable (parser, item.first.c_str (), item.second.c_str ()); + } + }; + + auto parse_func = [&filename] (ucl_parser *parser) { + return ucl_parser_add_file (parser, filename.c_str ()); + }; + + return parse_with_strategy_function (config_func, parse_func, err); + } + + static Ucl parse_from_file (const std::string &filename, const variable_replacer &replacer, std::string &err) + { + auto config_func = [&replacer] (ucl_parser *parser) { + ucl_parser_set_variables_handler (parser, ucl_variable_replacer, + &const_cast(replacer)); + }; + + auto parse_func = [&filename] (ucl_parser *parser) { + return ucl_parser_add_file (parser, filename.c_str ()); + }; + + return parse_with_strategy_function (config_func, parse_func, err); + } + + static std::vector find_variable (const std::string &in) + { + auto parser = ucl_parser_new (UCL_PARSER_DEFAULT); + + std::set vars; + ucl_parser_set_variables_handler (parser, ucl_variable_getter, &vars); + ucl_parser_add_chunk (parser, (const unsigned char *)in.data (), in.size ()); ucl_parser_free (parser); - // Obj will handle ownership - return Ucl (obj); + std::vector result; + std::move (vars.begin (), vars.end (), std::back_inserter (result)); + return result; } - static Ucl parse (const char * in, std::string & err) + static std::vector find_variable (const char *in) { - if (in) { - return parse (std::string(in), err); - } else { - err = "null input"; - return nullptr; + if (!in) { + return std::vector(); } + return find_variable (std::string (in)); } - static Ucl parse (std::istream &ifs, std::string &err) + static std::vector find_variable_from_file (const std::string &filename) { - return Ucl::parse (std::string(std::istreambuf_iterator(ifs), - std::istreambuf_iterator()), err); + auto parser = ucl_parser_new (UCL_PARSER_DEFAULT); + + std::set vars; + ucl_parser_set_variables_handler (parser, ucl_variable_getter, &vars); + ucl_parser_add_file (parser, filename.c_str ()); + ucl_parser_free (parser); + + std::vector result; + std::move (vars.begin (), vars.end (), std::back_inserter (result)); + return std::move (result); } - Ucl& operator= (Ucl rhs) - { - obj.swap (rhs.obj); - return *this; - } + Ucl& operator= (Ucl rhs) + { + obj.swap (rhs.obj); + return *this; + } bool operator== (const Ucl &rhs) const { return ucl_object_compare (obj.get(), rhs.obj.get ()) == 0; } bool operator< (const Ucl &rhs) const { return ucl_object_compare (obj.get(), rhs.obj.get ()) < 0; } bool operator!= (const Ucl &rhs) const { return !(*this == rhs); } bool operator<= (const Ucl &rhs) const { return !(rhs < *this); } bool operator> (const Ucl &rhs) const { return (rhs < *this); } bool operator>= (const Ucl &rhs) const { return !(*this < rhs); } explicit operator bool () const { if (!obj || type() == UCL_NULL) { return false; } if (type () == UCL_BOOLEAN) { return bool_value (); } return true; } const_iterator begin() const { return const_iterator(*this); } const_iterator cbegin() const { return const_iterator(*this); } const_iterator end() const { return const_iterator(); } const_iterator cend() const { return const_iterator(); } }; }; Index: head/contrib/libucl/include/ucl.h =================================================================== --- head/contrib/libucl/include/ucl.h (revision 306543) +++ head/contrib/libucl/include/ucl.h (revision 306544) @@ -1,1444 +1,1456 @@ /* Copyright (c) 2013-2015, Vsevolod Stakhov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef UCL_H_ #define UCL_H_ #include #include #include #include #include #include #include #ifdef _WIN32 # define UCL_EXTERN __declspec(dllexport) #else # define UCL_EXTERN #endif /** * @mainpage * This is a reference manual for UCL API. You may find the description of UCL format by following this * [github repository](https://github.com/vstakhov/libucl). * * This manual has several main sections: * - @ref structures * - @ref utils * - @ref parser * - @ref emitter */ /** * @file ucl.h * @brief UCL parsing and emitting functions * * UCL is universal configuration language, which is a form of * JSON with less strict rules that make it more comfortable for * using as a configuration language */ #ifdef __cplusplus extern "C" { #endif /* * Memory allocation utilities * UCL_ALLOC(size) - allocate memory for UCL * UCL_FREE(size, ptr) - free memory of specified size at ptr * Default: malloc and free */ #ifndef UCL_ALLOC #define UCL_ALLOC(size) malloc(size) #endif #ifndef UCL_FREE #define UCL_FREE(size, ptr) free(ptr) #endif #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) #define UCL_WARN_UNUSED_RESULT \ __attribute__((warn_unused_result)) #else #define UCL_WARN_UNUSED_RESULT #endif #ifdef __GNUC__ #define UCL_DEPRECATED(func) func __attribute__ ((deprecated)) #elif defined(_MSC_VER) #define UCL_DEPRECATED(func) __declspec(deprecated) func #else #define UCL_DEPRECATED(func) func #endif /** * @defgroup structures Structures and types * UCL defines several enumeration types used for error reporting or specifying flags and attributes. * * @{ */ /** * The common error codes returned by ucl parser */ typedef enum ucl_error { UCL_EOK = 0, /**< No error */ UCL_ESYNTAX, /**< Syntax error occurred during parsing */ UCL_EIO, /**< IO error occurred during parsing */ UCL_ESTATE, /**< Invalid state machine state */ UCL_ENESTED, /**< Input has too many recursion levels */ UCL_EMACRO, /**< Error processing a macro */ UCL_EINTERNAL, /**< Internal unclassified error */ UCL_ESSL, /**< SSL error */ UCL_EMERGE /**< A merge error occured */ } ucl_error_t; /** * #ucl_object_t may have one of specified types, some types are compatible with each other and some are not. * For example, you can always convert #UCL_TIME to #UCL_FLOAT. Also you can convert #UCL_FLOAT to #UCL_INTEGER * by loosing floating point. Every object may be converted to a string by #ucl_object_tostring_forced() function. * */ typedef enum ucl_type { UCL_OBJECT = 0, /**< UCL object - key/value pairs */ UCL_ARRAY, /**< UCL array */ UCL_INT, /**< Integer number */ UCL_FLOAT, /**< Floating point number */ UCL_STRING, /**< Null terminated string */ UCL_BOOLEAN, /**< Boolean value */ UCL_TIME, /**< Time value (floating point number of seconds) */ UCL_USERDATA, /**< Opaque userdata pointer (may be used in macros) */ UCL_NULL /**< Null value */ } ucl_type_t; /** * You can use one of these types to serialise #ucl_object_t by using ucl_object_emit(). */ typedef enum ucl_emitter { UCL_EMIT_JSON = 0, /**< Emit fine formatted JSON */ UCL_EMIT_JSON_COMPACT, /**< Emit compacted JSON */ UCL_EMIT_CONFIG, /**< Emit human readable config format */ UCL_EMIT_YAML, /**< Emit embedded YAML format */ UCL_EMIT_MSGPACK, /**< Emit msgpack output */ UCL_EMIT_MAX /**< Unsupported emitter type */ } ucl_emitter_t; /** * These flags defines parser behaviour. If you specify #UCL_PARSER_ZEROCOPY you must ensure * that the input memory is not freed if an object is in use. Moreover, if you want to use * zero-terminated keys and string values then you should not use zero-copy mode, as in this case * UCL still has to perform copying implicitly. */ typedef enum ucl_parser_flags { UCL_PARSER_DEFAULT = 0, /**< No special flags */ UCL_PARSER_KEY_LOWERCASE = (1 << 0), /**< Convert all keys to lower case */ UCL_PARSER_ZEROCOPY = (1 << 1), /**< Parse input in zero-copy mode if possible */ UCL_PARSER_NO_TIME = (1 << 2), /**< Do not parse time and treat time values as strings */ UCL_PARSER_NO_IMPLICIT_ARRAYS = (1 << 3), /** Create explicit arrays instead of implicit ones */ UCL_PARSER_SAVE_COMMENTS = (1 << 4), /** Save comments in the parser context */ UCL_PARSER_DISABLE_MACRO = (1 << 5) /** Treat macros as comments */ } ucl_parser_flags_t; /** * String conversion flags, that are used in #ucl_object_fromstring_common function. */ typedef enum ucl_string_flags { UCL_STRING_RAW = 0x0, /**< Treat string as is */ UCL_STRING_ESCAPE = (1 << 0), /**< Perform JSON escape */ UCL_STRING_TRIM = (1 << 1), /**< Trim leading and trailing whitespaces */ UCL_STRING_PARSE_BOOLEAN = (1 << 2), /**< Parse passed string and detect boolean */ UCL_STRING_PARSE_INT = (1 << 3), /**< Parse passed string and detect integer number */ UCL_STRING_PARSE_DOUBLE = (1 << 4), /**< Parse passed string and detect integer or float number */ UCL_STRING_PARSE_TIME = (1 << 5), /**< Parse time strings */ UCL_STRING_PARSE_NUMBER = UCL_STRING_PARSE_INT|UCL_STRING_PARSE_DOUBLE|UCL_STRING_PARSE_TIME, /**< Parse passed string and detect number */ UCL_STRING_PARSE = UCL_STRING_PARSE_BOOLEAN|UCL_STRING_PARSE_NUMBER, /**< Parse passed string (and detect booleans and numbers) */ UCL_STRING_PARSE_BYTES = (1 << 6) /**< Treat numbers as bytes */ } ucl_string_flags_t; /** * Basic flags for an object */ typedef enum ucl_object_flags { UCL_OBJECT_ALLOCATED_KEY = (1 << 0), /**< An object has key allocated internally */ UCL_OBJECT_ALLOCATED_VALUE = (1 << 1), /**< An object has a string value allocated internally */ UCL_OBJECT_NEED_KEY_ESCAPE = (1 << 2), /**< The key of an object need to be escaped on output */ UCL_OBJECT_EPHEMERAL = (1 << 3), /**< Temporary object that does not need to be freed really */ UCL_OBJECT_MULTILINE = (1 << 4), /**< String should be displayed as multiline string */ UCL_OBJECT_MULTIVALUE = (1 << 5), /**< Object is a key with multiple values */ UCL_OBJECT_INHERITED = (1 << 6), /**< Object has been inherited from another */ UCL_OBJECT_BINARY = (1 << 7) /**< Object contains raw binary data */ } ucl_object_flags_t; /** * Duplicate policy types */ enum ucl_duplicate_strategy { UCL_DUPLICATE_APPEND = 0, /**< Default policy to merge based on priorities */ UCL_DUPLICATE_MERGE, /**< Merge new object with old one */ UCL_DUPLICATE_REWRITE, /**< Rewrite old keys */ UCL_DUPLICATE_ERROR /**< Stop parsing on duplicate found */ }; /** * Input format type */ enum ucl_parse_type { UCL_PARSE_UCL = 0, /**< Default ucl format */ UCL_PARSE_MSGPACK, /**< Message pack input format */ UCL_PARSE_CSEXP /**< Canonical S-expressions */ }; /** * UCL object structure. Please mention that the most of fields should not be touched by * UCL users. In future, this structure may be converted to private one. */ typedef struct ucl_object_s { /** * Variant value type */ union { int64_t iv; /**< Int value of an object */ const char *sv; /**< String value of an object */ double dv; /**< Double value of an object */ void *av; /**< Array */ void *ov; /**< Object */ void* ud; /**< Opaque user data */ } value; const char *key; /**< Key of an object */ struct ucl_object_s *next; /**< Array handle */ struct ucl_object_s *prev; /**< Array handle */ uint32_t keylen; /**< Lenght of a key */ uint32_t len; /**< Size of an object */ uint32_t ref; /**< Reference count */ uint16_t flags; /**< Object flags */ uint16_t type; /**< Real type */ unsigned char* trash_stack[2]; /**< Pointer to allocated chunks */ } ucl_object_t; /** * Destructor type for userdata objects * @param ud user specified data pointer */ typedef void (*ucl_userdata_dtor)(void *ud); typedef const char* (*ucl_userdata_emitter)(void *ud); /** @} */ /** * @defgroup utils Utility functions * A number of utility functions simplify handling of UCL objects * * @{ */ /** * Copy and return a key of an object, returned key is zero-terminated * @param obj CL object * @return zero terminated key */ UCL_EXTERN char* ucl_copy_key_trash (const ucl_object_t *obj); /** * Copy and return a string value of an object, returned key is zero-terminated * @param obj CL object * @return zero terminated string representation of object value */ UCL_EXTERN char* ucl_copy_value_trash (const ucl_object_t *obj); /** * Creates a new object * @return new object */ UCL_EXTERN ucl_object_t* ucl_object_new (void) UCL_WARN_UNUSED_RESULT; /** * Create new object with type specified * @param type type of a new object * @return new object */ UCL_EXTERN ucl_object_t* ucl_object_typed_new (ucl_type_t type) UCL_WARN_UNUSED_RESULT; /** * Create new object with type and priority specified * @param type type of a new object * @param priority priority of an object * @return new object */ UCL_EXTERN ucl_object_t* ucl_object_new_full (ucl_type_t type, unsigned priority) UCL_WARN_UNUSED_RESULT; /** * Create new object with userdata dtor * @param dtor destructor function * @param emitter emitter for userdata * @param ptr opaque pointer * @return new object */ UCL_EXTERN ucl_object_t* ucl_object_new_userdata (ucl_userdata_dtor dtor, ucl_userdata_emitter emitter, void *ptr) UCL_WARN_UNUSED_RESULT; /** * Perform deep copy of an object copying everything * @param other object to copy * @return new object with refcount equal to 1 */ UCL_EXTERN ucl_object_t * ucl_object_copy (const ucl_object_t *other) UCL_WARN_UNUSED_RESULT; /** * Return the type of an object * @return the object type */ UCL_EXTERN ucl_type_t ucl_object_type (const ucl_object_t *obj); /** * Converts ucl object type to its string representation * @param type type of object * @return constant string describing type */ UCL_EXTERN const char * ucl_object_type_to_string (ucl_type_t type); /** * Converts string that represents ucl type to real ucl type enum * @param input C string with name of type * @param res resulting target * @return true if `input` is a name of type stored in `res` */ UCL_EXTERN bool ucl_object_string_to_type (const char *input, ucl_type_t *res); /** * Convert any string to an ucl object making the specified transformations * @param str fixed size or NULL terminated string * @param len length (if len is zero, than str is treated as NULL terminated) * @param flags conversion flags * @return new object */ UCL_EXTERN ucl_object_t * ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags flags) UCL_WARN_UNUSED_RESULT; /** * Create a UCL object from the specified string * @param str NULL terminated string, will be json escaped * @return new object */ UCL_EXTERN ucl_object_t *ucl_object_fromstring (const char *str) UCL_WARN_UNUSED_RESULT; /** * Create a UCL object from the specified string * @param str fixed size string, will be json escaped * @param len length of a string * @return new object */ UCL_EXTERN ucl_object_t *ucl_object_fromlstring (const char *str, size_t len) UCL_WARN_UNUSED_RESULT; /** * Create an object from an integer number * @param iv number * @return new object */ UCL_EXTERN ucl_object_t* ucl_object_fromint (int64_t iv) UCL_WARN_UNUSED_RESULT; /** * Create an object from a float number * @param dv number * @return new object */ UCL_EXTERN ucl_object_t* ucl_object_fromdouble (double dv) UCL_WARN_UNUSED_RESULT; /** * Create an object from a boolean * @param bv bool value * @return new object */ UCL_EXTERN ucl_object_t* ucl_object_frombool (bool bv) UCL_WARN_UNUSED_RESULT; /** * Insert a object 'elt' to the hash 'top' and associate it with key 'key' * @param top destination object (must be of type UCL_OBJECT) * @param elt element to insert (must NOT be NULL) * @param key key to associate with this object (either const or preallocated) * @param keylen length of the key (or 0 for NULL terminated keys) * @param copy_key make an internal copy of key * @return true if key has been inserted */ UCL_EXTERN bool ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt, const char *key, size_t keylen, bool copy_key); /** * Replace a object 'elt' to the hash 'top' and associate it with key 'key', old object will be unrefed, * if no object has been found this function works like ucl_object_insert_key() * @param top destination object (must be of type UCL_OBJECT) * @param elt element to insert (must NOT be NULL) * @param key key to associate with this object (either const or preallocated) * @param keylen length of the key (or 0 for NULL terminated keys) * @param copy_key make an internal copy of key * @return true if key has been inserted */ UCL_EXTERN bool ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt, const char *key, size_t keylen, bool copy_key); /** * Merge the keys from one object to another object. Overwrite on conflict * @param top destination object (must be of type UCL_OBJECT) * @param elt element to insert (must be of type UCL_OBJECT) * @param copy copy rather than reference the elements * @return true if all keys have been merged */ UCL_EXTERN bool ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy); /** * Delete a object associated with key 'key', old object will be unrefered, * @param top object * @param key key associated to the object to remove * @param keylen length of the key (or 0 for NULL terminated keys) */ UCL_EXTERN bool ucl_object_delete_keyl (ucl_object_t *top, const char *key, size_t keylen); /** * Delete a object associated with key 'key', old object will be unrefered, * @param top object * @param key key associated to the object to remove */ UCL_EXTERN bool ucl_object_delete_key (ucl_object_t *top, const char *key); /** * Removes `key` from `top` object, returning the object that was removed. This * object is not released, caller must unref the returned object when it is no * longer needed. * @param top object * @param key key to remove * @param keylen length of the key (or 0 for NULL terminated keys) * @return removed object or NULL if object has not been found */ UCL_EXTERN ucl_object_t* ucl_object_pop_keyl (ucl_object_t *top, const char *key, size_t keylen) UCL_WARN_UNUSED_RESULT; /** * Removes `key` from `top` object returning the object that was removed. This * object is not released, caller must unref the returned object when it is no * longer needed. * @param top object * @param key key to remove * @return removed object or NULL if object has not been found */ UCL_EXTERN ucl_object_t* ucl_object_pop_key (ucl_object_t *top, const char *key) UCL_WARN_UNUSED_RESULT; /** * Insert a object 'elt' to the hash 'top' and associate it with key 'key', if * the specified key exist, try to merge its content * @param top destination object (must be of type UCL_OBJECT) * @param elt element to insert (must NOT be NULL) * @param key key to associate with this object (either const or preallocated) * @param keylen length of the key (or 0 for NULL terminated keys) * @param copy_key make an internal copy of key * @return true if key has been inserted */ UCL_EXTERN bool ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt, const char *key, size_t keylen, bool copy_key); /** * Append an element to the end of array object * @param top destination object (must NOT be NULL) * @param elt element to append (must NOT be NULL) * @return true if value has been inserted */ UCL_EXTERN bool ucl_array_append (ucl_object_t *top, ucl_object_t *elt); /** * Append an element to the start of array object * @param top destination object (must NOT be NULL) * @param elt element to append (must NOT be NULL) * @return true if value has been inserted */ UCL_EXTERN bool ucl_array_prepend (ucl_object_t *top, ucl_object_t *elt); /** * Merge all elements of second array into the first array * @param top destination array (must be of type UCL_ARRAY) * @param elt array to copy elements from (must be of type UCL_ARRAY) * @param copy copy elements instead of referencing them * @return true if arrays were merged */ UCL_EXTERN bool ucl_array_merge (ucl_object_t *top, ucl_object_t *elt, bool copy); /** * Removes an element `elt` from the array `top`, returning the object that was * removed. This object is not released, caller must unref the returned object * when it is no longer needed. * @param top array ucl object * @param elt element to remove * @return removed element or NULL if `top` is NULL or not an array */ UCL_EXTERN ucl_object_t* ucl_array_delete (ucl_object_t *top, ucl_object_t *elt); /** * Returns the first element of the array `top` * @param top array ucl object * @return element or NULL if `top` is NULL or not an array */ UCL_EXTERN const ucl_object_t* ucl_array_head (const ucl_object_t *top); /** * Returns the last element of the array `top` * @param top array ucl object * @return element or NULL if `top` is NULL or not an array */ UCL_EXTERN const ucl_object_t* ucl_array_tail (const ucl_object_t *top); /** * Removes the last element from the array `top`, returning the object that was * removed. This object is not released, caller must unref the returned object * when it is no longer needed. * @param top array ucl object * @return removed element or NULL if `top` is NULL or not an array */ UCL_EXTERN ucl_object_t* ucl_array_pop_last (ucl_object_t *top); /** * Removes the first element from the array `top`, returning the object that was * removed. This object is not released, caller must unref the returned object * when it is no longer needed. * @param top array ucl object * @return removed element or NULL if `top` is NULL or not an array */ UCL_EXTERN ucl_object_t* ucl_array_pop_first (ucl_object_t *top); /** * Return object identified by index of the array `top` * @param top object to get a key from (must be of type UCL_ARRAY) * @param index array index to return * @return object at the specified index or NULL if index is not found */ UCL_EXTERN const ucl_object_t* ucl_array_find_index (const ucl_object_t *top, unsigned int index); /** * Return the index of `elt` in the array `top` * @param top object to get a key from (must be of type UCL_ARRAY) * @param elt element to find index of (must NOT be NULL) * @return index of `elt` in the array `top or (unsigned int)-1 if `elt` is not found */ UCL_EXTERN unsigned int ucl_array_index_of (ucl_object_t *top, ucl_object_t *elt); /** * Replace an element in an array with a different element, returning the object * that was replaced. This object is not released, caller must unref the * returned object when it is no longer needed. * @param top destination object (must be of type UCL_ARRAY) * @param elt element to append (must NOT be NULL) * @param index array index in destination to overwrite with elt * @return object that was replaced or NULL if index is not found */ ucl_object_t * ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt, unsigned int index); /** * Append a element to another element forming an implicit array * @param head head to append (may be NULL) * @param elt new element * @return the new implicit array */ UCL_EXTERN ucl_object_t * ucl_elt_append (ucl_object_t *head, ucl_object_t *elt); /** * Converts an object to double value * @param obj CL object * @param target target double variable * @return true if conversion was successful */ UCL_EXTERN bool ucl_object_todouble_safe (const ucl_object_t *obj, double *target); /** * Unsafe version of \ref ucl_obj_todouble_safe * @param obj CL object * @return double value */ UCL_EXTERN double ucl_object_todouble (const ucl_object_t *obj); /** * Converts an object to integer value * @param obj CL object * @param target target integer variable * @return true if conversion was successful */ UCL_EXTERN bool ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target); /** * Unsafe version of \ref ucl_obj_toint_safe * @param obj CL object * @return int value */ UCL_EXTERN int64_t ucl_object_toint (const ucl_object_t *obj); /** * Converts an object to boolean value * @param obj CL object * @param target target boolean variable * @return true if conversion was successful */ UCL_EXTERN bool ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target); /** * Unsafe version of \ref ucl_obj_toboolean_safe * @param obj CL object * @return boolean value */ UCL_EXTERN bool ucl_object_toboolean (const ucl_object_t *obj); /** * Converts an object to string value * @param obj CL object * @param target target string variable, no need to free value * @return true if conversion was successful */ UCL_EXTERN bool ucl_object_tostring_safe (const ucl_object_t *obj, const char **target); /** * Unsafe version of \ref ucl_obj_tostring_safe * @param obj CL object * @return string value */ UCL_EXTERN const char* ucl_object_tostring (const ucl_object_t *obj); /** * Convert any object to a string in JSON notation if needed * @param obj CL object * @return string value */ UCL_EXTERN const char* ucl_object_tostring_forced (const ucl_object_t *obj); /** * Return string as char * and len, string may be not zero terminated, more efficient that \ref ucl_obj_tostring as it * allows zero-copy (if #UCL_PARSER_ZEROCOPY has been used during parsing) * @param obj CL object * @param target target string variable, no need to free value * @param tlen target length * @return true if conversion was successful */ UCL_EXTERN bool ucl_object_tolstring_safe (const ucl_object_t *obj, const char **target, size_t *tlen); /** * Unsafe version of \ref ucl_obj_tolstring_safe * @param obj CL object * @return string value */ UCL_EXTERN const char* ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen); /** * Return object identified by a key in the specified object * @param obj object to get a key from (must be of type UCL_OBJECT) * @param key key to search * @return object matching the specified key or NULL if key was not found */ UCL_EXTERN const ucl_object_t* ucl_object_lookup (const ucl_object_t *obj, const char *key); #define ucl_object_find_key ucl_object_lookup /** * Return object identified by a key in the specified object, if the first key is * not found then look for the next one. This process is repeated unless * the next argument in the list is not NULL. So, `ucl_object_find_any_key(obj, key, NULL)` * is equal to `ucl_object_find_key(obj, key)` * @param obj object to get a key from (must be of type UCL_OBJECT) * @param key key to search * @param ... list of alternative keys to search (NULL terminated) * @return object matching the specified key or NULL if key was not found */ UCL_EXTERN const ucl_object_t* ucl_object_lookup_any (const ucl_object_t *obj, const char *key, ...); #define ucl_object_find_any_key ucl_object_lookup_any /** * Return object identified by a fixed size key in the specified object * @param obj object to get a key from (must be of type UCL_OBJECT) * @param key key to search * @param klen length of a key * @return object matching the specified key or NULL if key was not found */ UCL_EXTERN const ucl_object_t* ucl_object_lookup_len (const ucl_object_t *obj, const char *key, size_t klen); #define ucl_object_find_keyl ucl_object_lookup_len /** * Return object identified by dot notation string * @param obj object to search in * @param path dot.notation.path to the path to lookup. May use numeric .index on arrays * @return object matched the specified path or NULL if path is not found */ UCL_EXTERN const ucl_object_t *ucl_object_lookup_path (const ucl_object_t *obj, const char *path); #define ucl_lookup_path ucl_object_lookup_path /** * Return object identified by object notation string using arbitrary delimiter * @param obj object to search in * @param path dot.notation.path to the path to lookup. May use numeric .index on arrays * @param sep the sepatorator to use in place of . (incase keys have . in them) * @return object matched the specified path or NULL if path is not found */ UCL_EXTERN const ucl_object_t *ucl_object_lookup_path_char (const ucl_object_t *obj, const char *path, char sep); #define ucl_lookup_path_char ucl_object_lookup_path_char /** * Returns a key of an object as a NULL terminated string * @param obj CL object * @return key or NULL if there is no key */ UCL_EXTERN const char* ucl_object_key (const ucl_object_t *obj); /** * Returns a key of an object as a fixed size string (may be more efficient) * @param obj CL object * @param len target key length * @return key pointer */ UCL_EXTERN const char* ucl_object_keyl (const ucl_object_t *obj, size_t *len); /** * Increase reference count for an object * @param obj object to ref * @return the referenced object */ UCL_EXTERN ucl_object_t* ucl_object_ref (const ucl_object_t *obj); /** * Free ucl object * @param obj ucl object to free */ UCL_DEPRECATED(UCL_EXTERN void ucl_object_free (ucl_object_t *obj)); /** * Decrease reference count for an object * @param obj object to unref */ UCL_EXTERN void ucl_object_unref (ucl_object_t *obj); /** * Compare objects `o1` and `o2` * @param o1 the first object * @param o2 the second object * @return values >0, 0 and <0 if `o1` is more than, equal and less than `o2`. * The order of comparison: * 1) Type of objects * 2) Size of objects * 3) Content of objects */ UCL_EXTERN int ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2); /** * Compare objects `o1` and `o2` useful for sorting * @param o1 the first object * @param o2 the second object * @return values >0, 0 and <0 if `o1` is more than, equal and less than `o2`. * The order of comparison: * 1) Type of objects * 2) Size of objects * 3) Content of objects */ UCL_EXTERN int ucl_object_compare_qsort (const ucl_object_t **o1, const ucl_object_t **o2); /** * Sort UCL array using `cmp` compare function * @param ar * @param cmp */ UCL_EXTERN void ucl_object_array_sort (ucl_object_t *ar, int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2)); /** * Get the priority for specific UCL object * @param obj any ucl object * @return priority of an object */ UCL_EXTERN unsigned int ucl_object_get_priority (const ucl_object_t *obj); /** * Set explicit priority of an object. * @param obj any ucl object * @param priority new priroity value (only 4 least significant bits are considred) */ UCL_EXTERN void ucl_object_set_priority (ucl_object_t *obj, unsigned int priority); /** * Opaque iterator object */ typedef void* ucl_object_iter_t; /** * Get next key from an object * @param obj object to iterate * @param iter opaque iterator, must be set to NULL on the first call: * ucl_object_iter_t it = NULL; * while ((cur = ucl_iterate_object (obj, &it)) != NULL) ... * @return the next object or NULL */ UCL_EXTERN const ucl_object_t* ucl_object_iterate (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values); #define ucl_iterate_object ucl_object_iterate /** * Create new safe iterator for the specified object * @param obj object to iterate * @return new iterator object that should be used with safe iterators API only */ UCL_EXTERN ucl_object_iter_t ucl_object_iterate_new (const ucl_object_t *obj) UCL_WARN_UNUSED_RESULT; /** * Reset initialized iterator to a new object * @param obj new object to iterate * @return modified iterator object */ UCL_EXTERN ucl_object_iter_t ucl_object_iterate_reset (ucl_object_iter_t it, const ucl_object_t *obj); /** * Get the next object from the `obj`. This fucntion iterates over arrays, objects * and implicit arrays * @param iter safe iterator * @return the next object in sequence */ UCL_EXTERN const ucl_object_t* ucl_object_iterate_safe (ucl_object_iter_t iter, bool expand_values); /** * Free memory associated with the safe iterator * @param it safe iterator object */ UCL_EXTERN void ucl_object_iterate_free (ucl_object_iter_t it); /** @} */ /** * @defgroup parser Parsing functions * These functions are used to parse UCL objects * * @{ */ /** * Macro handler for a parser * @param data the content of macro * @param len the length of content * @param arguments arguments object * @param ud opaque user data * @param err error pointer * @return true if macro has been parsed */ typedef bool (*ucl_macro_handler) (const unsigned char *data, size_t len, const ucl_object_t *arguments, void* ud); /** * Context dependent macro handler for a parser * @param data the content of macro * @param len the length of content * @param arguments arguments object * @param context previously parsed context * @param ud opaque user data * @param err error pointer * @return true if macro has been parsed */ typedef bool (*ucl_context_macro_handler) (const unsigned char *data, size_t len, const ucl_object_t *arguments, const ucl_object_t *context, void* ud); /* Opaque parser */ struct ucl_parser; /** * Creates new parser object * @param pool pool to allocate memory from * @return new parser object */ UCL_EXTERN struct ucl_parser* ucl_parser_new (int flags); /** * Sets the default priority for the parser applied to chunks that does not * specify priority explicitly * @param parser parser object * @param prio default priority (0 .. 16) * @return true if parser's default priority was set */ UCL_EXTERN bool ucl_parser_set_default_priority (struct ucl_parser *parser, unsigned prio); /** * Register new handler for a macro * @param parser parser object * @param macro macro name (without leading dot) * @param handler handler (it is called immediately after macro is parsed) * @param ud opaque user data for a handler */ UCL_EXTERN void ucl_parser_register_macro (struct ucl_parser *parser, const char *macro, ucl_macro_handler handler, void* ud); /** * Register new context dependent handler for a macro * @param parser parser object * @param macro macro name (without leading dot) * @param handler handler (it is called immediately after macro is parsed) * @param ud opaque user data for a handler */ UCL_EXTERN void ucl_parser_register_context_macro (struct ucl_parser *parser, const char *macro, ucl_context_macro_handler handler, void* ud); /** * Handler to detect unregistered variables * @param data variable data * @param len length of variable * @param replace (out) replace value for variable * @param replace_len (out) replace length for variable * @param need_free (out) UCL will free `dest` after usage * @param ud opaque userdata * @return true if variable */ typedef bool (*ucl_variable_handler) (const unsigned char *data, size_t len, unsigned char **replace, size_t *replace_len, bool *need_free, void* ud); /** * Register new parser variable * @param parser parser object * @param var variable name * @param value variable value */ UCL_EXTERN void ucl_parser_register_variable (struct ucl_parser *parser, const char *var, const char *value); /** * Set handler for unknown variables * @param parser parser structure * @param handler desired handler * @param ud opaque data for the handler */ UCL_EXTERN void ucl_parser_set_variables_handler (struct ucl_parser *parser, ucl_variable_handler handler, void *ud); /** * Load new chunk to a parser * @param parser parser structure * @param data the pointer to the beginning of a chunk * @param len the length of a chunk * @return true if chunk has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_chunk (struct ucl_parser *parser, const unsigned char *data, size_t len); /** * Load new chunk to a parser with the specified priority * @param parser parser structure * @param data the pointer to the beginning of a chunk * @param len the length of a chunk * @param priority the desired priority of a chunk (only 4 least significant bits * are considered for this parameter) * @return true if chunk has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_chunk_priority (struct ucl_parser *parser, const unsigned char *data, size_t len, unsigned priority); /** * Full version of ucl_add_chunk with priority and duplicate strategy * @param parser parser structure * @param data the pointer to the beginning of a chunk * @param len the length of a chunk * @param priority the desired priority of a chunk (only 4 least significant bits * are considered for this parameter) * @param strat duplicates merging strategy * @param parse_type input format * @return true if chunk has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data, size_t len, unsigned priority, enum ucl_duplicate_strategy strat, enum ucl_parse_type parse_type); /** * Load ucl object from a string * @param parser parser structure * @param data the pointer to the string * @param len the length of the string, if `len` is 0 then `data` must be zero-terminated string * @return true if string has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_string (struct ucl_parser *parser, const char *data,size_t len); /** * Load ucl object from a string * @param parser parser structure * @param data the pointer to the string * @param len the length of the string, if `len` is 0 then `data` must be zero-terminated string * @param priority the desired priority of a chunk (only 4 least significant bits * are considered for this parameter) * @return true if string has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_string_priority (struct ucl_parser *parser, const char *data, size_t len, unsigned priority); /** * Load and add data from a file * @param parser parser structure * @param filename the name of file - * @param err if *err is NULL it is set to parser error * @return true if chunk has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_file (struct ucl_parser *parser, const char *filename); /** * Load and add data from a file * @param parser parser structure * @param filename the name of file - * @param err if *err is NULL it is set to parser error * @param priority the desired priority of a chunk (only 4 least significant bits * are considered for this parameter) * @return true if chunk has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename, unsigned priority); + +/** + * Load and add data from a file + * @param parser parser structure + * @param filename the name of file + * @param priority the desired priority of a chunk (only 4 least significant bits + * are considered for this parameter) + * @param strat Merge strategy to use while parsing this file + * @param parse_type Parser type to use while parsing this file + * @return true if chunk has been added and false in case of error + */ +UCL_EXTERN bool ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename, + unsigned priority, enum ucl_duplicate_strategy strat, + enum ucl_parse_type parse_type); /** * Load and add data from a file descriptor * @param parser parser structure * @param filename the name of file * @param err if *err is NULL it is set to parser error * @return true if chunk has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_fd (struct ucl_parser *parser, int fd); /** * Load and add data from a file descriptor * @param parser parser structure * @param filename the name of file * @param err if *err is NULL it is set to parser error * @param priority the desired priority of a chunk (only 4 least significant bits * are considered for this parameter) * @return true if chunk has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_fd_priority (struct ucl_parser *parser, int fd, unsigned priority); /** * Provide a UCL_ARRAY of paths to search for include files. The object is * copied so caller must unref the object. * @param parser parser structure * @param paths UCL_ARRAY of paths to search * @return true if the path search array was replaced in the parser */ UCL_EXTERN bool ucl_set_include_path (struct ucl_parser *parser, ucl_object_t *paths); /** * Get a top object for a parser (refcount is increased) * @param parser parser structure * @param err if *err is NULL it is set to parser error * @return top parser object or NULL */ UCL_EXTERN ucl_object_t* ucl_parser_get_object (struct ucl_parser *parser); /** * Get the error string if parsing has been failed * @param parser parser object * @return error description */ UCL_EXTERN const char *ucl_parser_get_error (struct ucl_parser *parser); /** * Get the code of the last error * @param parser parser object * @return error code */ UCL_EXTERN int ucl_parser_get_error_code (struct ucl_parser *parser); /** * Get the current column number within parser * @param parser parser object * @return current column number */ UCL_EXTERN unsigned ucl_parser_get_column (struct ucl_parser *parser); /** * Get the current line number within parser * @param parser parser object * @return current line number */ UCL_EXTERN unsigned ucl_parser_get_linenum (struct ucl_parser *parser); /** * Clear the error in the parser * @param parser parser object */ UCL_EXTERN void ucl_parser_clear_error (struct ucl_parser *parser); /** * Free ucl parser object * @param parser parser object */ UCL_EXTERN void ucl_parser_free (struct ucl_parser *parser); /** * Get constant opaque pointer to comments structure for this parser. Increase * refcount to prevent this object to be destroyed on parser's destruction * @param parser parser structure * @return ucl comments pointer or NULL */ UCL_EXTERN const ucl_object_t * ucl_parser_get_comments (struct ucl_parser *parser); /** * Utility function to find a comment object for the specified object in the input * @param comments comments object * @param srch search object * @return string comment enclosed in ucl_object_t */ UCL_EXTERN const ucl_object_t * ucl_comments_find (const ucl_object_t *comments, const ucl_object_t *srch); /** * Move comment from `from` object to `to` object * @param comments comments object * @param what source object * @param whith destination object * @return `true` if `from` has comment and it has been moved to `to` */ UCL_EXTERN bool ucl_comments_move (ucl_object_t *comments, const ucl_object_t *from, const ucl_object_t *to); /** * Adds a new comment for an object * @param comments comments object * @param obj object to add comment to * @param comment string representation of a comment */ UCL_EXTERN void ucl_comments_add (ucl_object_t *comments, const ucl_object_t *obj, const char *comment); /** * Add new public key to parser for signatures check * @param parser parser object * @param key PEM representation of a key * @param len length of the key * @param err if *err is NULL it is set to parser error * @return true if a key has been successfully added */ UCL_EXTERN bool ucl_parser_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len); /** * Set FILENAME and CURDIR variables in parser * @param parser parser object * @param filename filename to set or NULL to set FILENAME to "undef" and CURDIR to getcwd() * @param need_expand perform realpath() if this variable is true and filename is not NULL * @return true if variables has been set */ UCL_EXTERN bool ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool need_expand); /** @} */ /** * @defgroup emitter Emitting functions * These functions are used to serialise UCL objects to some string representation. * * @{ */ struct ucl_emitter_context; /** * Structure using for emitter callbacks */ struct ucl_emitter_functions { /** Append a single character */ int (*ucl_emitter_append_character) (unsigned char c, size_t nchars, void *ud); /** Append a string of a specified length */ int (*ucl_emitter_append_len) (unsigned const char *str, size_t len, void *ud); /** Append a 64 bit integer */ int (*ucl_emitter_append_int) (int64_t elt, void *ud); /** Append floating point element */ int (*ucl_emitter_append_double) (double elt, void *ud); /** Free userdata */ void (*ucl_emitter_free_func)(void *ud); /** Opaque userdata pointer */ void *ud; }; struct ucl_emitter_operations { /** Write a primitive element */ void (*ucl_emitter_write_elt) (struct ucl_emitter_context *ctx, const ucl_object_t *obj, bool first, bool print_key); /** Start ucl object */ void (*ucl_emitter_start_object) (struct ucl_emitter_context *ctx, const ucl_object_t *obj, bool print_key); /** End ucl object */ void (*ucl_emitter_end_object) (struct ucl_emitter_context *ctx, const ucl_object_t *obj); /** Start ucl array */ void (*ucl_emitter_start_array) (struct ucl_emitter_context *ctx, const ucl_object_t *obj, bool print_key); void (*ucl_emitter_end_array) (struct ucl_emitter_context *ctx, const ucl_object_t *obj); }; /** * Structure that defines emitter functions */ struct ucl_emitter_context { /** Name of emitter (e.g. json, compact_json) */ const char *name; /** Unique id (e.g. UCL_EMIT_JSON for standard emitters */ int id; /** A set of output functions */ const struct ucl_emitter_functions *func; /** A set of output operations */ const struct ucl_emitter_operations *ops; /** Current amount of indent tabs */ unsigned int indent; /** Top level object */ const ucl_object_t *top; /** Optional comments */ const ucl_object_t *comments; }; /** * Emit object to a string * @param obj object * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is * #UCL_EMIT_CONFIG then emit config like object * @return dump of an object (must be freed after using) or NULL in case of error */ UCL_EXTERN unsigned char *ucl_object_emit (const ucl_object_t *obj, enum ucl_emitter emit_type); /** * Emit object to a string that can contain `\0` inside * @param obj object * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is * #UCL_EMIT_CONFIG then emit config like object * @param len the resulting length * @return dump of an object (must be freed after using) or NULL in case of error */ UCL_EXTERN unsigned char *ucl_object_emit_len (const ucl_object_t *obj, enum ucl_emitter emit_type, size_t *len); /** * Emit object to a string * @param obj object * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is * #UCL_EMIT_CONFIG then emit config like object * @param emitter a set of emitter functions * @param comments optional comments for the parser * @return dump of an object (must be freed after using) or NULL in case of error */ UCL_EXTERN bool ucl_object_emit_full (const ucl_object_t *obj, enum ucl_emitter emit_type, struct ucl_emitter_functions *emitter, const ucl_object_t *comments); /** * Start streamlined UCL object emitter * @param obj top UCL object * @param emit_type emit type * @param emitter a set of emitter functions * @return new streamlined context that should be freed by * `ucl_object_emit_streamline_finish` */ UCL_EXTERN struct ucl_emitter_context* ucl_object_emit_streamline_new ( const ucl_object_t *obj, enum ucl_emitter emit_type, struct ucl_emitter_functions *emitter); /** * Start object or array container for the streamlined output * @param ctx streamlined context * @param obj container object */ UCL_EXTERN void ucl_object_emit_streamline_start_container ( struct ucl_emitter_context *ctx, const ucl_object_t *obj); /** * Add a complete UCL object to streamlined output * @param ctx streamlined context * @param obj object to output */ UCL_EXTERN void ucl_object_emit_streamline_add_object ( struct ucl_emitter_context *ctx, const ucl_object_t *obj); /** * End previously added container * @param ctx streamlined context */ UCL_EXTERN void ucl_object_emit_streamline_end_container ( struct ucl_emitter_context *ctx); /** * Terminate streamlined container finishing all containers in it * @param ctx streamlined context */ UCL_EXTERN void ucl_object_emit_streamline_finish ( struct ucl_emitter_context *ctx); /** * Returns functions to emit object to memory * @param pmem target pointer (should be freed by caller) * @return emitter functions structure */ UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_memory_funcs ( void **pmem); /** * Returns functions to emit object to FILE * * @param fp FILE * object * @return emitter functions structure */ UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_file_funcs ( FILE *fp); /** * Returns functions to emit object to a file descriptor * @param fd file descriptor * @return emitter functions structure */ UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_fd_funcs ( int fd); /** * Free emitter functions * @param f pointer to functions */ UCL_EXTERN void ucl_object_emit_funcs_free (struct ucl_emitter_functions *f); /** @} */ /** * @defgroup schema Schema functions * These functions are used to validate UCL objects using json schema format * * @{ */ /** * Used to define UCL schema error */ enum ucl_schema_error_code { UCL_SCHEMA_OK = 0, /**< no error */ UCL_SCHEMA_TYPE_MISMATCH, /**< type of object is incorrect */ UCL_SCHEMA_INVALID_SCHEMA, /**< schema is invalid */ UCL_SCHEMA_MISSING_PROPERTY,/**< one or more missing properties */ UCL_SCHEMA_CONSTRAINT, /**< constraint found */ UCL_SCHEMA_MISSING_DEPENDENCY, /**< missing dependency */ UCL_SCHEMA_EXTERNAL_REF_MISSING, /**< cannot fetch external ref */ UCL_SCHEMA_EXTERNAL_REF_INVALID, /**< invalid external ref */ UCL_SCHEMA_INTERNAL_ERROR, /**< something bad happened */ UCL_SCHEMA_UNKNOWN /**< generic error */ }; /** * Generic ucl schema error */ struct ucl_schema_error { enum ucl_schema_error_code code; /**< error code */ char msg[128]; /**< error message */ const ucl_object_t *obj; /**< object where error occured */ }; /** * Validate object `obj` using schema object `schema`. * @param schema schema object * @param obj object to validate * @param err error pointer, if this parameter is not NULL and error has been * occured, then `err` is filled with the exact error definition. * @return true if `obj` is valid using `schema` */ UCL_EXTERN bool ucl_object_validate (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err); /** * Validate object `obj` using schema object `schema` and root schema at `root`. * @param schema schema object * @param obj object to validate * @param root root schema object * @param err error pointer, if this parameter is not NULL and error has been * occured, then `err` is filled with the exact error definition. * @return true if `obj` is valid using `schema` */ UCL_EXTERN bool ucl_object_validate_root (const ucl_object_t *schema, const ucl_object_t *obj, const ucl_object_t *root, struct ucl_schema_error *err); /** * Validate object `obj` using schema object `schema` and root schema at `root` * using some external references provided. * @param schema schema object * @param obj object to validate * @param root root schema object * @param ext_refs external references (might be modified during validation) * @param err error pointer, if this parameter is not NULL and error has been * occured, then `err` is filled with the exact error definition. * @return true if `obj` is valid using `schema` */ UCL_EXTERN bool ucl_object_validate_root_ext (const ucl_object_t *schema, const ucl_object_t *obj, const ucl_object_t *root, ucl_object_t *ext_refs, struct ucl_schema_error *err); /** @} */ #ifdef __cplusplus } #endif /* * XXX: Poorly named API functions, need to replace them with the appropriate * named function. All API functions *must* use naming ucl_object_*. Usage of * ucl_obj* should be avoided. */ #define ucl_obj_todouble_safe ucl_object_todouble_safe #define ucl_obj_todouble ucl_object_todouble #define ucl_obj_tostring ucl_object_tostring #define ucl_obj_tostring_safe ucl_object_tostring_safe #define ucl_obj_tolstring ucl_object_tolstring #define ucl_obj_tolstring_safe ucl_object_tolstring_safe #define ucl_obj_toint ucl_object_toint #define ucl_obj_toint_safe ucl_object_toint_safe #define ucl_obj_toboolean ucl_object_toboolean #define ucl_obj_toboolean_safe ucl_object_toboolean_safe #define ucl_obj_get_key ucl_object_find_key #define ucl_obj_get_keyl ucl_object_find_keyl #define ucl_obj_unref ucl_object_unref #define ucl_obj_ref ucl_object_ref #define ucl_obj_free ucl_object_free #endif /* UCL_H_ */ Index: head/contrib/libucl/m4/gcov.m4 =================================================================== --- head/contrib/libucl/m4/gcov.m4 (nonexistent) +++ head/contrib/libucl/m4/gcov.m4 (revision 306544) @@ -0,0 +1,89 @@ +# SYNOPSIS +# +# Add code coverage support with gcov/lcov. +# +# AX_CODE_COVERAGE() +# +# DESCRIPTION +# +# Provides a --enable-coverage option which checks for available +# gcov/lcov binaries and provides ENABLE_CODE_COVERAGE conditional. +# +# LAST MODIFICATION +# +# $Id: coverage.m4 40881 2013-08-20 17:54:39Z damon $ +# +# COPYLEFT +# +# Copyright (c) 2012 Roy H. Stogner +# Copyright (c) 2010 Karl W. Schulz +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. + +AC_DEFUN([AX_CODE_COVERAGE], +[ + +AC_ARG_ENABLE(coverage, AC_HELP_STRING([--enable-coverage],[configure code coverage analysis tools])) + +HAVE_GCOV_TOOLS=0 + +GCOV_FLAGS="" + +if test "x$enable_coverage" = "xyes"; then + + # ---------------------------- + # Check for gcov/lcov binaries + # ---------------------------- + + AC_ARG_VAR([GCOV], [Coverage testing command]) + if test "x$GCOV" = "x"; then + AC_PATH_PROG(GCOV, gcov, no) + else + AC_PATH_PROG(GCOV, $GCOV, no) + fi + + AC_PATH_PROG(LCOV, lcov, no) + AC_PATH_PROG(GENHTML, genhtml) + + # ---------------------------------- + # include coverage compiler options + # ---------------------------------- + AC_MSG_CHECKING([for clang]) + + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([], [[ + #ifndef __clang__ + not clang + #endif + ]])], + [CLANG=yes], [CLANG=no]) + + AC_MSG_RESULT([$CLANG]) + HAVE_GCOV_TOOLS=1 + COVERAGE_CFLAGS="-fprofile-arcs -ftest-coverage" + COVERAGE_LDFLAGS="--coverage -fprofile-arcs -ftest-coverage" + COVERAGE_OPTFLAGS="-O0" + + # Test for C... + CFLAGS="${GCOV_FLAGS} ${CFLAGS}" + CXXFLAGS="${GCOV_FLAGS} ${CXXFLAGS}" + if test "x$GCC" = "xyes" -a "x$CLANG" = "xno"; then + COVERAGE_LIBS="-lgcov" + else + COVERAGE_LIBS="" + fi +fi + +AC_SUBST([GCOV]) +AC_SUBST([LCOV]) +AC_SUBST([GENHTML]) +AC_SUBST([GENHTML_OPTIONS]) +AC_SUBST([COVERAGE_CFLAGS]) +AC_SUBST([COVERAGE_OPTFLAGS]) +AC_SUBST([COVERAGE_LDFLAGS]) +AC_SUBST([COVERAGE_LIBS]) +AM_CONDITIONAL(CODE_COVERAGE_ENABLED,test x$HAVE_GCOV_TOOLS = x1) + +]) Index: head/contrib/libucl/python/test.sh =================================================================== --- head/contrib/libucl/python/test.sh (revision 306543) +++ head/contrib/libucl/python/test.sh (nonexistent) @@ -1,6 +0,0 @@ -#!/bin/sh -set -xe -python3.4 setup.py build_ext --inplace -./test_uclmodule.py -v -rm -rfv build -rm ucl.so Property changes on: head/contrib/libucl/python/test.sh ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:executable ## -1 +0,0 ## -* \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/contrib/libucl/python/test_uclmodule.py =================================================================== --- head/contrib/libucl/python/test_uclmodule.py (revision 306543) +++ head/contrib/libucl/python/test_uclmodule.py (nonexistent) @@ -1,148 +0,0 @@ -#!/usr/bin/env python -import json -import unittest -import ucl -import sys - -if sys.version_info[:2] == (2, 7): - unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - - -class TestUcl(unittest.TestCase): - def test_no_args(self): - with self.assertRaises(TypeError): - ucl.load() - - def test_multi_args(self): - with self.assertRaises(TypeError): - ucl.load(0,0) - - def test_none(self): - r = ucl.load(None) - self.assertEqual(r, None) - - def test_int(self): - r = ucl.load("a : 1") - self.assertEqual(ucl.load("a : 1"), { "a" : 1 } ) - - def test_braced_int(self): - self.assertEqual(ucl.load("{a : 1}"), { "a" : 1 } ) - - def test_nested_int(self): - self.assertEqual(ucl.load("a : { b : 1 }"), { "a" : { "b" : 1 } }) - - def test_str(self): - self.assertEqual(ucl.load("a : b"), {"a" : "b"}) - - def test_float(self): - self.assertEqual(ucl.load("a : 1.1"), {"a" : 1.1}) - - def test_boolean(self): - totest = ( - "a : True;" \ - "b : False" - ) - correct = {"a" : True, "b" : False} - self.assertEqual(ucl.load(totest), correct) - - def test_empty_ucl(self): - r = ucl.load("{}") - self.assertEqual(r, {}) - - def test_single_brace(self): - self.assertEqual(ucl.load("{"), {}) - - def test_single_back_brace(self): - ucl.load("}") - - def test_single_square_forward(self): - self.assertEqual(ucl.load("["), []) - - def test_invalid_ucl(self): - with self.assertRaisesRegex(ValueError, "unfinished key$"): - ucl.load('{ "var"') - - def test_comment_ignored(self): - self.assertEqual(ucl.load("{/*1*/}"), {}) - - def test_1_in(self): - with open("../tests/basic/1.in", "r") as in1: - self.assertEqual(ucl.load(in1.read()), {'key1': 'value'}) - - def test_every_type(self): - totest="""{ - "key1": value; - "key2": value2; - "key3": "value;" - "key4": 1.0, - "key5": -0xdeadbeef - "key6": 0xdeadbeef.1 - "key7": 0xreadbeef - "key8": -1e-10, - "key9": 1 - "key10": true - "key11": no - "key12": yes - }""" - correct = { - 'key1': 'value', - 'key2': 'value2', - 'key3': 'value;', - 'key4': 1.0, - 'key5': -3735928559, - 'key6': '0xdeadbeef.1', - 'key7': '0xreadbeef', - 'key8': -1e-10, - 'key9': 1, - 'key10': True, - 'key11': False, - 'key12': True, - } - self.assertEqual(ucl.load(totest), correct) - - def test_validation_useless(self): - with self.assertRaises(NotImplementedError): - ucl.validate("","") - - -class TestUclDump(unittest.TestCase): - def test_no_args(self): - with self.assertRaises(TypeError): - ucl.dump() - - def test_multi_args(self): - with self.assertRaises(TypeError): - ucl.dump(0, 0) - - def test_none(self): - self.assertEqual(ucl.dump(None), None) - - def test_int(self): - self.assertEqual(ucl.dump({ "a" : 1 }), "a = 1;\n") - - def test_nested_int(self): - self.assertEqual(ucl.dump({ "a" : { "b" : 1 } }), "a {\n b = 1;\n}\n") - - def test_int_array(self): - self.assertEqual(ucl.dump({ "a" : [1,2,3,4]}), "a [\n 1,\n 2,\n 3,\n 4,\n]\n") - - def test_str(self): - self.assertEqual(ucl.dump({"a" : "b"}), "a = \"b\";\n") - - def test_float(self): - self.assertEqual(ucl.dump({"a" : 1.1}), "a = 1.100000;\n") - - def test_boolean(self): - totest = {"a" : True, "b" : False} - correct = "a = true;\nb = false;\n" - self.assertEqual(ucl.dump(totest), correct) - - def test_empty_ucl(self): - self.assertEqual(ucl.dump({}), "") - - def test_json(self): - self.assertEqual(ucl.dump({ "a" : 1, "b": "bleh;" }, ucl.UCL_EMIT_JSON), '{\n "a": 1,\n "b": "bleh;"\n}') - - -if __name__ == '__main__': - unittest.main() Property changes on: head/contrib/libucl/python/test_uclmodule.py ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:executable ## -1 +0,0 ## -* \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/contrib/libucl/python/setup.py =================================================================== --- head/contrib/libucl/python/setup.py (revision 306543) +++ head/contrib/libucl/python/setup.py (revision 306544) @@ -1,37 +1,43 @@ -import distutils.ccompiler -import distutils.sysconfig -from distutils.core import setup, Extension +try: + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension + import os +import sys +tests_require = [] -compiler = distutils.ccompiler.new_compiler() -search_paths=[os.path.expanduser('~/{}'), '/opt/local/{}', '/usr/local/{}', '/usr/{}'] -lib_paths = [ a.format("lib") for a in search_paths] -inc_paths = [ a.format("include") for a in search_paths] +if sys.version < '2.7': + tests_require.append('unittest2') -uclmodule = Extension('ucl', - include_dirs = inc_paths, - library_dirs = lib_paths, - libraries = ['ucl'], - sources = ['src/uclmodule.c'], - runtime_library_dirs = lib_paths, - language='c') +uclmodule = Extension( + 'ucl', + libraries = ['ucl'], + sources = ['src/uclmodule.c'], + language = 'c' +) -setup(name='ucl', - version='1.0', - description='ucl parser and emmitter', +setup( + name = 'ucl', + version = '0.8', + description = 'ucl parser and emmitter', ext_modules = [uclmodule], - author="Eitan Adler", - author_email="lists@eitanadler.com", - url="https://github.com/vstakhov/libucl/", - license="MIT", - classifiers=["Development Status :: 3 - Alpha", + test_suite = 'tests', + tests_require = tests_require, + author = "Eitan Adler, Denis Volpato Martins", + author_email = "lists@eitanadler.com", + url = "https://github.com/vstakhov/libucl/", + license = "MIT", + classifiers = [ + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: DFSG approved", "License :: OSI Approved :: MIT License", "Programming Language :: C", + "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Libraries", - ] - ) + ] +) Index: head/contrib/libucl/python/src/uclmodule.c =================================================================== --- head/contrib/libucl/python/src/uclmodule.c (revision 306543) +++ head/contrib/libucl/python/src/uclmodule.c (revision 306544) @@ -1,275 +1,334 @@ // Attempts to load a UCL structure from a string #include #include +static PyObject *SchemaError; + static PyObject * _basic_ucl_type (ucl_object_t const *obj) { switch (obj->type) { case UCL_INT: return Py_BuildValue ("L", (long long)ucl_object_toint (obj)); case UCL_FLOAT: return Py_BuildValue ("d", ucl_object_todouble (obj)); case UCL_STRING: return Py_BuildValue ("s", ucl_object_tostring (obj)); case UCL_BOOLEAN: - return ucl_object_toboolean (obj) ? Py_True : Py_False; + return PyBool_FromLong (ucl_object_toboolean (obj)); case UCL_TIME: return Py_BuildValue ("d", ucl_object_todouble (obj)); + case UCL_NULL: + Py_RETURN_NONE; } return NULL; } static PyObject * _iterate_valid_ucl (ucl_object_t const *obj) { const ucl_object_t *tmp; ucl_object_iter_t it = NULL; tmp = obj; while ((obj = ucl_object_iterate (tmp, &it, false))) { PyObject *val; val = _basic_ucl_type(obj); if (!val) { PyObject *key = NULL; if (obj->key != NULL) { key = Py_BuildValue("s", ucl_object_key(obj)); } if (obj->type == UCL_OBJECT) { const ucl_object_t *cur; ucl_object_iter_t it_obj = NULL; val = PyDict_New(); while ((cur = ucl_object_iterate (obj, &it_obj, true))) { PyObject *keyobj = Py_BuildValue("s",ucl_object_key(cur)); PyDict_SetItem(val, keyobj, _iterate_valid_ucl(cur)); } } else if (obj->type == UCL_ARRAY) { const ucl_object_t *cur; ucl_object_iter_t it_obj = NULL; val = PyList_New(0); while ((cur = ucl_object_iterate (obj, &it_obj, true))) { PyList_Append(val, _iterate_valid_ucl(cur)); } } else if (obj->type == UCL_USERDATA) { // XXX: this should be // PyBytes_FromStringAndSize; where is the // length from? val = PyBytes_FromString(obj->value.ud); } } return val; } PyErr_SetString(PyExc_SystemError, "unhandled type"); return NULL; } static PyObject * _internal_load_ucl (char *uclstr) { PyObject *ret; struct ucl_parser *parser = ucl_parser_new (UCL_PARSER_NO_TIME); bool r = ucl_parser_add_string(parser, uclstr, 0); if (r) { if (ucl_parser_get_error (parser)) { PyErr_SetString(PyExc_ValueError, ucl_parser_get_error(parser)); ucl_parser_free(parser); ret = NULL; goto return_with_parser; } else { ucl_object_t *uclobj = ucl_parser_get_object(parser); ret = _iterate_valid_ucl(uclobj); ucl_object_unref(uclobj); goto return_with_parser; } } else { PyErr_SetString(PyExc_ValueError, ucl_parser_get_error (parser)); ret = NULL; goto return_with_parser; } return_with_parser: ucl_parser_free(parser); return ret; } static PyObject* ucl_load (PyObject *self, PyObject *args) { char *uclstr; if (PyArg_ParseTuple(args, "z", &uclstr)) { if (!uclstr) { Py_RETURN_NONE; } return _internal_load_ucl(uclstr); } return NULL; } static ucl_object_t * _iterate_python (PyObject *obj) { if (obj == Py_None) { return ucl_object_new(); - } else if (PyBool_Check (obj)) { + } + else if (PyBool_Check (obj)) { return ucl_object_frombool (obj == Py_True); - } else if (PyInt_Check (obj)) { + } +#if PY_MAJOR_VERSION < 3 + else if (PyInt_Check (obj)) { return ucl_object_fromint (PyInt_AsLong (obj)); - } else if (PyFloat_Check (obj)) { + } +#endif + else if (PyLong_Check (obj)) { + return ucl_object_fromint (PyLong_AsLong (obj)); + } + else if (PyFloat_Check (obj)) { return ucl_object_fromdouble (PyFloat_AsDouble (obj)); - } else if (PyString_Check (obj)) { + } + else if (PyUnicode_Check (obj)) { + ucl_object_t *ucl_str; + PyObject *str = PyUnicode_AsASCIIString(obj); + ucl_str = ucl_object_fromstring (PyBytes_AsString (str)); + Py_DECREF(str); + return ucl_str; + } +#if PY_MAJOR_VERSION < 3 + else if (PyString_Check (obj)) { return ucl_object_fromstring (PyString_AsString (obj)); - // } else if (PyDateTime_Check (obj)) { } +#endif else if (PyDict_Check(obj)) { PyObject *key, *value; Py_ssize_t pos = 0; ucl_object_t *top, *elm; + char *keystr = NULL; top = ucl_object_typed_new (UCL_OBJECT); while (PyDict_Next(obj, &pos, &key, &value)) { elm = _iterate_python(value); - ucl_object_insert_key (top, elm, PyString_AsString(key), 0, true); + + if (PyUnicode_Check(key)) { + PyObject *keyascii = PyUnicode_AsASCIIString(key); + keystr = PyBytes_AsString(keyascii); + Py_DECREF(keyascii); + } +#if PY_MAJOR_VERSION < 3 + else if (PyString_Check(key)) { + keystr = PyString_AsString(key); + } +#endif + else { + PyErr_SetString(PyExc_TypeError, "Unknown key type"); + return NULL; + } + + ucl_object_insert_key (top, elm, keystr, 0, true); } return top; } else if (PySequence_Check(obj)) { PyObject *value; Py_ssize_t len, pos; ucl_object_t *top, *elm; len = PySequence_Length(obj); top = ucl_object_typed_new (UCL_ARRAY); for (pos = 0; pos < len; pos++) { value = PySequence_GetItem(obj, pos); elm = _iterate_python(value); ucl_array_append(top, elm); } return top; } else { PyErr_SetString(PyExc_TypeError, "Unhandled object type"); return NULL; } return NULL; } static PyObject * ucl_dump (PyObject *self, PyObject *args) { PyObject *obj; ucl_emitter_t emitter; ucl_object_t *root = NULL; emitter = UCL_EMIT_CONFIG; if (!PyArg_ParseTuple(args, "O|i", &obj, &emitter)) { PyErr_SetString(PyExc_TypeError, "Unhandled object type"); return NULL; } if (emitter >= UCL_EMIT_MAX) { PyErr_SetString(PyExc_TypeError, "Invalid emitter type"); return NULL; } if (obj == Py_None) { Py_RETURN_NONE; } - if (!PyDict_Check(obj)) { - PyErr_SetString(PyExc_TypeError, "Argument must be dict"); - return NULL; - } - root = _iterate_python(obj); if (root) { PyObject *ret; char *buf; buf = (char *) ucl_object_emit (root, emitter); ucl_object_unref (root); +#if PY_MAJOR_VERSION < 3 ret = PyString_FromString (buf); +#else + ret = PyUnicode_FromString (buf); +#endif free(buf); return ret; } return NULL; } static PyObject * ucl_validate (PyObject *self, PyObject *args) { - char *uclstr, *schema; + PyObject *dataobj, *schemaobj; + ucl_object_t *data, *schema; + bool r; + struct ucl_schema_error err; - if (PyArg_ParseTuple(args, "zz", &uclstr, &schema)) { - if (!uclstr || !schema) { - Py_RETURN_NONE; - } + if (!PyArg_ParseTuple (args, "OO", &schemaobj, &dataobj)) { + PyErr_SetString (PyExc_TypeError, "Unhandled object type"); + return NULL; + } - PyErr_SetString(PyExc_NotImplementedError, "schema validation is not yet supported"); + schema = _iterate_python(schemaobj); + if (!schema) + return NULL; + + data = _iterate_python(dataobj); + if (!data) + return NULL; + + // validation + r = ucl_object_validate (schema, data, &err); + ucl_object_unref (schema); + ucl_object_unref (data); + + if (!r) { + PyErr_SetString (SchemaError, err.msg); + return NULL; } - return NULL; + Py_RETURN_TRUE; } static PyMethodDef uclMethods[] = { {"load", ucl_load, METH_VARARGS, "Load UCL from stream"}, {"dump", ucl_dump, METH_VARARGS, "Dump UCL to stream"}, {"validate", ucl_validate, METH_VARARGS, "Validate ucl stream against schema"}, {NULL, NULL, 0, NULL} }; static void init_macros(PyObject *mod) { PyModule_AddIntMacro(mod, UCL_EMIT_JSON); PyModule_AddIntMacro(mod, UCL_EMIT_JSON_COMPACT); PyModule_AddIntMacro(mod, UCL_EMIT_CONFIG); PyModule_AddIntMacro(mod, UCL_EMIT_YAML); PyModule_AddIntMacro(mod, UCL_EMIT_MSGPACK); + + SchemaError = PyErr_NewException("ucl.SchemaError", NULL, NULL); + Py_INCREF(SchemaError); + PyModule_AddObject(mod, "SchemaError", SchemaError); } #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef uclmodule = { PyModuleDef_HEAD_INIT, "ucl", NULL, -1, uclMethods }; PyMODINIT_FUNC PyInit_ucl (void) { PyObject *mod = PyModule_Create (&uclmodule); init_macros (mod); return mod; } #else void initucl (void) { PyObject *mod = Py_InitModule ("ucl", uclMethods); init_macros (mod); } #endif Index: head/contrib/libucl/python/tests/__init__.py =================================================================== Property changes on: head/contrib/libucl/python/tests/__init__.py ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/contrib/libucl/python/tests/compat.py =================================================================== --- head/contrib/libucl/python/tests/compat.py (nonexistent) +++ head/contrib/libucl/python/tests/compat.py (revision 306544) @@ -0,0 +1,8 @@ +try: + import unittest2 as unittest +except ImportError: + import unittest + +# Python 2.7 - 3.1 +if not hasattr(unittest.TestCase, 'assertRaisesRegex'): + unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp Property changes on: head/contrib/libucl/python/tests/compat.py ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/contrib/libucl/python/tests/test_dump.py =================================================================== --- head/contrib/libucl/python/tests/test_dump.py (nonexistent) +++ head/contrib/libucl/python/tests/test_dump.py (revision 306544) @@ -0,0 +1,66 @@ +from .compat import unittest +import ucl +import sys + +class DumpTest(unittest.TestCase): + def test_no_args(self): + with self.assertRaises(TypeError): + ucl.dump() + + def test_none(self): + self.assertEqual(ucl.dump(None), None) + + def test_null(self): + data = { "a" : None } + valid = "a = null;\n" + self.assertEqual(ucl.dump(data), valid) + + def test_int(self): + data = { "a" : 1 } + valid = "a = 1;\n" + self.assertEqual(ucl.dump(data), valid) + + def test_nested_int(self): + data = { "a" : { "b" : 1 } } + valid = "a {\n b = 1;\n}\n" + self.assertEqual(ucl.dump(data), valid) + + def test_int_array(self): + data = { "a" : [1,2,3,4] } + valid = "a [\n 1,\n 2,\n 3,\n 4,\n]\n" + self.assertEqual(ucl.dump(data), valid) + + def test_str(self): + data = { "a" : "b" } + valid = "a = \"b\";\n" + self.assertEqual(ucl.dump(data), valid) + + @unittest.skipIf(sys.version_info[0] > 2, "Python3 uses unicode only") + def test_unicode(self): + data = { unicode("a") : unicode("b") } + valid = unicode("a = \"b\";\n") + self.assertEqual(ucl.dump(data), valid) + + def test_float(self): + data = { "a" : 1.1 } + valid = "a = 1.100000;\n" + self.assertEqual(ucl.dump(data), valid) + + def test_boolean(self): + data = { "a" : True, "b" : False } + valid = [ + "a = true;\nb = false;\n", + "b = false;\na = true;\n" + ] + self.assertIn(ucl.dump(data), valid) + + def test_empty_ucl(self): + self.assertEqual(ucl.dump({}), "") + + def test_json(self): + data = { "a" : 1, "b": "bleh;" } + valid = [ + '{\n "a": 1,\n "b": "bleh;"\n}', + '{\n "b": "bleh;",\n "a": 1\n}' + ] + self.assertIn(ucl.dump(data, ucl.UCL_EMIT_JSON), valid) Property changes on: head/contrib/libucl/python/tests/test_dump.py ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/contrib/libucl/python/tests/test_load.py =================================================================== --- head/contrib/libucl/python/tests/test_load.py (nonexistent) +++ head/contrib/libucl/python/tests/test_load.py (revision 306544) @@ -0,0 +1,107 @@ +from .compat import unittest +import ucl + +class LoadTest(unittest.TestCase): + def test_no_args(self): + with self.assertRaises(TypeError): + ucl.load() + + def test_multi_args(self): + with self.assertRaises(TypeError): + ucl.load(0,0) + + def test_none(self): + self.assertEqual(ucl.load(None), None) + + def test_null(self): + data = "a: null" + valid = { "a" : None } + self.assertEqual(ucl.load(data), valid) + + def test_int(self): + data = "a : 1" + valid = { "a" : 1 } + self.assertEqual(ucl.load(data), valid) + + def test_braced_int(self): + data = "{a : 1}" + valid = { "a" : 1 } + self.assertEqual(ucl.load(data), valid) + + def test_nested_int(self): + data = "a : { b : 1 }" + valid = { "a" : { "b" : 1 } } + self.assertEqual(ucl.load(data), valid) + + def test_str(self): + data = "a : b" + valid = { "a" : "b" } + self.assertEqual(ucl.load(data), valid) + + def test_float(self): + data = "a : 1.1" + valid = {"a" : 1.1} + self.assertEqual(ucl.load(data), valid) + + def test_boolean(self): + data = ( + "a : True;" \ + "b : False" + ) + valid = { "a" : True, "b" : False } + self.assertEqual(ucl.load(data), valid) + + def test_empty_ucl(self): + self.assertEqual(ucl.load("{}"), {}) + + def test_single_brace(self): + self.assertEqual(ucl.load("{"), {}) + + def test_single_back_brace(self): + self.assertEqual(ucl.load("}"), {}) + + def test_single_square_forward(self): + self.assertEqual(ucl.load("["), []) + + def test_invalid_ucl(self): + with self.assertRaisesRegex(ValueError, "unfinished key$"): + ucl.load('{ "var"') + + def test_comment_ignored(self): + self.assertEqual(ucl.load("{/*1*/}"), {}) + + def test_1_in(self): + valid = { 'key1': 'value' } + with open("../tests/basic/1.in", "r") as in1: + self.assertEqual(ucl.load(in1.read()), valid) + + def test_every_type(self): + data = ("""{ + "key1": value; + "key2": value2; + "key3": "value;" + "key4": 1.0, + "key5": -0xdeadbeef + "key6": 0xdeadbeef.1 + "key7": 0xreadbeef + "key8": -1e-10, + "key9": 1 + "key10": true + "key11": no + "key12": yes + }""") + valid = { + 'key1': 'value', + 'key2': 'value2', + 'key3': 'value;', + 'key4': 1.0, + 'key5': -3735928559, + 'key6': '0xdeadbeef.1', + 'key7': '0xreadbeef', + 'key8': -1e-10, + 'key9': 1, + 'key10': True, + 'key11': False, + 'key12': True, + } + self.assertEqual(ucl.load(data), valid) Property changes on: head/contrib/libucl/python/tests/test_load.py ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/contrib/libucl/python/tests/test_validation.py =================================================================== --- head/contrib/libucl/python/tests/test_validation.py (nonexistent) +++ head/contrib/libucl/python/tests/test_validation.py (revision 306544) @@ -0,0 +1,50 @@ +from .compat import unittest +import ucl +import json +import os.path +import glob +import re + +TESTS_SCHEMA_FOLDER = '../tests/schema/*.json' + +comment_re = re.compile('\/\*((?!\*\/).)*?\*\/', re.DOTALL | re.MULTILINE) +def json_remove_comments(content): + return comment_re.sub('', content) + +class ValidationTest(unittest.TestCase): + def validate(self, jsonfile): + def perform_test(schema, data, valid, description): + msg = '%s (valid=%r)' % (description, valid) + if valid: + self.assertTrue(ucl.validate(schema, data), msg) + else: + with self.assertRaises(ucl.SchemaError): + ucl.validate(schema, data) + self.fail(msg) # fail() will be called only if SchemaError is not raised + + with open(jsonfile) as f: + try: + # data = json.load(f) + data = json.loads(json_remove_comments(f.read())) + except ValueError as e: + raise self.skipTest('Failed to load JSON: %s' % str(e)) + + for testgroup in data: + for test in testgroup['tests']: + perform_test(testgroup['schema'], test['data'], + test['valid'], test['description']) + + @classmethod + def setupValidationTests(cls): + """Creates each test dynamically from a folder""" + def test_gen(filename): + def run_test(self): + self.validate(filename) + return run_test + + for jsonfile in glob.glob(TESTS_SCHEMA_FOLDER): + testname = os.path.splitext(os.path.basename(jsonfile))[0] + setattr(cls, 'test_%s' % testname, test_gen(jsonfile)) + + +ValidationTest.setupValidationTests() Property changes on: head/contrib/libucl/python/tests/test_validation.py ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/contrib/libucl/src/ucl_parser.c =================================================================== --- head/contrib/libucl/src/ucl_parser.c (revision 306543) +++ head/contrib/libucl/src/ucl_parser.c (revision 306544) @@ -1,2732 +1,2740 @@ /* Copyright (c) 2013, Vsevolod Stakhov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "ucl.h" #include "ucl_internal.h" #include "ucl_chartable.h" /** * @file ucl_parser.c * The implementation of ucl parser */ struct ucl_parser_saved_state { unsigned int line; unsigned int column; size_t remain; const unsigned char *pos; }; /** * Move up to len characters * @param parser * @param begin * @param len * @return new position in chunk */ #define ucl_chunk_skipc(chunk, p) do{ \ if (*(p) == '\n') { \ (chunk)->line ++; \ (chunk)->column = 0; \ } \ else (chunk)->column ++; \ (p++); \ (chunk)->pos ++; \ (chunk)->remain --; \ } while (0) static inline void ucl_set_err (struct ucl_parser *parser, int code, const char *str, UT_string **err) { const char *fmt_string, *filename; struct ucl_chunk *chunk = parser->chunks; if (parser->cur_file) { filename = parser->cur_file; } else { filename = ""; } if (chunk->pos < chunk->end) { if (isgraph (*chunk->pos)) { fmt_string = "error while parsing %s: " "line: %d, column: %d - '%s', character: '%c'"; } else { fmt_string = "error while parsing %s: " "line: %d, column: %d - '%s', character: '0x%02x'"; } ucl_create_err (err, fmt_string, filename, chunk->line, chunk->column, str, *chunk->pos); } else { ucl_create_err (err, "error while parsing %s: at the end of chunk: %s", filename, str); } parser->err_code = code; } static void ucl_save_comment (struct ucl_parser *parser, const char *begin, size_t len) { ucl_object_t *nobj; if (len > 0 && begin != NULL) { nobj = ucl_object_fromstring_common (begin, len, 0); if (parser->last_comment) { /* We need to append data to an existing object */ DL_APPEND (parser->last_comment, nobj); } else { parser->last_comment = nobj; } } } static void ucl_attach_comment (struct ucl_parser *parser, ucl_object_t *obj, bool before) { if (parser->last_comment) { ucl_object_insert_key (parser->comments, parser->last_comment, (const char *)&obj, sizeof (void *), true); if (before) { parser->last_comment->flags |= UCL_OBJECT_INHERITED; } parser->last_comment = NULL; } } /** * Skip all comments from the current pos resolving nested and multiline comments * @param parser * @return */ static bool ucl_skip_comments (struct ucl_parser *parser) { struct ucl_chunk *chunk = parser->chunks; const unsigned char *p, *beg = NULL; int comments_nested = 0; bool quoted = false; p = chunk->pos; start: if (chunk->remain > 0 && *p == '#') { if (parser->state != UCL_STATE_SCOMMENT && parser->state != UCL_STATE_MCOMMENT) { beg = p; while (p < chunk->end) { if (*p == '\n') { if (parser->flags & UCL_PARSER_SAVE_COMMENTS) { ucl_save_comment (parser, beg, p - beg); beg = NULL; } ucl_chunk_skipc (chunk, p); goto start; } ucl_chunk_skipc (chunk, p); } } } else if (chunk->remain >= 2 && *p == '/') { if (p[1] == '*') { beg = p; ucl_chunk_skipc (chunk, p); comments_nested ++; ucl_chunk_skipc (chunk, p); while (p < chunk->end) { if (*p == '"' && *(p - 1) != '\\') { quoted = !quoted; } if (!quoted) { if (*p == '*') { ucl_chunk_skipc (chunk, p); if (*p == '/') { comments_nested --; if (comments_nested == 0) { if (parser->flags & UCL_PARSER_SAVE_COMMENTS) { ucl_save_comment (parser, beg, p - beg + 1); beg = NULL; } ucl_chunk_skipc (chunk, p); goto start; } } ucl_chunk_skipc (chunk, p); } else if (p[0] == '/' && chunk->remain >= 2 && p[1] == '*') { comments_nested ++; ucl_chunk_skipc (chunk, p); ucl_chunk_skipc (chunk, p); continue; } } ucl_chunk_skipc (chunk, p); } if (comments_nested != 0) { ucl_set_err (parser, UCL_ENESTED, "unfinished multiline comment", &parser->err); return false; } } } if (beg && p > beg && (parser->flags & UCL_PARSER_SAVE_COMMENTS)) { ucl_save_comment (parser, beg, p - beg); } return true; } /** * Return multiplier for a character * @param c multiplier character * @param is_bytes if true use 1024 multiplier * @return multiplier */ static inline unsigned long ucl_lex_num_multiplier (const unsigned char c, bool is_bytes) { const struct { char c; long mult_normal; long mult_bytes; } multipliers[] = { {'m', 1000 * 1000, 1024 * 1024}, {'k', 1000, 1024}, {'g', 1000 * 1000 * 1000, 1024 * 1024 * 1024} }; int i; for (i = 0; i < 3; i ++) { if (tolower (c) == multipliers[i].c) { if (is_bytes) { return multipliers[i].mult_bytes; } return multipliers[i].mult_normal; } } return 1; } /** * Return multiplier for time scaling * @param c * @return */ static inline double ucl_lex_time_multiplier (const unsigned char c) { const struct { char c; double mult; } multipliers[] = { {'m', 60}, {'h', 60 * 60}, {'d', 60 * 60 * 24}, {'w', 60 * 60 * 24 * 7}, {'y', 60 * 60 * 24 * 365} }; int i; for (i = 0; i < 5; i ++) { if (tolower (c) == multipliers[i].c) { return multipliers[i].mult; } } return 1; } /** * Return true if a character is a end of an atom * @param c * @return */ static inline bool ucl_lex_is_atom_end (const unsigned char c) { return ucl_test_character (c, UCL_CHARACTER_VALUE_END); } static inline bool ucl_lex_is_comment (const unsigned char c1, const unsigned char c2) { if (c1 == '/') { if (c2 == '*') { return true; } } else if (c1 == '#') { return true; } return false; } /** * Check variable found * @param parser * @param ptr * @param remain * @param out_len * @param strict * @param found * @return */ static inline const char * ucl_check_variable_safe (struct ucl_parser *parser, const char *ptr, size_t remain, size_t *out_len, bool strict, bool *found) { struct ucl_variable *var; unsigned char *dst; size_t dstlen; bool need_free = false; LL_FOREACH (parser->variables, var) { if (strict) { if (remain == var->var_len) { if (memcmp (ptr, var->var, var->var_len) == 0) { *out_len += var->value_len; *found = true; return (ptr + var->var_len); } } } else { if (remain >= var->var_len) { if (memcmp (ptr, var->var, var->var_len) == 0) { *out_len += var->value_len; *found = true; return (ptr + var->var_len); } } } } /* XXX: can only handle ${VAR} */ if (!(*found) && parser->var_handler != NULL && strict) { /* Call generic handler */ if (parser->var_handler (ptr, remain, &dst, &dstlen, &need_free, parser->var_data)) { + *out_len += dstlen; *found = true; if (need_free) { free (dst); } return (ptr + remain); } } return ptr; } /** * Check for a variable in a given string * @param parser * @param ptr * @param remain * @param out_len * @param vars_found * @return */ static const char * ucl_check_variable (struct ucl_parser *parser, const char *ptr, size_t remain, size_t *out_len, bool *vars_found) { const char *p, *end, *ret = ptr; bool found = false; if (*ptr == '{') { /* We need to match the variable enclosed in braces */ p = ptr + 1; end = ptr + remain; while (p < end) { if (*p == '}') { ret = ucl_check_variable_safe (parser, ptr + 1, p - ptr - 1, out_len, true, &found); if (found) { /* {} must be excluded actually */ ret ++; if (!*vars_found) { *vars_found = true; } } else { *out_len += 2; } break; } p ++; } } else if (*ptr != '$') { /* Not count escaped dollar sign */ ret = ucl_check_variable_safe (parser, ptr, remain, out_len, false, &found); if (found && !*vars_found) { *vars_found = true; } if (!found) { (*out_len) ++; } } else { ret ++; (*out_len) ++; } return ret; } /** * Expand a single variable * @param parser * @param ptr * @param remain * @param dest * @return */ static const char * ucl_expand_single_variable (struct ucl_parser *parser, const char *ptr, size_t remain, unsigned char **dest) { unsigned char *d = *dest, *dst; const char *p = ptr + 1, *ret; struct ucl_variable *var; size_t dstlen; bool need_free = false; bool found = false; bool strict = false; ret = ptr + 1; remain --; if (*p == '$') { *d++ = *p++; *dest = d; return p; } else if (*p == '{') { p ++; strict = true; ret += 2; remain -= 2; } LL_FOREACH (parser->variables, var) { if (remain >= var->var_len) { if (memcmp (p, var->var, var->var_len) == 0) { memcpy (d, var->value, var->value_len); ret += var->var_len; d += var->value_len; found = true; break; } } } if (!found) { if (strict && parser->var_handler != NULL) { - if (parser->var_handler (ptr, remain, &dst, &dstlen, &need_free, + size_t var_len = 0; + while (var_len < remain && p[var_len] != '}') + var_len ++; + + if (parser->var_handler (p, var_len, &dst, &dstlen, &need_free, parser->var_data)) { memcpy (d, dst, dstlen); - ret += dstlen; - d += remain; + ret += var_len; + d += dstlen; + if (need_free) { + free (dst); + } found = true; } } /* Leave variable as is */ if (!found) { if (strict) { /* Copy '${' */ memcpy (d, ptr, 2); d += 2; ret --; } else { memcpy (d, ptr, 1); d ++; } } } *dest = d; return ret; } /** * Expand variables in string * @param parser * @param dst * @param src * @param in_len * @return */ static ssize_t ucl_expand_variable (struct ucl_parser *parser, unsigned char **dst, const char *src, size_t in_len) { const char *p, *end = src + in_len; unsigned char *d; size_t out_len = 0; bool vars_found = false; if (parser->flags & UCL_PARSER_DISABLE_MACRO) { *dst = NULL; return in_len; } p = src; while (p != end) { if (*p == '$') { p = ucl_check_variable (parser, p + 1, end - p - 1, &out_len, &vars_found); } else { p ++; out_len ++; } } if (!vars_found) { /* Trivial case */ *dst = NULL; return in_len; } *dst = UCL_ALLOC (out_len + 1); if (*dst == NULL) { return in_len; } d = *dst; p = src; while (p != end) { if (*p == '$') { p = ucl_expand_single_variable (parser, p, end - p, &d); } else { *d++ = *p++; } } *d = '\0'; return out_len; } /** * Store or copy pointer to the trash stack * @param parser parser object * @param src src string * @param dst destination buffer (trash stack pointer) * @param dst_const const destination pointer (e.g. value of object) * @param in_len input length * @param need_unescape need to unescape source (and copy it) * @param need_lowercase need to lowercase value (and copy) * @param need_expand need to expand variables (and copy as well) * @return output length (excluding \0 symbol) */ static inline ssize_t ucl_copy_or_store_ptr (struct ucl_parser *parser, const unsigned char *src, unsigned char **dst, const char **dst_const, size_t in_len, bool need_unescape, bool need_lowercase, bool need_expand) { ssize_t ret = -1, tret; unsigned char *tmp; if (need_unescape || need_lowercase || (need_expand && parser->variables != NULL) || !(parser->flags & UCL_PARSER_ZEROCOPY)) { /* Copy string */ *dst = UCL_ALLOC (in_len + 1); if (*dst == NULL) { ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for a string", &parser->err); return false; } if (need_lowercase) { ret = ucl_strlcpy_tolower (*dst, src, in_len + 1); } else { ret = ucl_strlcpy_unsafe (*dst, src, in_len + 1); } if (need_unescape) { ret = ucl_unescape_json_string (*dst, ret); } if (need_expand) { tmp = *dst; tret = ret; ret = ucl_expand_variable (parser, dst, tmp, ret); if (*dst == NULL) { /* Nothing to expand */ *dst = tmp; ret = tret; } else { /* Free unexpanded value */ UCL_FREE (in_len + 1, tmp); } } *dst_const = *dst; } else { *dst_const = src; ret = in_len; } return ret; } /** * Create and append an object at the specified level * @param parser * @param is_array * @param level * @return */ static inline ucl_object_t * ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser, bool is_array, int level) { struct ucl_stack *st; if (!is_array) { if (obj == NULL) { obj = ucl_object_new_full (UCL_OBJECT, parser->chunks->priority); } else { obj->type = UCL_OBJECT; } if (obj->value.ov == NULL) { obj->value.ov = ucl_hash_create (parser->flags & UCL_PARSER_KEY_LOWERCASE); } parser->state = UCL_STATE_KEY; } else { if (obj == NULL) { obj = ucl_object_new_full (UCL_ARRAY, parser->chunks->priority); } else { obj->type = UCL_ARRAY; } parser->state = UCL_STATE_VALUE; } st = UCL_ALLOC (sizeof (struct ucl_stack)); if (st == NULL) { ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for an object", &parser->err); ucl_object_unref (obj); return NULL; } st->obj = obj; st->level = level; LL_PREPEND (parser->stack, st); parser->cur_obj = obj; return obj; } int ucl_maybe_parse_number (ucl_object_t *obj, const char *start, const char *end, const char **pos, bool allow_double, bool number_bytes, bool allow_time) { const char *p = start, *c = start; char *endptr; bool got_dot = false, got_exp = false, need_double = false, is_time = false, valid_start = false, is_hex = false, is_neg = false; double dv = 0; int64_t lv = 0; if (*p == '-') { is_neg = true; c ++; p ++; } while (p < end) { if (is_hex && isxdigit (*p)) { p ++; } else if (isdigit (*p)) { valid_start = true; p ++; } else if (!is_hex && (*p == 'x' || *p == 'X')) { is_hex = true; allow_double = false; c = p + 1; } else if (allow_double) { if (p == c) { /* Empty digits sequence, not a number */ *pos = start; return EINVAL; } else if (*p == '.') { if (got_dot) { /* Double dots, not a number */ *pos = start; return EINVAL; } else { got_dot = true; need_double = true; p ++; } } else if (*p == 'e' || *p == 'E') { if (got_exp) { /* Double exp, not a number */ *pos = start; return EINVAL; } else { got_exp = true; need_double = true; p ++; if (p >= end) { *pos = start; return EINVAL; } if (!isdigit (*p) && *p != '+' && *p != '-') { /* Wrong exponent sign */ *pos = start; return EINVAL; } else { p ++; } } } else { /* Got the end of the number, need to check */ break; } } else { break; } } if (!valid_start) { *pos = start; return EINVAL; } errno = 0; if (need_double) { dv = strtod (c, &endptr); } else { if (is_hex) { lv = strtoimax (c, &endptr, 16); } else { lv = strtoimax (c, &endptr, 10); } } if (errno == ERANGE) { *pos = start; return ERANGE; } /* Now check endptr */ if (endptr == NULL || ucl_lex_is_atom_end (*endptr) || *endptr == '\0') { p = endptr; goto set_obj; } if (endptr < end && endptr != start) { p = endptr; switch (*p) { case 'm': case 'M': case 'g': case 'G': case 'k': case 'K': if (end - p >= 2) { if (p[1] == 's' || p[1] == 'S') { /* Milliseconds */ if (!need_double) { need_double = true; dv = lv; } is_time = true; if (p[0] == 'm' || p[0] == 'M') { dv /= 1000.; } else { dv *= ucl_lex_num_multiplier (*p, false); } p += 2; goto set_obj; } else if (number_bytes || (p[1] == 'b' || p[1] == 'B')) { /* Bytes */ if (need_double) { need_double = false; lv = dv; } lv *= ucl_lex_num_multiplier (*p, true); p += 2; goto set_obj; } else if (ucl_lex_is_atom_end (p[1])) { if (need_double) { dv *= ucl_lex_num_multiplier (*p, false); } else { lv *= ucl_lex_num_multiplier (*p, number_bytes); } p ++; goto set_obj; } else if (allow_time && end - p >= 3) { if (tolower (p[0]) == 'm' && tolower (p[1]) == 'i' && tolower (p[2]) == 'n') { /* Minutes */ if (!need_double) { need_double = true; dv = lv; } is_time = true; dv *= 60.; p += 3; goto set_obj; } } } else { if (need_double) { dv *= ucl_lex_num_multiplier (*p, false); } else { lv *= ucl_lex_num_multiplier (*p, number_bytes); } p ++; goto set_obj; } break; case 'S': case 's': if (allow_time && (p == end - 1 || ucl_lex_is_atom_end (p[1]))) { if (!need_double) { need_double = true; dv = lv; } p ++; is_time = true; goto set_obj; } break; case 'h': case 'H': case 'd': case 'D': case 'w': case 'W': case 'Y': case 'y': if (allow_time && (p == end - 1 || ucl_lex_is_atom_end (p[1]))) { if (!need_double) { need_double = true; dv = lv; } is_time = true; dv *= ucl_lex_time_multiplier (*p); p ++; goto set_obj; } break; case '\t': case ' ': while (p < end && ucl_test_character(*p, UCL_CHARACTER_WHITESPACE)) { p++; } if (ucl_lex_is_atom_end(*p)) goto set_obj; break; } } else if (endptr == end) { /* Just a number at the end of chunk */ p = endptr; goto set_obj; } *pos = c; return EINVAL; set_obj: if (obj != NULL) { if (allow_double && (need_double || is_time)) { if (!is_time) { obj->type = UCL_FLOAT; } else { obj->type = UCL_TIME; } obj->value.dv = is_neg ? (-dv) : dv; } else { obj->type = UCL_INT; obj->value.iv = is_neg ? (-lv) : lv; } } *pos = p; return 0; } /** * Parse possible number * @param parser * @param chunk * @param obj * @return true if a number has been parsed */ static bool ucl_lex_number (struct ucl_parser *parser, struct ucl_chunk *chunk, ucl_object_t *obj) { const unsigned char *pos; int ret; ret = ucl_maybe_parse_number (obj, chunk->pos, chunk->end, (const char **)&pos, true, false, ((parser->flags & UCL_PARSER_NO_TIME) == 0)); if (ret == 0) { chunk->remain -= pos - chunk->pos; chunk->column += pos - chunk->pos; chunk->pos = pos; return true; } else if (ret == ERANGE) { ucl_set_err (parser, UCL_ESYNTAX, "numeric value out of range", &parser->err); } return false; } /** * Parse quoted string with possible escapes * @param parser * @param chunk * @param need_unescape * @param ucl_escape * @param var_expand * @return true if a string has been parsed */ static bool ucl_lex_json_string (struct ucl_parser *parser, struct ucl_chunk *chunk, bool *need_unescape, bool *ucl_escape, bool *var_expand) { const unsigned char *p = chunk->pos; unsigned char c; int i; while (p < chunk->end) { c = *p; if (c < 0x1F) { /* Unmasked control character */ if (c == '\n') { ucl_set_err (parser, UCL_ESYNTAX, "unexpected newline", &parser->err); } else { ucl_set_err (parser, UCL_ESYNTAX, "unexpected control character", &parser->err); } return false; } else if (c == '\\') { ucl_chunk_skipc (chunk, p); c = *p; if (p >= chunk->end) { ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character", &parser->err); return false; } else if (ucl_test_character (c, UCL_CHARACTER_ESCAPE)) { if (c == 'u') { ucl_chunk_skipc (chunk, p); for (i = 0; i < 4 && p < chunk->end; i ++) { if (!isxdigit (*p)) { ucl_set_err (parser, UCL_ESYNTAX, "invalid utf escape", &parser->err); return false; } ucl_chunk_skipc (chunk, p); } if (p >= chunk->end) { ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character", &parser->err); return false; } } else { ucl_chunk_skipc (chunk, p); } } *need_unescape = true; *ucl_escape = true; continue; } else if (c == '"') { ucl_chunk_skipc (chunk, p); return true; } else if (ucl_test_character (c, UCL_CHARACTER_UCL_UNSAFE)) { *ucl_escape = true; } else if (c == '$') { *var_expand = true; } ucl_chunk_skipc (chunk, p); } ucl_set_err (parser, UCL_ESYNTAX, "no quote at the end of json string", &parser->err); return false; } static void ucl_parser_append_elt (struct ucl_parser *parser, ucl_hash_t *cont, ucl_object_t *top, ucl_object_t *elt) { ucl_object_t *nobj; if ((parser->flags & UCL_PARSER_NO_IMPLICIT_ARRAYS) == 0) { /* Implicit array */ top->flags |= UCL_OBJECT_MULTIVALUE; DL_APPEND (top, elt); parser->stack->obj->len ++; } else { if ((top->flags & UCL_OBJECT_MULTIVALUE) != 0) { /* Just add to the explicit array */ ucl_array_append (top, elt); } else { /* Convert to an array */ nobj = ucl_object_typed_new (UCL_ARRAY); nobj->key = top->key; nobj->keylen = top->keylen; nobj->flags |= UCL_OBJECT_MULTIVALUE; ucl_array_append (nobj, top); ucl_array_append (nobj, elt); ucl_hash_replace (cont, top, nobj); } } } bool ucl_parser_process_object_element (struct ucl_parser *parser, ucl_object_t *nobj) { ucl_hash_t *container; ucl_object_t *tobj; char errmsg[256]; container = parser->stack->obj->value.ov; tobj = __DECONST (ucl_object_t *, ucl_hash_search_obj (container, nobj)); if (tobj == NULL) { container = ucl_hash_insert_object (container, nobj, parser->flags & UCL_PARSER_KEY_LOWERCASE); nobj->prev = nobj; nobj->next = NULL; parser->stack->obj->len ++; } else { unsigned priold = ucl_object_get_priority (tobj), prinew = ucl_object_get_priority (nobj); switch (parser->chunks->strategy) { case UCL_DUPLICATE_APPEND: /* * The logic here is the following: * * - if we have two objects with the same priority, then we form an * implicit or explicit array * - if a new object has bigger priority, then we overwrite an old one * - if a new object has lower priority, then we ignore it */ /* Special case for inherited objects */ if (tobj->flags & UCL_OBJECT_INHERITED) { prinew = priold + 1; } if (priold == prinew) { ucl_parser_append_elt (parser, container, tobj, nobj); } else if (priold > prinew) { /* * We add this new object to a list of trash objects just to ensure * that it won't come to any real object * XXX: rather inefficient approach */ DL_APPEND (parser->trash_objs, nobj); } else { ucl_hash_replace (container, tobj, nobj); ucl_object_unref (tobj); } break; case UCL_DUPLICATE_REWRITE: /* We just rewrite old values regardless of priority */ ucl_hash_replace (container, tobj, nobj); ucl_object_unref (tobj); break; case UCL_DUPLICATE_ERROR: snprintf(errmsg, sizeof(errmsg), "duplicate element for key '%s' found", nobj->key); ucl_set_err (parser, UCL_EMERGE, errmsg, &parser->err); return false; case UCL_DUPLICATE_MERGE: /* * Here we do have some old object so we just push it on top of objects stack * Check priority and then perform the merge on the remaining objects */ if (tobj->type == UCL_OBJECT || tobj->type == UCL_ARRAY) { ucl_object_unref (nobj); nobj = tobj; } else if (priold == prinew) { ucl_parser_append_elt (parser, container, tobj, nobj); } else if (priold > prinew) { /* * We add this new object to a list of trash objects just to ensure * that it won't come to any real object * XXX: rather inefficient approach */ DL_APPEND (parser->trash_objs, nobj); } else { ucl_hash_replace (container, tobj, nobj); ucl_object_unref (tobj); } break; } } parser->stack->obj->value.ov = container; parser->cur_obj = nobj; ucl_attach_comment (parser, nobj, false); return true; } /** * Parse a key in an object * @param parser * @param chunk * @param next_key * @param end_of_object * @return true if a key has been parsed */ static bool ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk, bool *next_key, bool *end_of_object) { const unsigned char *p, *c = NULL, *end, *t; const char *key = NULL; bool got_quote = false, got_eq = false, got_semicolon = false, need_unescape = false, ucl_escape = false, var_expand = false, got_content = false, got_sep = false; ucl_object_t *nobj; ssize_t keylen; p = chunk->pos; if (*p == '.') { /* It is macro actually */ if (!(parser->flags & UCL_PARSER_DISABLE_MACRO)) { ucl_chunk_skipc (chunk, p); } parser->prev_state = parser->state; parser->state = UCL_STATE_MACRO_NAME; *end_of_object = false; return true; } while (p < chunk->end) { /* * A key must start with alpha, number, '/' or '_' and end with space character */ if (c == NULL) { if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) { if (!ucl_skip_comments (parser)) { return false; } p = chunk->pos; } else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { ucl_chunk_skipc (chunk, p); } else if (ucl_test_character (*p, UCL_CHARACTER_KEY_START)) { /* The first symbol */ c = p; ucl_chunk_skipc (chunk, p); got_content = true; } else if (*p == '"') { /* JSON style key */ c = p + 1; got_quote = true; got_content = true; ucl_chunk_skipc (chunk, p); } else if (*p == '}') { /* We have actually end of an object */ *end_of_object = true; return true; } else if (*p == '.') { ucl_chunk_skipc (chunk, p); parser->prev_state = parser->state; parser->state = UCL_STATE_MACRO_NAME; return true; } else { /* Invalid identifier */ ucl_set_err (parser, UCL_ESYNTAX, "key must begin with a letter", &parser->err); return false; } } else { /* Parse the body of a key */ if (!got_quote) { if (ucl_test_character (*p, UCL_CHARACTER_KEY)) { got_content = true; ucl_chunk_skipc (chunk, p); } else if (ucl_test_character (*p, UCL_CHARACTER_KEY_SEP)) { end = p; break; } else { ucl_set_err (parser, UCL_ESYNTAX, "invalid character in a key", &parser->err); return false; } } else { /* We need to parse json like quoted string */ if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) { return false; } /* Always escape keys obtained via json */ end = chunk->pos - 1; p = chunk->pos; break; } } } if (p >= chunk->end && got_content) { ucl_set_err (parser, UCL_ESYNTAX, "unfinished key", &parser->err); return false; } else if (!got_content) { return true; } *end_of_object = false; /* We are now at the end of the key, need to parse the rest */ while (p < chunk->end) { if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) { ucl_chunk_skipc (chunk, p); } else if (*p == '=') { if (!got_eq && !got_semicolon) { ucl_chunk_skipc (chunk, p); got_eq = true; } else { ucl_set_err (parser, UCL_ESYNTAX, "unexpected '=' character", &parser->err); return false; } } else if (*p == ':') { if (!got_eq && !got_semicolon) { ucl_chunk_skipc (chunk, p); got_semicolon = true; } else { ucl_set_err (parser, UCL_ESYNTAX, "unexpected ':' character", &parser->err); return false; } } else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) { /* Check for comment */ if (!ucl_skip_comments (parser)) { return false; } p = chunk->pos; } else { /* Start value */ break; } } if (p >= chunk->end && got_content) { ucl_set_err (parser, UCL_ESYNTAX, "unfinished key", &parser->err); return false; } got_sep = got_semicolon || got_eq; if (!got_sep) { /* * Maybe we have more keys nested, so search for termination character. * Possible choices: * 1) key1 key2 ... keyN [:=] value <- we treat that as error * 2) key1 ... keyN {} or [] <- we treat that as nested objects * 3) key1 value[;,\n] <- we treat that as linear object */ t = p; *next_key = false; while (ucl_test_character (*t, UCL_CHARACTER_WHITESPACE)) { t ++; } /* Check first non-space character after a key */ if (*t != '{' && *t != '[') { while (t < chunk->end) { if (*t == ',' || *t == ';' || *t == '\n' || *t == '\r') { break; } else if (*t == '{' || *t == '[') { *next_key = true; break; } t ++; } } } /* Create a new object */ nobj = ucl_object_new_full (UCL_NULL, parser->chunks->priority); keylen = ucl_copy_or_store_ptr (parser, c, &nobj->trash_stack[UCL_TRASH_KEY], &key, end - c, need_unescape, parser->flags & UCL_PARSER_KEY_LOWERCASE, false); if (keylen == -1) { ucl_object_unref (nobj); return false; } else if (keylen == 0) { ucl_set_err (parser, UCL_ESYNTAX, "empty keys are not allowed", &parser->err); ucl_object_unref (nobj); return false; } nobj->key = key; nobj->keylen = keylen; if (!ucl_parser_process_object_element (parser, nobj)) { return false; } if (ucl_escape) { nobj->flags |= UCL_OBJECT_NEED_KEY_ESCAPE; } return true; } /** * Parse a cl string * @param parser * @param chunk * @param var_expand * @param need_unescape * @return true if a key has been parsed */ static bool ucl_parse_string_value (struct ucl_parser *parser, struct ucl_chunk *chunk, bool *var_expand, bool *need_unescape) { const unsigned char *p; enum { UCL_BRACE_ROUND = 0, UCL_BRACE_SQUARE, UCL_BRACE_FIGURE }; int braces[3][2] = {{0, 0}, {0, 0}, {0, 0}}; p = chunk->pos; while (p < chunk->end) { /* Skip pairs of figure braces */ if (*p == '{') { braces[UCL_BRACE_FIGURE][0] ++; } else if (*p == '}') { braces[UCL_BRACE_FIGURE][1] ++; if (braces[UCL_BRACE_FIGURE][1] <= braces[UCL_BRACE_FIGURE][0]) { /* This is not a termination symbol, continue */ ucl_chunk_skipc (chunk, p); continue; } } /* Skip pairs of square braces */ else if (*p == '[') { braces[UCL_BRACE_SQUARE][0] ++; } else if (*p == ']') { braces[UCL_BRACE_SQUARE][1] ++; if (braces[UCL_BRACE_SQUARE][1] <= braces[UCL_BRACE_SQUARE][0]) { /* This is not a termination symbol, continue */ ucl_chunk_skipc (chunk, p); continue; } } else if (*p == '$') { *var_expand = true; } else if (*p == '\\') { *need_unescape = true; ucl_chunk_skipc (chunk, p); if (p < chunk->end) { ucl_chunk_skipc (chunk, p); } continue; } if (ucl_lex_is_atom_end (*p) || (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) { break; } ucl_chunk_skipc (chunk, p); } return true; } /** * Parse multiline string ending with \n{term}\n * @param parser * @param chunk * @param term * @param term_len * @param beg * @param var_expand * @return size of multiline string or 0 in case of error */ static int ucl_parse_multiline_string (struct ucl_parser *parser, struct ucl_chunk *chunk, const unsigned char *term, int term_len, unsigned char const **beg, bool *var_expand) { const unsigned char *p, *c, *tend; bool newline = false; int len = 0; p = chunk->pos; c = p; while (p < chunk->end) { if (newline) { if (chunk->end - p < term_len) { return 0; } else if (memcmp (p, term, term_len) == 0) { tend = p + term_len; if (*tend != '\n' && *tend != ';' && *tend != ',') { /* Incomplete terminator */ ucl_chunk_skipc (chunk, p); continue; } len = p - c; chunk->remain -= term_len; chunk->pos = p + term_len; chunk->column = term_len; *beg = c; break; } } if (*p == '\n') { newline = true; } else { if (*p == '$') { *var_expand = true; } newline = false; } ucl_chunk_skipc (chunk, p); } return len; } static inline ucl_object_t* ucl_parser_get_container (struct ucl_parser *parser) { ucl_object_t *t, *obj = NULL; if (parser == NULL || parser->stack == NULL || parser->stack->obj == NULL) { return NULL; } if (parser->stack->obj->type == UCL_ARRAY) { /* Object must be allocated */ obj = ucl_object_new_full (UCL_NULL, parser->chunks->priority); t = parser->stack->obj; if (!ucl_array_append (t, obj)) { ucl_object_unref (obj); return NULL; } parser->cur_obj = obj; ucl_attach_comment (parser, obj, false); } else { /* Object has been already allocated */ obj = parser->cur_obj; } return obj; } /** * Handle value data * @param parser * @param chunk * @return */ static bool ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk) { const unsigned char *p, *c; ucl_object_t *obj = NULL; unsigned int stripped_spaces; int str_len; bool need_unescape = false, ucl_escape = false, var_expand = false; p = chunk->pos; /* Skip any spaces and comments */ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) || (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) { while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { ucl_chunk_skipc (chunk, p); } if (!ucl_skip_comments (parser)) { return false; } p = chunk->pos; } while (p < chunk->end) { c = p; switch (*p) { case '"': ucl_chunk_skipc (chunk, p); if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) { return false; } obj = ucl_parser_get_container (parser); if (!obj) { return false; } str_len = chunk->pos - c - 2; obj->type = UCL_STRING; if ((str_len = ucl_copy_or_store_ptr (parser, c + 1, &obj->trash_stack[UCL_TRASH_VALUE], &obj->value.sv, str_len, need_unescape, false, var_expand)) == -1) { return false; } obj->len = str_len; parser->state = UCL_STATE_AFTER_VALUE; p = chunk->pos; return true; break; case '{': obj = ucl_parser_get_container (parser); /* We have a new object */ obj = ucl_parser_add_container (obj, parser, false, parser->stack->level); if (obj == NULL) { return false; } ucl_chunk_skipc (chunk, p); return true; break; case '[': obj = ucl_parser_get_container (parser); /* We have a new array */ obj = ucl_parser_add_container (obj, parser, true, parser->stack->level); if (obj == NULL) { return false; } ucl_chunk_skipc (chunk, p); return true; break; case ']': /* We have the array ending */ if (parser->stack && parser->stack->obj->type == UCL_ARRAY) { parser->state = UCL_STATE_AFTER_VALUE; return true; } else { goto parse_string; } break; case '<': obj = ucl_parser_get_container (parser); /* We have something like multiline value, which must be <<[A-Z]+\n */ if (chunk->end - p > 3) { if (memcmp (p, "<<", 2) == 0) { p += 2; /* We allow only uppercase characters in multiline definitions */ while (p < chunk->end && *p >= 'A' && *p <= 'Z') { p ++; } if (*p =='\n') { /* Set chunk positions and start multiline parsing */ c += 2; chunk->remain -= p - c; chunk->pos = p + 1; chunk->column = 0; chunk->line ++; if ((str_len = ucl_parse_multiline_string (parser, chunk, c, p - c, &c, &var_expand)) == 0) { ucl_set_err (parser, UCL_ESYNTAX, "unterminated multiline value", &parser->err); return false; } obj->type = UCL_STRING; obj->flags |= UCL_OBJECT_MULTILINE; if ((str_len = ucl_copy_or_store_ptr (parser, c, &obj->trash_stack[UCL_TRASH_VALUE], &obj->value.sv, str_len - 1, false, false, var_expand)) == -1) { return false; } obj->len = str_len; parser->state = UCL_STATE_AFTER_VALUE; return true; } } } /* Fallback to ordinary strings */ default: parse_string: if (obj == NULL) { obj = ucl_parser_get_container (parser); } /* Parse atom */ if (ucl_test_character (*p, UCL_CHARACTER_VALUE_DIGIT_START)) { if (!ucl_lex_number (parser, chunk, obj)) { if (parser->state == UCL_STATE_ERROR) { return false; } } else { parser->state = UCL_STATE_AFTER_VALUE; return true; } /* Fallback to normal string */ } if (!ucl_parse_string_value (parser, chunk, &var_expand, &need_unescape)) { return false; } /* Cut trailing spaces */ stripped_spaces = 0; while (ucl_test_character (*(chunk->pos - 1 - stripped_spaces), UCL_CHARACTER_WHITESPACE)) { stripped_spaces ++; } str_len = chunk->pos - c - stripped_spaces; if (str_len <= 0) { ucl_set_err (parser, UCL_ESYNTAX, "string value must not be empty", &parser->err); return false; } else if (str_len == 4 && memcmp (c, "null", 4) == 0) { obj->len = 0; obj->type = UCL_NULL; } else if (!ucl_maybe_parse_boolean (obj, c, str_len)) { obj->type = UCL_STRING; if ((str_len = ucl_copy_or_store_ptr (parser, c, &obj->trash_stack[UCL_TRASH_VALUE], &obj->value.sv, str_len, need_unescape, false, var_expand)) == -1) { return false; } obj->len = str_len; } parser->state = UCL_STATE_AFTER_VALUE; p = chunk->pos; return true; break; } } return true; } /** * Handle after value data * @param parser * @param chunk * @return */ static bool ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk) { const unsigned char *p; bool got_sep = false; struct ucl_stack *st; p = chunk->pos; while (p < chunk->end) { if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) { /* Skip whitespaces */ ucl_chunk_skipc (chunk, p); } else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) { /* Skip comment */ if (!ucl_skip_comments (parser)) { return false; } /* Treat comment as a separator */ got_sep = true; p = chunk->pos; } else if (ucl_test_character (*p, UCL_CHARACTER_VALUE_END)) { if (*p == '}' || *p == ']') { if (parser->stack == NULL) { ucl_set_err (parser, UCL_ESYNTAX, "end of array or object detected without corresponding start", &parser->err); return false; } if ((*p == '}' && parser->stack->obj->type == UCL_OBJECT) || (*p == ']' && parser->stack->obj->type == UCL_ARRAY)) { /* Pop all nested objects from a stack */ st = parser->stack; parser->stack = st->next; UCL_FREE (sizeof (struct ucl_stack), st); if (parser->cur_obj) { ucl_attach_comment (parser, parser->cur_obj, true); } while (parser->stack != NULL) { st = parser->stack; if (st->next == NULL || st->next->level == st->level) { break; } parser->stack = st->next; parser->cur_obj = st->obj; UCL_FREE (sizeof (struct ucl_stack), st); } } else { ucl_set_err (parser, UCL_ESYNTAX, "unexpected terminating symbol detected", &parser->err); return false; } if (parser->stack == NULL) { /* Ignore everything after a top object */ return true; } else { ucl_chunk_skipc (chunk, p); } got_sep = true; } else { /* Got a separator */ got_sep = true; ucl_chunk_skipc (chunk, p); } } else { /* Anything else */ if (!got_sep) { ucl_set_err (parser, UCL_ESYNTAX, "delimiter is missing", &parser->err); return false; } return true; } } return true; } static bool ucl_skip_macro_as_comment (struct ucl_parser *parser, struct ucl_chunk *chunk) { const unsigned char *p, *c; enum { macro_skip_start = 0, macro_has_symbols, macro_has_obrace, macro_has_quote, macro_has_backslash, macro_has_sqbrace, macro_save } state = macro_skip_start, prev_state = macro_skip_start; p = chunk->pos; c = chunk->pos; while (p < chunk->end) { switch (state) { case macro_skip_start: if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) { state = macro_has_symbols; } else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { state = macro_save; continue; } ucl_chunk_skipc (chunk, p); break; case macro_has_symbols: if (*p == '{') { state = macro_has_sqbrace; } else if (*p == '(') { state = macro_has_obrace; } else if (*p == '"') { state = macro_has_quote; } else if (*p == '\n') { state = macro_save; continue; } ucl_chunk_skipc (chunk, p); break; case macro_has_obrace: if (*p == '\\') { prev_state = state; state = macro_has_backslash; } else if (*p == ')') { state = macro_has_symbols; } ucl_chunk_skipc (chunk, p); break; case macro_has_sqbrace: if (*p == '\\') { prev_state = state; state = macro_has_backslash; } else if (*p == '}') { state = macro_save; } ucl_chunk_skipc (chunk, p); break; case macro_has_quote: if (*p == '\\') { prev_state = state; state = macro_has_backslash; } else if (*p == '"') { state = macro_save; } ucl_chunk_skipc (chunk, p); break; case macro_has_backslash: state = prev_state; ucl_chunk_skipc (chunk, p); break; case macro_save: if (parser->flags & UCL_PARSER_SAVE_COMMENTS) { ucl_save_comment (parser, c, p - c); } return true; } } return false; } /** * Handle macro data * @param parser * @param chunk * @param marco * @param macro_start * @param macro_len * @return */ static bool ucl_parse_macro_value (struct ucl_parser *parser, struct ucl_chunk *chunk, struct ucl_macro *macro, unsigned char const **macro_start, size_t *macro_len) { const unsigned char *p, *c; bool need_unescape = false, ucl_escape = false, var_expand = false; p = chunk->pos; switch (*p) { case '"': /* We have macro value encoded in quotes */ c = p; ucl_chunk_skipc (chunk, p); if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) { return false; } *macro_start = c + 1; *macro_len = chunk->pos - c - 2; p = chunk->pos; break; case '{': /* We got a multiline macro body */ ucl_chunk_skipc (chunk, p); /* Skip spaces at the beginning */ while (p < chunk->end) { if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { ucl_chunk_skipc (chunk, p); } else { break; } } c = p; while (p < chunk->end) { if (*p == '}') { break; } ucl_chunk_skipc (chunk, p); } *macro_start = c; *macro_len = p - c; ucl_chunk_skipc (chunk, p); break; default: /* Macro is not enclosed in quotes or braces */ c = p; while (p < chunk->end) { if (ucl_lex_is_atom_end (*p)) { break; } ucl_chunk_skipc (chunk, p); } *macro_start = c; *macro_len = p - c; break; } /* We are at the end of a macro */ /* Skip ';' and space characters and return to previous state */ while (p < chunk->end) { if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) && *p != ';') { break; } ucl_chunk_skipc (chunk, p); } return true; } /** * Parse macro arguments as UCL object * @param parser parser structure * @param chunk the current data chunk * @return */ static ucl_object_t * ucl_parse_macro_arguments (struct ucl_parser *parser, struct ucl_chunk *chunk) { ucl_object_t *res = NULL; struct ucl_parser *params_parser; int obraces = 1, ebraces = 0, state = 0; const unsigned char *p, *c; size_t args_len = 0; struct ucl_parser_saved_state saved; saved.column = chunk->column; saved.line = chunk->line; saved.pos = chunk->pos; saved.remain = chunk->remain; p = chunk->pos; if (*p != '(' || chunk->remain < 2) { return NULL; } /* Set begin and start */ ucl_chunk_skipc (chunk, p); c = p; while ((p) < (chunk)->end) { switch (state) { case 0: /* Parse symbols and check for '(', ')' and '"' */ if (*p == '(') { obraces ++; } else if (*p == ')') { ebraces ++; } else if (*p == '"') { state = 1; } /* Check pairing */ if (obraces == ebraces) { state = 99; } else { args_len ++; } /* Check overflow */ if (chunk->remain == 0) { goto restore_chunk; } ucl_chunk_skipc (chunk, p); break; case 1: /* We have quote character, so skip all but quotes */ if (*p == '"' && *(p - 1) != '\\') { state = 0; } if (chunk->remain == 0) { goto restore_chunk; } args_len ++; ucl_chunk_skipc (chunk, p); break; case 99: /* * We have read the full body of arguments, so we need to parse and set * object from that */ params_parser = ucl_parser_new (parser->flags); if (!ucl_parser_add_chunk (params_parser, c, args_len)) { ucl_set_err (parser, UCL_ESYNTAX, "macro arguments parsing error", &parser->err); } else { res = ucl_parser_get_object (params_parser); } ucl_parser_free (params_parser); return res; break; } } return res; restore_chunk: chunk->column = saved.column; chunk->line = saved.line; chunk->pos = saved.pos; chunk->remain = saved.remain; return NULL; } #define SKIP_SPACES_COMMENTS(parser, chunk, p) do { \ while ((p) < (chunk)->end) { \ if (!ucl_test_character (*(p), UCL_CHARACTER_WHITESPACE_UNSAFE)) { \ if ((chunk)->remain >= 2 && ucl_lex_is_comment ((p)[0], (p)[1])) { \ if (!ucl_skip_comments (parser)) { \ return false; \ } \ p = (chunk)->pos; \ } \ break; \ } \ ucl_chunk_skipc (chunk, p); \ } \ } while(0) /** * Handle the main states of rcl parser * @param parser parser structure * @return true if chunk has been parsed and false in case of error */ static bool ucl_state_machine (struct ucl_parser *parser) { ucl_object_t *obj, *macro_args; struct ucl_chunk *chunk = parser->chunks; const unsigned char *p, *c = NULL, *macro_start = NULL; unsigned char *macro_escaped; size_t macro_len = 0; struct ucl_macro *macro = NULL; bool next_key = false, end_of_object = false, ret; if (parser->top_obj == NULL) { parser->state = UCL_STATE_INIT; } p = chunk->pos; while (chunk->pos < chunk->end) { switch (parser->state) { case UCL_STATE_INIT: /* * At the init state we can either go to the parse array or object * if we got [ or { correspondingly or can just treat new data as * a key of newly created object */ if (!ucl_skip_comments (parser)) { parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; return false; } else { /* Skip any spaces */ while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { ucl_chunk_skipc (chunk, p); } p = chunk->pos; if (*p == '[') { parser->state = UCL_STATE_VALUE; ucl_chunk_skipc (chunk, p); } else { parser->state = UCL_STATE_KEY; if (*p == '{') { ucl_chunk_skipc (chunk, p); } } if (parser->top_obj == NULL) { if (parser->state == UCL_STATE_VALUE) { obj = ucl_parser_add_container (NULL, parser, true, 0); } else { obj = ucl_parser_add_container (NULL, parser, false, 0); } if (obj == NULL) { return false; } parser->top_obj = obj; parser->cur_obj = obj; } } break; case UCL_STATE_KEY: /* Skip any spaces */ while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { ucl_chunk_skipc (chunk, p); } if (p == chunk->end || *p == '}') { /* We have the end of an object */ parser->state = UCL_STATE_AFTER_VALUE; continue; } if (parser->stack == NULL) { /* No objects are on stack, but we want to parse a key */ ucl_set_err (parser, UCL_ESYNTAX, "top object is finished but the parser " "expects a key", &parser->err); parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; return false; } if (!ucl_parse_key (parser, chunk, &next_key, &end_of_object)) { parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; return false; } if (end_of_object) { p = chunk->pos; parser->state = UCL_STATE_AFTER_VALUE; continue; } else if (parser->state != UCL_STATE_MACRO_NAME) { if (next_key && parser->stack->obj->type == UCL_OBJECT) { /* Parse more keys and nest objects accordingly */ obj = ucl_parser_add_container (parser->cur_obj, parser, false, parser->stack->level + 1); if (obj == NULL) { return false; } } else { parser->state = UCL_STATE_VALUE; } } else { c = chunk->pos; } p = chunk->pos; break; case UCL_STATE_VALUE: /* We need to check what we do have */ if (!parser->cur_obj || !ucl_parse_value (parser, chunk)) { parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; return false; } /* State is set in ucl_parse_value call */ p = chunk->pos; break; case UCL_STATE_AFTER_VALUE: if (!ucl_parse_after_value (parser, chunk)) { parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; return false; } if (parser->stack != NULL) { if (parser->stack->obj->type == UCL_OBJECT) { parser->state = UCL_STATE_KEY; } else { /* Array */ parser->state = UCL_STATE_VALUE; } } else { /* Skip everything at the end */ return true; } p = chunk->pos; break; case UCL_STATE_MACRO_NAME: if (parser->flags & UCL_PARSER_DISABLE_MACRO) { if (!ucl_skip_macro_as_comment (parser, chunk)) { /* We have invalid macro */ ucl_create_err (&parser->err, "error on line %d at column %d: invalid macro", chunk->line, chunk->column); parser->state = UCL_STATE_ERROR; return false; } else { p = chunk->pos; parser->state = parser->prev_state; } } else { if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) && *p != '(') { ucl_chunk_skipc (chunk, p); } else { if (c != NULL && p - c > 0) { /* We got macro name */ macro_len = (size_t) (p - c); HASH_FIND (hh, parser->macroes, c, macro_len, macro); if (macro == NULL) { ucl_create_err (&parser->err, "error on line %d at column %d: " "unknown macro: '%.*s', character: '%c'", chunk->line, chunk->column, (int) (p - c), c, *chunk->pos); parser->state = UCL_STATE_ERROR; return false; } /* Now we need to skip all spaces */ SKIP_SPACES_COMMENTS(parser, chunk, p); parser->state = UCL_STATE_MACRO; } else { /* We have invalid macro name */ ucl_create_err (&parser->err, "error on line %d at column %d: invalid macro name", chunk->line, chunk->column); parser->state = UCL_STATE_ERROR; return false; } } } break; case UCL_STATE_MACRO: if (*chunk->pos == '(') { macro_args = ucl_parse_macro_arguments (parser, chunk); p = chunk->pos; if (macro_args) { SKIP_SPACES_COMMENTS(parser, chunk, p); } } else { macro_args = NULL; } if (!ucl_parse_macro_value (parser, chunk, macro, ¯o_start, ¯o_len)) { parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; return false; } macro_len = ucl_expand_variable (parser, ¯o_escaped, macro_start, macro_len); parser->state = parser->prev_state; if (macro_escaped == NULL && macro != NULL) { if (macro->is_context) { ret = macro->h.context_handler (macro_start, macro_len, macro_args, parser->top_obj, macro->ud); } else { ret = macro->h.handler (macro_start, macro_len, macro_args, macro->ud); } } else if (macro != NULL) { if (macro->is_context) { ret = macro->h.context_handler (macro_escaped, macro_len, macro_args, parser->top_obj, macro->ud); } else { ret = macro->h.handler (macro_escaped, macro_len, macro_args, macro->ud); } UCL_FREE (macro_len + 1, macro_escaped); } else { ret = false; ucl_set_err (parser, UCL_EINTERNAL, "internal error: parser has macro undefined", &parser->err); } /* * Chunk can be modified within macro handler */ chunk = parser->chunks; p = chunk->pos; if (macro_args) { ucl_object_unref (macro_args); } if (!ret) { return false; } break; default: ucl_set_err (parser, UCL_EINTERNAL, "internal error: parser is in an unknown state", &parser->err); parser->state = UCL_STATE_ERROR; return false; } } if (parser->last_comment) { if (parser->cur_obj) { ucl_attach_comment (parser, parser->cur_obj, true); } else if (parser->stack && parser->stack->obj) { ucl_attach_comment (parser, parser->stack->obj, true); } else if (parser->top_obj) { ucl_attach_comment (parser, parser->top_obj, true); } else { ucl_object_unref (parser->last_comment); } } return true; } struct ucl_parser* ucl_parser_new (int flags) { struct ucl_parser *parser; parser = UCL_ALLOC (sizeof (struct ucl_parser)); if (parser == NULL) { return NULL; } memset (parser, 0, sizeof (struct ucl_parser)); ucl_parser_register_macro (parser, "include", ucl_include_handler, parser); ucl_parser_register_macro (parser, "try_include", ucl_try_include_handler, parser); ucl_parser_register_macro (parser, "includes", ucl_includes_handler, parser); ucl_parser_register_macro (parser, "priority", ucl_priority_handler, parser); ucl_parser_register_macro (parser, "load", ucl_load_handler, parser); ucl_parser_register_context_macro (parser, "inherit", ucl_inherit_handler, parser); parser->flags = flags; parser->includepaths = NULL; if (flags & UCL_PARSER_SAVE_COMMENTS) { parser->comments = ucl_object_typed_new (UCL_OBJECT); } /* Initial assumption about filevars */ ucl_parser_set_filevars (parser, NULL, false); return parser; } bool ucl_parser_set_default_priority (struct ucl_parser *parser, unsigned prio) { if (parser == NULL) { return false; } parser->default_priority = prio; return true; } void ucl_parser_register_macro (struct ucl_parser *parser, const char *macro, ucl_macro_handler handler, void* ud) { struct ucl_macro *new; if (macro == NULL || handler == NULL) { return; } new = UCL_ALLOC (sizeof (struct ucl_macro)); if (new == NULL) { return; } memset (new, 0, sizeof (struct ucl_macro)); new->h.handler = handler; new->name = strdup (macro); new->ud = ud; HASH_ADD_KEYPTR (hh, parser->macroes, new->name, strlen (new->name), new); } void ucl_parser_register_context_macro (struct ucl_parser *parser, const char *macro, ucl_context_macro_handler handler, void* ud) { struct ucl_macro *new; if (macro == NULL || handler == NULL) { return; } new = UCL_ALLOC (sizeof (struct ucl_macro)); if (new == NULL) { return; } memset (new, 0, sizeof (struct ucl_macro)); new->h.context_handler = handler; new->name = strdup (macro); new->ud = ud; new->is_context = true; HASH_ADD_KEYPTR (hh, parser->macroes, new->name, strlen (new->name), new); } void ucl_parser_register_variable (struct ucl_parser *parser, const char *var, const char *value) { struct ucl_variable *new = NULL, *cur; if (var == NULL) { return; } /* Find whether a variable already exists */ LL_FOREACH (parser->variables, cur) { if (strcmp (cur->var, var) == 0) { new = cur; break; } } if (value == NULL) { if (new != NULL) { /* Remove variable */ DL_DELETE (parser->variables, new); free (new->var); free (new->value); UCL_FREE (sizeof (struct ucl_variable), new); } else { /* Do nothing */ return; } } else { if (new == NULL) { new = UCL_ALLOC (sizeof (struct ucl_variable)); if (new == NULL) { return; } memset (new, 0, sizeof (struct ucl_variable)); new->var = strdup (var); new->var_len = strlen (var); new->value = strdup (value); new->value_len = strlen (value); DL_APPEND (parser->variables, new); } else { free (new->value); new->value = strdup (value); new->value_len = strlen (value); } } } void ucl_parser_set_variables_handler (struct ucl_parser *parser, ucl_variable_handler handler, void *ud) { parser->var_handler = handler; parser->var_data = ud; } bool ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data, size_t len, unsigned priority, enum ucl_duplicate_strategy strat, enum ucl_parse_type parse_type) { struct ucl_chunk *chunk; if (parser == NULL) { return false; } if (data == NULL && len != 0) { ucl_create_err (&parser->err, "invalid chunk added"); return false; } if (parser->state != UCL_STATE_ERROR) { chunk = UCL_ALLOC (sizeof (struct ucl_chunk)); if (chunk == NULL) { ucl_create_err (&parser->err, "cannot allocate chunk structure"); return false; } chunk->begin = data; chunk->remain = len; chunk->pos = chunk->begin; chunk->end = chunk->begin + len; chunk->line = 1; chunk->column = 0; chunk->priority = priority; chunk->strategy = strat; chunk->parse_type = parse_type; LL_PREPEND (parser->chunks, chunk); parser->recursion ++; if (parser->recursion > UCL_MAX_RECURSION) { ucl_create_err (&parser->err, "maximum include nesting limit is reached: %d", parser->recursion); return false; } if (len > 0) { /* Need to parse something */ switch (parse_type) { default: case UCL_PARSE_UCL: return ucl_state_machine (parser); case UCL_PARSE_MSGPACK: return ucl_parse_msgpack (parser); } } else { /* Just add empty chunk and go forward */ if (parser->top_obj == NULL) { /* * In case of empty object, create one to indicate that we've * read something */ parser->top_obj = ucl_object_new_full (UCL_OBJECT, priority); } return true; } } ucl_create_err (&parser->err, "a parser is in an invalid state"); return false; } bool ucl_parser_add_chunk_priority (struct ucl_parser *parser, const unsigned char *data, size_t len, unsigned priority) { /* We dereference parser, so this check is essential */ if (parser == NULL) { return false; } return ucl_parser_add_chunk_full (parser, data, len, priority, UCL_DUPLICATE_APPEND, UCL_PARSE_UCL); } bool ucl_parser_add_chunk (struct ucl_parser *parser, const unsigned char *data, size_t len) { if (parser == NULL) { return false; } return ucl_parser_add_chunk_full (parser, data, len, parser->default_priority, UCL_DUPLICATE_APPEND, UCL_PARSE_UCL); } bool ucl_parser_add_string_priority (struct ucl_parser *parser, const char *data, size_t len, unsigned priority) { if (data == NULL) { ucl_create_err (&parser->err, "invalid string added"); return false; } if (len == 0) { len = strlen (data); } return ucl_parser_add_chunk_priority (parser, (const unsigned char *)data, len, priority); } bool ucl_parser_add_string (struct ucl_parser *parser, const char *data, size_t len) { if (parser == NULL) { return false; } return ucl_parser_add_string_priority (parser, (const unsigned char *)data, len, parser->default_priority); } bool ucl_set_include_path (struct ucl_parser *parser, ucl_object_t *paths) { if (parser == NULL || paths == NULL) { return false; } if (parser->includepaths == NULL) { parser->includepaths = ucl_object_copy (paths); } else { ucl_object_unref (parser->includepaths); parser->includepaths = ucl_object_copy (paths); } if (parser->includepaths == NULL) { return false; } return true; } Index: head/contrib/libucl/src/ucl_schema.c =================================================================== --- head/contrib/libucl/src/ucl_schema.c (revision 306543) +++ head/contrib/libucl/src/ucl_schema.c (revision 306544) @@ -1,1081 +1,1091 @@ /* * Copyright (c) 2014, Vsevolod Stakhov * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "ucl.h" #include "ucl_internal.h" #include "tree.h" #include "utlist.h" #ifdef HAVE_STDARG_H #include #endif #ifdef HAVE_STDIO_H #include #endif #ifdef HAVE_REGEX_H #include #endif #ifdef HAVE_MATH_H #include #endif static bool ucl_schema_validate (const ucl_object_t *schema, const ucl_object_t *obj, bool try_array, struct ucl_schema_error *err, const ucl_object_t *root, ucl_object_t *ext_ref); /* * Create validation error */ static void ucl_schema_create_error (struct ucl_schema_error *err, enum ucl_schema_error_code code, const ucl_object_t *obj, const char *fmt, ...) { va_list va; if (err != NULL) { err->code = code; err->obj = obj; va_start (va, fmt); vsnprintf (err->msg, sizeof (err->msg), fmt, va); va_end (va); } } /* * Check whether we have a pattern specified */ static const ucl_object_t * -ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern) +ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern, bool recursive) { const ucl_object_t *res = NULL; #ifdef HAVE_REGEX_H regex_t reg; const ucl_object_t *elt; ucl_object_iter_t iter = NULL; if (regcomp (®, pattern, REG_EXTENDED | REG_NOSUB) == 0) { - while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { - if (regexec (®, ucl_object_key (elt), 0, NULL, 0) == 0) { - res = elt; - break; + if (recursive) { + while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { + if (regexec (®, ucl_object_key (elt), 0, NULL, 0) == 0) { + res = elt; + break; + } } + } else { + if (regexec (®, ucl_object_key (obj), 0, NULL, 0) == 0) + res = obj; } regfree (®); } #endif return res; } /* * Check dependencies for an object */ static bool ucl_schema_validate_dependencies (const ucl_object_t *deps, const ucl_object_t *obj, struct ucl_schema_error *err, const ucl_object_t *root, ucl_object_t *ext_ref) { const ucl_object_t *elt, *cur, *cur_dep; ucl_object_iter_t iter = NULL, piter; bool ret = true; while (ret && (cur = ucl_object_iterate (deps, &iter, true)) != NULL) { elt = ucl_object_lookup (obj, ucl_object_key (cur)); if (elt != NULL) { /* Need to check dependencies */ if (cur->type == UCL_ARRAY) { piter = NULL; while (ret && (cur_dep = ucl_object_iterate (cur, &piter, true)) != NULL) { if (ucl_object_lookup (obj, ucl_object_tostring (cur_dep)) == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_MISSING_DEPENDENCY, elt, "dependency %s is missing for key %s", ucl_object_tostring (cur_dep), ucl_object_key (cur)); ret = false; break; } } } else if (cur->type == UCL_OBJECT) { ret = ucl_schema_validate (cur, obj, true, err, root, ext_ref); } } } return ret; } /* * Validate object */ static bool ucl_schema_validate_object (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err, const ucl_object_t *root, ucl_object_t *ext_ref) { const ucl_object_t *elt, *prop, *found, *additional_schema = NULL, *required = NULL, *pat, *pelt; ucl_object_iter_t iter = NULL, piter = NULL; bool ret = true, allow_additional = true; int64_t minmax; while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if (elt->type == UCL_OBJECT && strcmp (ucl_object_key (elt), "properties") == 0) { piter = NULL; while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) { found = ucl_object_lookup (obj, ucl_object_key (prop)); if (found) { ret = ucl_schema_validate (prop, found, true, err, root, ext_ref); } } } else if (strcmp (ucl_object_key (elt), "additionalProperties") == 0) { if (elt->type == UCL_BOOLEAN) { if (!ucl_object_toboolean (elt)) { /* Deny additional fields completely */ allow_additional = false; } } else if (elt->type == UCL_OBJECT) { /* Define validator for additional fields */ additional_schema = elt; } else { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "additionalProperties attribute is invalid in schema"); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "required") == 0) { if (elt->type == UCL_ARRAY) { required = elt; } else { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "required attribute is invalid in schema"); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "minProperties") == 0 && ucl_object_toint_safe (elt, &minmax)) { if (obj->len < minmax) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object has not enough properties: %u, minimum is: %u", obj->len, (unsigned)minmax); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "maxProperties") == 0 && ucl_object_toint_safe (elt, &minmax)) { if (obj->len > minmax) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object has too many properties: %u, maximum is: %u", obj->len, (unsigned)minmax); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) { + const ucl_object_t *vobj; + ucl_object_iter_t viter; piter = NULL; while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) { - found = ucl_schema_test_pattern (obj, ucl_object_key (prop)); - if (found) { - ret = ucl_schema_validate (prop, found, true, err, root, - ext_ref); + viter = NULL; + while (ret && (vobj = ucl_object_iterate (obj, &viter, true)) != NULL) { + found = ucl_schema_test_pattern (vobj, ucl_object_key (prop), false); + if (found) { + ret = ucl_schema_validate (prop, found, true, err, root, + ext_ref); + } } } } else if (elt->type == UCL_OBJECT && strcmp (ucl_object_key (elt), "dependencies") == 0) { ret = ucl_schema_validate_dependencies (elt, obj, err, root, ext_ref); } } if (ret) { /* Additional properties */ if (!allow_additional || additional_schema != NULL) { /* Check if we have exactly the same properties in schema and object */ iter = NULL; prop = ucl_object_lookup (schema, "properties"); while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { found = ucl_object_lookup (prop, ucl_object_key (elt)); if (found == NULL) { /* Try patternProperties */ piter = NULL; pat = ucl_object_lookup (schema, "patternProperties"); while ((pelt = ucl_object_iterate (pat, &piter, true)) != NULL) { - found = ucl_schema_test_pattern (obj, ucl_object_key (pelt)); + found = ucl_schema_test_pattern (obj, ucl_object_key (pelt), true); if (found != NULL) { break; } } } if (found == NULL) { if (!allow_additional) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object has non-allowed property %s", ucl_object_key (elt)); ret = false; break; } else if (additional_schema != NULL) { if (!ucl_schema_validate (additional_schema, elt, true, err, root, ext_ref)) { ret = false; break; } } } } } /* Required properties */ if (required != NULL) { iter = NULL; while ((elt = ucl_object_iterate (required, &iter, true)) != NULL) { if (ucl_object_lookup (obj, ucl_object_tostring (elt)) == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_MISSING_PROPERTY, obj, "object has missing property %s", ucl_object_tostring (elt)); ret = false; break; } } } } return ret; } static bool ucl_schema_validate_number (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err) { const ucl_object_t *elt, *test; ucl_object_iter_t iter = NULL; bool ret = true, exclusive = false; double constraint, val; const double alpha = 1e-16; while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "multipleOf") == 0) { constraint = ucl_object_todouble (elt); if (constraint <= 0) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "multipleOf must be greater than zero"); ret = false; break; } val = ucl_object_todouble (obj); if (fabs (remainder (val, constraint)) > alpha) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "number %.4f is not multiple of %.4f, remainder is %.7f", val, constraint); ret = false; break; } } else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "maximum") == 0) { constraint = ucl_object_todouble (elt); test = ucl_object_lookup (schema, "exclusiveMaximum"); if (test && test->type == UCL_BOOLEAN) { exclusive = ucl_object_toboolean (test); } val = ucl_object_todouble (obj); if (val > constraint || (exclusive && val >= constraint)) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "number is too big: %.3f, maximum is: %.3f", val, constraint); ret = false; break; } } else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "minimum") == 0) { constraint = ucl_object_todouble (elt); test = ucl_object_lookup (schema, "exclusiveMinimum"); if (test && test->type == UCL_BOOLEAN) { exclusive = ucl_object_toboolean (test); } val = ucl_object_todouble (obj); if (val < constraint || (exclusive && val <= constraint)) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "number is too small: %.3f, minimum is: %.3f", val, constraint); ret = false; break; } } } return ret; } static bool ucl_schema_validate_string (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err) { const ucl_object_t *elt; ucl_object_iter_t iter = NULL; bool ret = true; int64_t constraint; #ifdef HAVE_REGEX_H regex_t re; #endif while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if (elt->type == UCL_INT && strcmp (ucl_object_key (elt), "maxLength") == 0) { constraint = ucl_object_toint (elt); if (obj->len > constraint) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "string is too big: %.3f, maximum is: %.3f", obj->len, constraint); ret = false; break; } } else if (elt->type == UCL_INT && strcmp (ucl_object_key (elt), "minLength") == 0) { constraint = ucl_object_toint (elt); if (obj->len < constraint) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "string is too short: %.3f, minimum is: %.3f", obj->len, constraint); ret = false; break; } } #ifdef HAVE_REGEX_H else if (elt->type == UCL_STRING && strcmp (ucl_object_key (elt), "pattern") == 0) { if (regcomp (&re, ucl_object_tostring (elt), REG_EXTENDED | REG_NOSUB) != 0) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "cannot compile pattern %s", ucl_object_tostring (elt)); ret = false; break; } if (regexec (&re, ucl_object_tostring (obj), 0, NULL, 0) != 0) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "string doesn't match regexp %s", ucl_object_tostring (elt)); ret = false; } regfree (&re); } #endif } return ret; } struct ucl_compare_node { const ucl_object_t *obj; TREE_ENTRY(ucl_compare_node) link; struct ucl_compare_node *next; }; typedef TREE_HEAD(_tree, ucl_compare_node) ucl_compare_tree_t; TREE_DEFINE(ucl_compare_node, link) static int ucl_schema_elt_compare (struct ucl_compare_node *n1, struct ucl_compare_node *n2) { const ucl_object_t *o1 = n1->obj, *o2 = n2->obj; return ucl_object_compare (o1, o2); } static bool ucl_schema_array_is_unique (const ucl_object_t *obj, struct ucl_schema_error *err) { ucl_compare_tree_t tree = TREE_INITIALIZER (ucl_schema_elt_compare); ucl_object_iter_t iter = NULL; const ucl_object_t *elt; struct ucl_compare_node *node, test, *nodes = NULL, *tmp; bool ret = true; while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { test.obj = elt; node = TREE_FIND (&tree, ucl_compare_node, link, &test); if (node != NULL) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, elt, "duplicate values detected while uniqueItems is true"); ret = false; break; } node = calloc (1, sizeof (*node)); if (node == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_UNKNOWN, elt, "cannot allocate tree node"); ret = false; break; } node->obj = elt; TREE_INSERT (&tree, ucl_compare_node, link, node); LL_PREPEND (nodes, node); } LL_FOREACH_SAFE (nodes, node, tmp) { free (node); } return ret; } static bool ucl_schema_validate_array (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err, const ucl_object_t *root, ucl_object_t *ext_ref) { const ucl_object_t *elt, *it, *found, *additional_schema = NULL, *first_unvalidated = NULL; ucl_object_iter_t iter = NULL, piter = NULL; bool ret = true, allow_additional = true, need_unique = false; int64_t minmax; unsigned int idx = 0; while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if (strcmp (ucl_object_key (elt), "items") == 0) { if (elt->type == UCL_ARRAY) { found = ucl_array_head (obj); while (ret && (it = ucl_object_iterate (elt, &piter, true)) != NULL) { if (found) { ret = ucl_schema_validate (it, found, false, err, root, ext_ref); found = ucl_array_find_index (obj, ++idx); } } if (found != NULL) { /* The first element that is not validated */ first_unvalidated = found; } } else if (elt->type == UCL_OBJECT) { /* Validate all items using the specified schema */ while (ret && (it = ucl_object_iterate (obj, &piter, true)) != NULL) { ret = ucl_schema_validate (elt, it, false, err, root, ext_ref); } } else { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "items attribute is invalid in schema"); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "additionalItems") == 0) { if (elt->type == UCL_BOOLEAN) { if (!ucl_object_toboolean (elt)) { /* Deny additional fields completely */ allow_additional = false; } } else if (elt->type == UCL_OBJECT) { /* Define validator for additional fields */ additional_schema = elt; } else { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "additionalItems attribute is invalid in schema"); ret = false; break; } } else if (elt->type == UCL_BOOLEAN && strcmp (ucl_object_key (elt), "uniqueItems") == 0) { need_unique = ucl_object_toboolean (elt); } else if (strcmp (ucl_object_key (elt), "minItems") == 0 && ucl_object_toint_safe (elt, &minmax)) { if (obj->len < minmax) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "array has not enough items: %u, minimum is: %u", obj->len, (unsigned)minmax); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "maxItems") == 0 && ucl_object_toint_safe (elt, &minmax)) { if (obj->len > minmax) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "array has too many items: %u, maximum is: %u", obj->len, (unsigned)minmax); ret = false; break; } } } if (ret) { /* Additional properties */ if (!allow_additional || additional_schema != NULL) { if (first_unvalidated != NULL) { if (!allow_additional) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "array has undefined item"); ret = false; } else if (additional_schema != NULL) { elt = ucl_array_find_index (obj, idx); while (elt) { if (!ucl_schema_validate (additional_schema, elt, false, err, root, ext_ref)) { ret = false; break; } elt = ucl_array_find_index (obj, idx ++); } } } } /* Required properties */ if (ret && need_unique) { ret = ucl_schema_array_is_unique (obj, err); } } return ret; } /* * Returns whether this object is allowed for this type */ static bool ucl_schema_type_is_allowed (const ucl_object_t *type, const ucl_object_t *obj, struct ucl_schema_error *err) { ucl_object_iter_t iter = NULL; const ucl_object_t *elt; const char *type_str; ucl_type_t t; if (type == NULL) { /* Any type is allowed */ return true; } if (type->type == UCL_ARRAY) { /* One of allowed types */ while ((elt = ucl_object_iterate (type, &iter, true)) != NULL) { if (ucl_schema_type_is_allowed (elt, obj, err)) { return true; } } } else if (type->type == UCL_STRING) { type_str = ucl_object_tostring (type); if (!ucl_object_string_to_type (type_str, &t)) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, type, "Type attribute is invalid in schema"); return false; } if (obj->type != t) { /* Some types are actually compatible */ if (obj->type == UCL_TIME && t == UCL_FLOAT) { return true; } else if (obj->type == UCL_INT && t == UCL_FLOAT) { return true; } else { ucl_schema_create_error (err, UCL_SCHEMA_TYPE_MISMATCH, obj, "Invalid type of %s, expected %s", ucl_object_type_to_string (obj->type), ucl_object_type_to_string (t)); } } else { /* Types are equal */ return true; } } return false; } /* * Check if object is equal to one of elements of enum */ static bool ucl_schema_validate_enum (const ucl_object_t *en, const ucl_object_t *obj, struct ucl_schema_error *err) { ucl_object_iter_t iter = NULL; const ucl_object_t *elt; bool ret = false; while ((elt = ucl_object_iterate (en, &iter, true)) != NULL) { if (ucl_object_compare (elt, obj) == 0) { ret = true; break; } } if (!ret) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object is not one of enumerated patterns"); } return ret; } /* * Check a single ref component */ static const ucl_object_t * ucl_schema_resolve_ref_component (const ucl_object_t *cur, const char *refc, int len, struct ucl_schema_error *err) { const ucl_object_t *res = NULL; char *err_str; int num, i; if (cur->type == UCL_OBJECT) { /* Find a key inside an object */ res = ucl_object_lookup_len (cur, refc, len); if (res == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur, "reference %s is invalid, missing path component", refc); return NULL; } } else if (cur->type == UCL_ARRAY) { /* We must figure out a number inside array */ num = strtoul (refc, &err_str, 10); if (err_str != NULL && *err_str != '/' && *err_str != '\0') { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur, "reference %s is invalid, invalid item number", refc); return NULL; } res = ucl_array_head (cur); i = 0; while (res != NULL) { if (i == num) { break; } res = res->next; } if (res == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur, "reference %s is invalid, item number %d does not exist", refc, num); return NULL; } } else { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res, "reference %s is invalid, contains primitive object in the path", refc); return NULL; } return res; } /* * Find reference schema */ static const ucl_object_t * ucl_schema_resolve_ref (const ucl_object_t *root, const char *ref, struct ucl_schema_error *err, ucl_object_t *ext_ref, ucl_object_t const ** nroot) { UT_string *url_err = NULL; struct ucl_parser *parser; const ucl_object_t *res = NULL, *ext_obj = NULL; ucl_object_t *url_obj; const char *p, *c, *hash_ptr = NULL; char *url_copy = NULL; unsigned char *url_buf; size_t url_buflen; if (ref[0] != '#') { hash_ptr = strrchr (ref, '#'); if (hash_ptr) { url_copy = malloc (hash_ptr - ref + 1); if (url_copy == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_INTERNAL_ERROR, root, "cannot allocate memory"); return NULL; } ucl_strlcpy (url_copy, ref, hash_ptr - ref + 1); p = url_copy; } else { /* Full URL */ p = ref; } ext_obj = ucl_object_lookup (ext_ref, p); if (ext_obj == NULL) { if (ucl_strnstr (p, "://", strlen (p)) != NULL) { if (!ucl_fetch_url (p, &url_buf, &url_buflen, &url_err, true)) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, root, "cannot fetch reference %s: %s", p, url_err != NULL ? utstring_body (url_err) : "unknown"); free (url_copy); return NULL; } } else { if (!ucl_fetch_file (p, &url_buf, &url_buflen, &url_err, true)) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, root, "cannot fetch reference %s: %s", p, url_err != NULL ? utstring_body (url_err) : "unknown"); free (url_copy); return NULL; } } parser = ucl_parser_new (0); if (!ucl_parser_add_chunk (parser, url_buf, url_buflen)) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, root, "cannot fetch reference %s: %s", p, ucl_parser_get_error (parser)); ucl_parser_free (parser); free (url_copy); return NULL; } url_obj = ucl_parser_get_object (parser); ext_obj = url_obj; ucl_object_insert_key (ext_ref, url_obj, p, 0, true); free (url_buf); } free (url_copy); if (hash_ptr) { p = hash_ptr + 1; } else { p = ""; } } else { p = ref + 1; } res = ext_obj != NULL ? ext_obj : root; *nroot = res; if (*p == '/') { p++; } else if (*p == '\0') { return res; } c = p; while (*p != '\0') { if (*p == '/') { if (p - c == 0) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res, "reference %s is invalid, empty path component", ref); return NULL; } /* Now we have some url part, so we need to figure out where we are */ res = ucl_schema_resolve_ref_component (res, c, p - c, err); if (res == NULL) { return NULL; } c = p + 1; } p ++; } if (p - c != 0) { res = ucl_schema_resolve_ref_component (res, c, p - c, err); } if (res == NULL || res->type != UCL_OBJECT) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res, "reference %s is invalid, cannot find specified object", ref); return NULL; } return res; } static bool ucl_schema_validate_values (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err) { const ucl_object_t *elt, *cur; int64_t constraint, i; elt = ucl_object_lookup (schema, "maxValues"); if (elt != NULL && elt->type == UCL_INT) { constraint = ucl_object_toint (elt); cur = obj; i = 0; while (cur) { if (i > constraint) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object has more values than defined: %ld", (long int)constraint); return false; } i ++; cur = cur->next; } } elt = ucl_object_lookup (schema, "minValues"); if (elt != NULL && elt->type == UCL_INT) { constraint = ucl_object_toint (elt); cur = obj; i = 0; while (cur) { if (i >= constraint) { break; } i ++; cur = cur->next; } if (i < constraint) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object has less values than defined: %ld", (long int)constraint); return false; } } return true; } static bool ucl_schema_validate (const ucl_object_t *schema, const ucl_object_t *obj, bool try_array, struct ucl_schema_error *err, const ucl_object_t *root, ucl_object_t *external_refs) { const ucl_object_t *elt, *cur, *ref_root; ucl_object_iter_t iter = NULL; bool ret; if (schema->type != UCL_OBJECT) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, schema, "schema is %s instead of object", ucl_object_type_to_string (schema->type)); return false; } if (try_array) { /* * Special case for multiple values */ if (!ucl_schema_validate_values (schema, obj, err)) { return false; } LL_FOREACH (obj, cur) { if (!ucl_schema_validate (schema, cur, false, err, root, external_refs)) { return false; } } return true; } elt = ucl_object_lookup (schema, "enum"); if (elt != NULL && elt->type == UCL_ARRAY) { if (!ucl_schema_validate_enum (elt, obj, err)) { return false; } } elt = ucl_object_lookup (schema, "allOf"); if (elt != NULL && elt->type == UCL_ARRAY) { iter = NULL; while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) { ret = ucl_schema_validate (cur, obj, true, err, root, external_refs); if (!ret) { return false; } } } elt = ucl_object_lookup (schema, "anyOf"); if (elt != NULL && elt->type == UCL_ARRAY) { iter = NULL; while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) { ret = ucl_schema_validate (cur, obj, true, err, root, external_refs); if (ret) { break; } } if (!ret) { return false; } else { /* Reset error */ err->code = UCL_SCHEMA_OK; } } elt = ucl_object_lookup (schema, "oneOf"); if (elt != NULL && elt->type == UCL_ARRAY) { iter = NULL; ret = false; while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) { if (!ret) { ret = ucl_schema_validate (cur, obj, true, err, root, external_refs); } else if (ucl_schema_validate (cur, obj, true, err, root, external_refs)) { ret = false; break; } } if (!ret) { return false; } } elt = ucl_object_lookup (schema, "not"); if (elt != NULL && elt->type == UCL_OBJECT) { if (ucl_schema_validate (elt, obj, true, err, root, external_refs)) { return false; } else { /* Reset error */ err->code = UCL_SCHEMA_OK; } } elt = ucl_object_lookup (schema, "$ref"); if (elt != NULL) { ref_root = root; cur = ucl_schema_resolve_ref (root, ucl_object_tostring (elt), err, external_refs, &ref_root); if (cur == NULL) { return false; } if (!ucl_schema_validate (cur, obj, try_array, err, ref_root, external_refs)) { return false; } } elt = ucl_object_lookup (schema, "type"); if (!ucl_schema_type_is_allowed (elt, obj, err)) { return false; } switch (obj->type) { case UCL_OBJECT: return ucl_schema_validate_object (schema, obj, err, root, external_refs); break; case UCL_ARRAY: return ucl_schema_validate_array (schema, obj, err, root, external_refs); break; case UCL_INT: case UCL_FLOAT: return ucl_schema_validate_number (schema, obj, err); break; case UCL_STRING: return ucl_schema_validate_string (schema, obj, err); break; default: break; } return true; } bool ucl_object_validate (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err) { return ucl_object_validate_root_ext (schema, obj, schema, NULL, err); } bool ucl_object_validate_root (const ucl_object_t *schema, const ucl_object_t *obj, const ucl_object_t *root, struct ucl_schema_error *err) { return ucl_object_validate_root_ext (schema, obj, root, NULL, err); } bool ucl_object_validate_root_ext (const ucl_object_t *schema, const ucl_object_t *obj, const ucl_object_t *root, ucl_object_t *ext_refs, struct ucl_schema_error *err) { bool ret, need_unref = false; if (ext_refs == NULL) { ext_refs = ucl_object_typed_new (UCL_OBJECT); need_unref = true; } ret = ucl_schema_validate (schema, obj, true, err, root, ext_refs); if (need_unref) { ucl_object_unref (ext_refs); } return ret; } Index: head/contrib/libucl/src/ucl_util.c =================================================================== --- head/contrib/libucl/src/ucl_util.c (revision 306543) +++ head/contrib/libucl/src/ucl_util.c (revision 306544) @@ -1,3495 +1,3510 @@ /* Copyright (c) 2013, Vsevolod Stakhov * Copyright (c) 2015 Allan Jude * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "ucl.h" #include "ucl_internal.h" #include "ucl_chartable.h" #include "kvec.h" #include #include /* for snprintf */ #ifndef _WIN32 #include #endif #ifdef HAVE_LIBGEN_H #include /* For dirname */ #endif typedef kvec_t(ucl_object_t *) ucl_array_t; #define UCL_ARRAY_GET(ar, obj) ucl_array_t *ar = \ (ucl_array_t *)((obj) != NULL ? (obj)->value.av : NULL) #ifdef HAVE_OPENSSL #include #include #include #include #include #endif #ifdef CURL_FOUND /* Seems to be broken */ #define CURL_DISABLE_TYPECHECK 1 #include #endif #ifdef HAVE_FETCH_H #include #endif #ifdef _WIN32 #include #ifndef PROT_READ #define PROT_READ 1 #endif #ifndef PROT_WRITE #define PROT_WRITE 2 #endif #ifndef PROT_READWRITE #define PROT_READWRITE 3 #endif #ifndef MAP_SHARED #define MAP_SHARED 1 #endif #ifndef MAP_PRIVATE #define MAP_PRIVATE 2 #endif #ifndef MAP_FAILED #define MAP_FAILED ((void *) -1) #endif #ifdef _WIN32 #include #define NBBY CHAR_BIT #endif static void *ucl_mmap(char *addr, size_t length, int prot, int access, int fd, off_t offset) { void *map = NULL; HANDLE handle = INVALID_HANDLE_VALUE; switch (prot) { default: case PROT_READ: { handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READONLY, 0, length, 0); if (!handle) break; map = (void *) MapViewOfFile(handle, FILE_MAP_READ, 0, 0, length); CloseHandle(handle); break; } case PROT_WRITE: { handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0); if (!handle) break; map = (void *) MapViewOfFile(handle, FILE_MAP_WRITE, 0, 0, length); CloseHandle(handle); break; } case PROT_READWRITE: { handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0); if (!handle) break; map = (void *) MapViewOfFile(handle, FILE_MAP_ALL_ACCESS, 0, 0, length); CloseHandle(handle); break; } } if (map == (void *) NULL) { return (void *) MAP_FAILED; } return (void *) ((char *) map + offset); } static int ucl_munmap(void *map,size_t length) { if (!UnmapViewOfFile(map)) { return(-1); } return(0); } static char* ucl_realpath(const char *path, char *resolved_path) { char *p; char tmp[MAX_PATH + 1]; strncpy(tmp, path, sizeof(tmp)-1); p = tmp; while(*p) { if (*p == '/') *p = '\\'; p++; } return _fullpath(resolved_path, tmp, MAX_PATH); } #else #define ucl_mmap mmap #define ucl_munmap munmap #define ucl_realpath realpath #endif typedef void (*ucl_object_dtor) (ucl_object_t *obj); static void ucl_object_free_internal (ucl_object_t *obj, bool allow_rec, ucl_object_dtor dtor); static void ucl_object_dtor_unref (ucl_object_t *obj); static void ucl_object_dtor_free (ucl_object_t *obj) { if (obj->trash_stack[UCL_TRASH_KEY] != NULL) { UCL_FREE (obj->hh.keylen, obj->trash_stack[UCL_TRASH_KEY]); } if (obj->trash_stack[UCL_TRASH_VALUE] != NULL) { UCL_FREE (obj->len, obj->trash_stack[UCL_TRASH_VALUE]); } /* Do not free ephemeral objects */ if ((obj->flags & UCL_OBJECT_EPHEMERAL) == 0) { if (obj->type != UCL_USERDATA) { UCL_FREE (sizeof (ucl_object_t), obj); } else { struct ucl_object_userdata *ud = (struct ucl_object_userdata *)obj; if (ud->dtor) { ud->dtor (obj->value.ud); } UCL_FREE (sizeof (*ud), obj); } } } /* * This is a helper function that performs exactly the same as * `ucl_object_unref` but it doesn't iterate over elements allowing * to use it for individual elements of arrays and multiple values */ static void ucl_object_dtor_unref_single (ucl_object_t *obj) { if (obj != NULL) { #ifdef HAVE_ATOMIC_BUILTINS unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1); if (rc == 0) { #else if (--obj->ref == 0) { #endif ucl_object_free_internal (obj, false, ucl_object_dtor_unref); } } } static void ucl_object_dtor_unref (ucl_object_t *obj) { if (obj->ref == 0) { ucl_object_dtor_free (obj); } else { /* This may cause dtor unref being called one more time */ ucl_object_dtor_unref_single (obj); } } static void ucl_object_free_internal (ucl_object_t *obj, bool allow_rec, ucl_object_dtor dtor) { ucl_object_t *tmp, *sub; while (obj != NULL) { if (obj->type == UCL_ARRAY) { UCL_ARRAY_GET (vec, obj); unsigned int i; if (vec != NULL) { for (i = 0; i < vec->n; i ++) { sub = kv_A (*vec, i); if (sub != NULL) { tmp = sub; while (sub) { tmp = sub->next; dtor (sub); sub = tmp; } } } kv_destroy (*vec); UCL_FREE (sizeof (*vec), vec); } obj->value.av = NULL; } else if (obj->type == UCL_OBJECT) { if (obj->value.ov != NULL) { ucl_hash_destroy (obj->value.ov, (ucl_hash_free_func)dtor); } obj->value.ov = NULL; } tmp = obj->next; dtor (obj); obj = tmp; if (!allow_rec) { break; } } } void ucl_object_free (ucl_object_t *obj) { ucl_object_free_internal (obj, true, ucl_object_dtor_free); } size_t ucl_unescape_json_string (char *str, size_t len) { char *t = str, *h = str; int i, uval; if (len <= 1) { return len; } /* t is target (tortoise), h is source (hare) */ while (len) { if (*h == '\\') { h ++; if (len == 1) { /* * If \ is last, then do not try to go further * Issue: #74 */ len --; *t++ = '\\'; continue; } switch (*h) { case 'n': *t++ = '\n'; break; case 'r': *t++ = '\r'; break; case 'b': *t++ = '\b'; break; case 't': *t++ = '\t'; break; case 'f': *t++ = '\f'; break; case '\\': *t++ = '\\'; break; case '"': *t++ = '"'; break; case 'u': /* Unicode escape */ uval = 0; h ++; /* u character */ len --; if (len > 3) { for (i = 0; i < 4; i++) { uval <<= 4; if (isdigit (h[i])) { uval += h[i] - '0'; } else if (h[i] >= 'a' && h[i] <= 'f') { uval += h[i] - 'a' + 10; } else if (h[i] >= 'A' && h[i] <= 'F') { uval += h[i] - 'A' + 10; } else { break; } } /* Encode */ if(uval < 0x80) { t[0] = (char)uval; t ++; } else if(uval < 0x800) { t[0] = 0xC0 + ((uval & 0x7C0) >> 6); t[1] = 0x80 + ((uval & 0x03F)); t += 2; } else if(uval < 0x10000) { t[0] = 0xE0 + ((uval & 0xF000) >> 12); t[1] = 0x80 + ((uval & 0x0FC0) >> 6); t[2] = 0x80 + ((uval & 0x003F)); t += 3; } #if 0 /* It's not actually supported now */ else if(uval <= 0x10FFFF) { t[0] = 0xF0 + ((uval & 0x1C0000) >> 18); t[1] = 0x80 + ((uval & 0x03F000) >> 12); t[2] = 0x80 + ((uval & 0x000FC0) >> 6); t[3] = 0x80 + ((uval & 0x00003F)); t += 4; } #endif else { *t++ = '?'; } /* Consume 4 characters of source */ h += 4; len -= 4; if (len > 0) { len --; /* for '\' character */ } continue; } else { *t++ = 'u'; } break; default: *t++ = *h; break; } h ++; len --; } else { *t++ = *h++; } if (len > 0) { len --; } } *t = '\0'; return (t - str); } char * ucl_copy_key_trash (const ucl_object_t *obj) { ucl_object_t *deconst; if (obj == NULL) { return NULL; } if (obj->trash_stack[UCL_TRASH_KEY] == NULL && obj->key != NULL) { deconst = __DECONST (ucl_object_t *, obj); deconst->trash_stack[UCL_TRASH_KEY] = malloc (obj->keylen + 1); if (deconst->trash_stack[UCL_TRASH_KEY] != NULL) { memcpy (deconst->trash_stack[UCL_TRASH_KEY], obj->key, obj->keylen); deconst->trash_stack[UCL_TRASH_KEY][obj->keylen] = '\0'; } deconst->key = obj->trash_stack[UCL_TRASH_KEY]; deconst->flags |= UCL_OBJECT_ALLOCATED_KEY; } return obj->trash_stack[UCL_TRASH_KEY]; } char * ucl_copy_value_trash (const ucl_object_t *obj) { ucl_object_t *deconst; if (obj == NULL) { return NULL; } if (obj->trash_stack[UCL_TRASH_VALUE] == NULL) { deconst = __DECONST (ucl_object_t *, obj); if (obj->type == UCL_STRING) { /* Special case for strings */ if (obj->flags & UCL_OBJECT_BINARY) { deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len); if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) { memcpy (deconst->trash_stack[UCL_TRASH_VALUE], obj->value.sv, obj->len); deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE]; } } else { deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len + 1); if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) { memcpy (deconst->trash_stack[UCL_TRASH_VALUE], obj->value.sv, obj->len); deconst->trash_stack[UCL_TRASH_VALUE][obj->len] = '\0'; deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE]; } } } else { /* Just emit value in json notation */ deconst->trash_stack[UCL_TRASH_VALUE] = ucl_object_emit_single_json (obj); deconst->len = strlen (obj->trash_stack[UCL_TRASH_VALUE]); } deconst->flags |= UCL_OBJECT_ALLOCATED_VALUE; } return obj->trash_stack[UCL_TRASH_VALUE]; } ucl_object_t* ucl_parser_get_object (struct ucl_parser *parser) { if (parser->state != UCL_STATE_ERROR && parser->top_obj != NULL) { return ucl_object_ref (parser->top_obj); } return NULL; } void ucl_parser_free (struct ucl_parser *parser) { struct ucl_stack *stack, *stmp; struct ucl_macro *macro, *mtmp; struct ucl_chunk *chunk, *ctmp; struct ucl_pubkey *key, *ktmp; struct ucl_variable *var, *vtmp; ucl_object_t *tr, *trtmp; if (parser == NULL) { return; } if (parser->top_obj != NULL) { ucl_object_unref (parser->top_obj); } if (parser->includepaths != NULL) { ucl_object_unref (parser->includepaths); } LL_FOREACH_SAFE (parser->stack, stack, stmp) { free (stack); } HASH_ITER (hh, parser->macroes, macro, mtmp) { free (macro->name); HASH_DEL (parser->macroes, macro); UCL_FREE (sizeof (struct ucl_macro), macro); } LL_FOREACH_SAFE (parser->chunks, chunk, ctmp) { UCL_FREE (sizeof (struct ucl_chunk), chunk); } LL_FOREACH_SAFE (parser->keys, key, ktmp) { UCL_FREE (sizeof (struct ucl_pubkey), key); } LL_FOREACH_SAFE (parser->variables, var, vtmp) { free (var->value); free (var->var); UCL_FREE (sizeof (struct ucl_variable), var); } LL_FOREACH_SAFE (parser->trash_objs, tr, trtmp) { ucl_object_free_internal (tr, false, ucl_object_dtor_free); } if (parser->err != NULL) { utstring_free (parser->err); } if (parser->cur_file) { free (parser->cur_file); } if (parser->comments) { ucl_object_unref (parser->comments); } UCL_FREE (sizeof (struct ucl_parser), parser); } const char * ucl_parser_get_error(struct ucl_parser *parser) { if (parser == NULL) { return NULL; } if (parser->err == NULL) { return NULL; } return utstring_body (parser->err); } int ucl_parser_get_error_code(struct ucl_parser *parser) { if (parser == NULL) { return 0; } return parser->err_code; } unsigned ucl_parser_get_column(struct ucl_parser *parser) { if (parser == NULL || parser->chunks == NULL) { return 0; } return parser->chunks->column; } unsigned ucl_parser_get_linenum(struct ucl_parser *parser) { if (parser == NULL || parser->chunks == NULL) { return 0; } return parser->chunks->line; } void ucl_parser_clear_error(struct ucl_parser *parser) { if (parser != NULL && parser->err != NULL) { utstring_free(parser->err); parser->err = NULL; parser->err_code = 0; } } bool ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len) { #ifndef HAVE_OPENSSL ucl_create_err (&parser->err, "cannot check signatures without openssl"); return false; #else # if (OPENSSL_VERSION_NUMBER < 0x10000000L) ucl_create_err (&parser->err, "cannot check signatures, openssl version is unsupported"); return EXIT_FAILURE; # else struct ucl_pubkey *nkey; BIO *mem; mem = BIO_new_mem_buf ((void *)key, len); nkey = UCL_ALLOC (sizeof (struct ucl_pubkey)); if (nkey == NULL) { ucl_create_err (&parser->err, "cannot allocate memory for key"); return false; } nkey->key = PEM_read_bio_PUBKEY (mem, &nkey->key, NULL, NULL); BIO_free (mem); if (nkey->key == NULL) { UCL_FREE (sizeof (struct ucl_pubkey), nkey); ucl_create_err (&parser->err, "%s", ERR_error_string (ERR_get_error (), NULL)); return false; } LL_PREPEND (parser->keys, nkey); # endif #endif return true; } #ifdef CURL_FOUND struct ucl_curl_cbdata { unsigned char *buf; size_t buflen; }; static size_t ucl_curl_write_callback (void* contents, size_t size, size_t nmemb, void* ud) { struct ucl_curl_cbdata *cbdata = ud; size_t realsize = size * nmemb; cbdata->buf = realloc (cbdata->buf, cbdata->buflen + realsize + 1); if (cbdata->buf == NULL) { return 0; } memcpy (&(cbdata->buf[cbdata->buflen]), contents, realsize); cbdata->buflen += realsize; cbdata->buf[cbdata->buflen] = 0; return realsize; } #endif /** * Fetch a url and save results to the memory buffer * @param url url to fetch * @param len length of url * @param buf target buffer * @param buflen target length * @return */ bool ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen, UT_string **err, bool must_exist) { #ifdef HAVE_FETCH_H struct url *fetch_url; struct url_stat us; FILE *in; fetch_url = fetchParseURL (url); if (fetch_url == NULL) { ucl_create_err (err, "invalid URL %s: %s", url, strerror (errno)); return false; } if ((in = fetchXGet (fetch_url, &us, "")) == NULL) { if (!must_exist) { ucl_create_err (err, "cannot fetch URL %s: %s", url, strerror (errno)); } fetchFreeURL (fetch_url); return false; } *buflen = us.size; *buf = malloc (*buflen); if (*buf == NULL) { ucl_create_err (err, "cannot allocate buffer for URL %s: %s", url, strerror (errno)); fclose (in); fetchFreeURL (fetch_url); return false; } if (fread (*buf, *buflen, 1, in) != 1) { ucl_create_err (err, "cannot read URL %s: %s", url, strerror (errno)); fclose (in); fetchFreeURL (fetch_url); return false; } fetchFreeURL (fetch_url); return true; #elif defined(CURL_FOUND) CURL *curl; int r; struct ucl_curl_cbdata cbdata; curl = curl_easy_init (); if (curl == NULL) { ucl_create_err (err, "CURL interface is broken"); return false; } if ((r = curl_easy_setopt (curl, CURLOPT_URL, url)) != CURLE_OK) { ucl_create_err (err, "invalid URL %s: %s", url, curl_easy_strerror (r)); curl_easy_cleanup (curl); return false; } curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ucl_curl_write_callback); cbdata.buf = NULL; cbdata.buflen = 0; curl_easy_setopt (curl, CURLOPT_WRITEDATA, &cbdata); if ((r = curl_easy_perform (curl)) != CURLE_OK) { if (!must_exist) { ucl_create_err (err, "error fetching URL %s: %s", url, curl_easy_strerror (r)); } curl_easy_cleanup (curl); if (cbdata.buf) { free (cbdata.buf); } return false; } *buf = cbdata.buf; *buflen = cbdata.buflen; return true; #else ucl_create_err (err, "URL support is disabled"); return false; #endif } /** * Fetch a file and save results to the memory buffer * @param filename filename to fetch * @param len length of filename * @param buf target buffer * @param buflen target length * @return */ bool ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *buflen, UT_string **err, bool must_exist) { int fd; struct stat st; if (stat (filename, &st) == -1 || !S_ISREG (st.st_mode)) { if (must_exist) { ucl_create_err (err, "cannot stat file %s: %s", filename, strerror (errno)); } return false; } if (st.st_size == 0) { /* Do not map empty files */ *buf = NULL; *buflen = 0; } else { if ((fd = open (filename, O_RDONLY)) == -1) { ucl_create_err (err, "cannot open file %s: %s", filename, strerror (errno)); return false; } if ((*buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { close (fd); ucl_create_err (err, "cannot mmap file %s: %s", filename, strerror (errno)); *buf = NULL; return false; } *buflen = st.st_size; close (fd); } return true; } #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L) static inline bool ucl_sig_check (const unsigned char *data, size_t datalen, const unsigned char *sig, size_t siglen, struct ucl_parser *parser) { struct ucl_pubkey *key; char dig[EVP_MAX_MD_SIZE]; unsigned int diglen; EVP_PKEY_CTX *key_ctx; EVP_MD_CTX *sign_ctx = NULL; sign_ctx = EVP_MD_CTX_create (); LL_FOREACH (parser->keys, key) { key_ctx = EVP_PKEY_CTX_new (key->key, NULL); if (key_ctx != NULL) { if (EVP_PKEY_verify_init (key_ctx) <= 0) { EVP_PKEY_CTX_free (key_ctx); continue; } if (EVP_PKEY_CTX_set_rsa_padding (key_ctx, RSA_PKCS1_PADDING) <= 0) { EVP_PKEY_CTX_free (key_ctx); continue; } if (EVP_PKEY_CTX_set_signature_md (key_ctx, EVP_sha256 ()) <= 0) { EVP_PKEY_CTX_free (key_ctx); continue; } EVP_DigestInit (sign_ctx, EVP_sha256 ()); EVP_DigestUpdate (sign_ctx, data, datalen); EVP_DigestFinal (sign_ctx, dig, &diglen); if (EVP_PKEY_verify (key_ctx, sig, siglen, dig, diglen) == 1) { EVP_MD_CTX_destroy (sign_ctx); EVP_PKEY_CTX_free (key_ctx); return true; } EVP_PKEY_CTX_free (key_ctx); } } EVP_MD_CTX_destroy (sign_ctx); return false; } #endif struct ucl_include_params { bool check_signature; bool must_exist; bool use_glob; bool use_prefix; bool soft_fail; bool allow_glob; unsigned priority; enum ucl_duplicate_strategy strat; enum ucl_parse_type parse_type; const char *prefix; const char *target; }; /** * Include an url to configuration * @param data * @param len * @param parser * @param err * @return */ static bool ucl_include_url (const unsigned char *data, size_t len, struct ucl_parser *parser, struct ucl_include_params *params) { bool res; unsigned char *buf = NULL; size_t buflen = 0; struct ucl_chunk *chunk; char urlbuf[PATH_MAX]; int prev_state; snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data); if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, params->must_exist)) { return !params->must_exist; } if (params->check_signature) { #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L) unsigned char *sigbuf = NULL; size_t siglen = 0; /* We need to check signature first */ snprintf (urlbuf, sizeof (urlbuf), "%.*s.sig", (int)len, data); if (!ucl_fetch_url (urlbuf, &sigbuf, &siglen, &parser->err, true)) { return false; } if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) { ucl_create_err (&parser->err, "cannot verify url %s: %s", urlbuf, ERR_error_string (ERR_get_error (), NULL)); if (siglen > 0) { ucl_munmap (sigbuf, siglen); } return false; } if (siglen > 0) { ucl_munmap (sigbuf, siglen); } #endif } prev_state = parser->state; parser->state = UCL_STATE_INIT; res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority, params->strat, params->parse_type); if (res == true) { /* Remove chunk from the stack */ chunk = parser->chunks; if (chunk != NULL) { parser->chunks = chunk->next; UCL_FREE (sizeof (struct ucl_chunk), chunk); } } parser->state = prev_state; free (buf); return res; } /** * Include a single file to the parser * @param data * @param len * @param parser * @param check_signature * @param must_exist * @param allow_glob * @param priority * @return */ static bool ucl_include_file_single (const unsigned char *data, size_t len, struct ucl_parser *parser, struct ucl_include_params *params) { bool res; struct ucl_chunk *chunk; unsigned char *buf = NULL; char *old_curfile, *ext; size_t buflen = 0; char filebuf[PATH_MAX], realbuf[PATH_MAX]; int prev_state; struct ucl_variable *cur_var, *tmp_var, *old_curdir = NULL, *old_filename = NULL; ucl_object_t *nest_obj = NULL, *old_obj = NULL, *new_obj = NULL; ucl_hash_t *container = NULL; struct ucl_stack *st = NULL; snprintf (filebuf, sizeof (filebuf), "%.*s", (int)len, data); if (ucl_realpath (filebuf, realbuf) == NULL) { if (params->soft_fail) { return false; } if (!params->must_exist) { return true; } ucl_create_err (&parser->err, "cannot open file %s: %s", filebuf, strerror (errno)); return false; } if (parser->cur_file && strcmp (realbuf, parser->cur_file) == 0) { /* We are likely including the file itself */ if (params->soft_fail) { return false; } ucl_create_err (&parser->err, "trying to include the file %s from itself", realbuf); return false; } if (!ucl_fetch_file (realbuf, &buf, &buflen, &parser->err, params->must_exist)) { if (params->soft_fail) { return false; } return (!params->must_exist || false); } if (params->check_signature) { #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L) unsigned char *sigbuf = NULL; size_t siglen = 0; /* We need to check signature first */ snprintf (filebuf, sizeof (filebuf), "%s.sig", realbuf); if (!ucl_fetch_file (filebuf, &sigbuf, &siglen, &parser->err, true)) { return false; } if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) { ucl_create_err (&parser->err, "cannot verify file %s: %s", filebuf, ERR_error_string (ERR_get_error (), NULL)); if (sigbuf) { ucl_munmap (sigbuf, siglen); } return false; } if (sigbuf) { ucl_munmap (sigbuf, siglen); } #endif } old_curfile = parser->cur_file; parser->cur_file = strdup (realbuf); /* Store old file vars */ DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) { if (strcmp (cur_var->var, "CURDIR") == 0) { old_curdir = cur_var; DL_DELETE (parser->variables, cur_var); } else if (strcmp (cur_var->var, "FILENAME") == 0) { old_filename = cur_var; DL_DELETE (parser->variables, cur_var); } } ucl_parser_set_filevars (parser, realbuf, false); prev_state = parser->state; parser->state = UCL_STATE_INIT; if (params->use_prefix && params->prefix == NULL) { /* Auto generate a key name based on the included filename */ params->prefix = basename (realbuf); ext = strrchr (params->prefix, '.'); if (ext != NULL && (strcmp (ext, ".conf") == 0 || strcmp (ext, ".ucl") == 0)) { /* Strip off .conf or .ucl */ *ext = '\0'; } } if (params->prefix != NULL) { /* This is a prefixed include */ container = parser->stack->obj->value.ov; old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container, params->prefix, strlen (params->prefix))); if (strcasecmp (params->target, "array") == 0 && old_obj == NULL) { /* Create an array with key: prefix */ old_obj = ucl_object_new_full (UCL_ARRAY, params->priority); old_obj->key = params->prefix; old_obj->keylen = strlen (params->prefix); ucl_copy_key_trash(old_obj); old_obj->prev = old_obj; old_obj->next = NULL; container = ucl_hash_insert_object (container, old_obj, parser->flags & UCL_PARSER_KEY_LOWERCASE); parser->stack->obj->len ++; nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority); nest_obj->prev = nest_obj; nest_obj->next = NULL; ucl_array_append (old_obj, nest_obj); } else if (old_obj == NULL) { /* Create an object with key: prefix */ nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority); if (nest_obj == NULL) { ucl_create_err (&parser->err, "cannot allocate memory for an object"); if (buf) { ucl_munmap (buf, buflen); } return false; } nest_obj->key = params->prefix; nest_obj->keylen = strlen (params->prefix); ucl_copy_key_trash(nest_obj); nest_obj->prev = nest_obj; nest_obj->next = NULL; container = ucl_hash_insert_object (container, nest_obj, parser->flags & UCL_PARSER_KEY_LOWERCASE); parser->stack->obj->len ++; } else if (strcasecmp (params->target, "array") == 0 || ucl_object_type(old_obj) == UCL_ARRAY) { if (ucl_object_type(old_obj) == UCL_ARRAY) { /* Append to the existing array */ nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority); if (nest_obj == NULL) { ucl_create_err (&parser->err, "cannot allocate memory for an object"); if (buf) { ucl_munmap (buf, buflen); } return false; } nest_obj->prev = nest_obj; nest_obj->next = NULL; ucl_array_append (old_obj, nest_obj); } else { /* Convert the object to an array */ new_obj = ucl_object_typed_new (UCL_ARRAY); if (new_obj == NULL) { ucl_create_err (&parser->err, "cannot allocate memory for an object"); if (buf) { ucl_munmap (buf, buflen); } return false; } new_obj->key = old_obj->key; new_obj->keylen = old_obj->keylen; new_obj->flags |= UCL_OBJECT_MULTIVALUE; new_obj->prev = new_obj; new_obj->next = NULL; nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority); if (nest_obj == NULL) { ucl_create_err (&parser->err, "cannot allocate memory for an object"); if (buf) { ucl_munmap (buf, buflen); } return false; } nest_obj->prev = nest_obj; nest_obj->next = NULL; ucl_array_append (new_obj, old_obj); ucl_array_append (new_obj, nest_obj); ucl_hash_replace (container, old_obj, new_obj); } } else { if (ucl_object_type (old_obj) == UCL_OBJECT) { /* Append to existing Object*/ nest_obj = old_obj; } else { /* The key is not an object */ ucl_create_err (&parser->err, "Conflicting type for key: %s", params->prefix); if (buf) { ucl_munmap (buf, buflen); } return false; } } /* Put all of the content of the include inside that object */ parser->stack->obj->value.ov = container; st = UCL_ALLOC (sizeof (struct ucl_stack)); if (st == NULL) { ucl_create_err (&parser->err, "cannot allocate memory for an object"); ucl_object_unref (nest_obj); if (buf) { ucl_munmap (buf, buflen); } return false; } st->obj = nest_obj; st->level = parser->stack->level; LL_PREPEND (parser->stack, st); parser->cur_obj = nest_obj; } res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority, params->strat, params->parse_type); if (!res) { if (!params->must_exist) { /* Free error */ utstring_free (parser->err); parser->err = NULL; res = true; } } /* Stop nesting the include, take 1 level off the stack */ if (params->prefix != NULL && nest_obj != NULL) { parser->stack = st->next; UCL_FREE (sizeof (struct ucl_stack), st); } /* Remove chunk from the stack */ chunk = parser->chunks; if (chunk != NULL) { parser->chunks = chunk->next; UCL_FREE (sizeof (struct ucl_chunk), chunk); parser->recursion --; } /* Restore old file vars */ if (parser->cur_file) { free (parser->cur_file); } parser->cur_file = old_curfile; DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) { if (strcmp (cur_var->var, "CURDIR") == 0 && old_curdir) { DL_DELETE (parser->variables, cur_var); free (cur_var->var); free (cur_var->value); UCL_FREE (sizeof (struct ucl_variable), cur_var); } else if (strcmp (cur_var->var, "FILENAME") == 0 && old_filename) { DL_DELETE (parser->variables, cur_var); free (cur_var->var); free (cur_var->value); UCL_FREE (sizeof (struct ucl_variable), cur_var); } } if (old_filename) { DL_APPEND (parser->variables, old_filename); } if (old_curdir) { DL_APPEND (parser->variables, old_curdir); } parser->state = prev_state; if (buflen > 0) { ucl_munmap (buf, buflen); } return res; } /** * Include a file to configuration * @param data * @param len * @param parser * @param err * @return */ static bool ucl_include_file (const unsigned char *data, size_t len, struct ucl_parser *parser, struct ucl_include_params *params) { const unsigned char *p = data, *end = data + len; bool need_glob = false; int cnt = 0; char glob_pattern[PATH_MAX]; size_t i; #ifndef _WIN32 if (!params->allow_glob) { return ucl_include_file_single (data, len, parser, params); } else { /* Check for special symbols in a filename */ while (p != end) { if (*p == '*' || *p == '?') { need_glob = true; break; } p ++; } if (need_glob) { glob_t globbuf; memset (&globbuf, 0, sizeof (globbuf)); ucl_strlcpy (glob_pattern, (const char *)data, (len + 1 < sizeof (glob_pattern) ? len + 1 : sizeof (glob_pattern))); if (glob (glob_pattern, 0, NULL, &globbuf) != 0) { return (!params->must_exist || false); } for (i = 0; i < globbuf.gl_pathc; i ++) { if (!ucl_include_file_single ((unsigned char *)globbuf.gl_pathv[i], strlen (globbuf.gl_pathv[i]), parser, params)) { if (params->soft_fail) { continue; } globfree (&globbuf); return false; } cnt ++; } globfree (&globbuf); if (cnt == 0 && params->must_exist) { ucl_create_err (&parser->err, "cannot match any files for pattern %s", glob_pattern); return false; } } else { return ucl_include_file_single (data, len, parser, params); } } #else /* Win32 compilers do not support globbing. Therefore, for Win32, treat allow_glob/need_glob as a NOOP and just return */ return ucl_include_file_single (data, len, parser, params); #endif return true; } /** * Common function to handle .*include* macros * @param data * @param len * @param args * @param parser * @param default_try * @param default_sign * @return */ static bool ucl_include_common (const unsigned char *data, size_t len, const ucl_object_t *args, struct ucl_parser *parser, bool default_try, bool default_sign) { bool allow_url = false, search = false; const char *duplicate; const ucl_object_t *param; ucl_object_iter_t it = NULL, ip = NULL; char ipath[PATH_MAX]; struct ucl_include_params params; /* Default values */ params.soft_fail = default_try; params.allow_glob = false; params.check_signature = default_sign; params.use_prefix = false; params.target = "object"; params.prefix = NULL; params.priority = 0; params.parse_type = UCL_PARSE_UCL; params.strat = UCL_DUPLICATE_APPEND; params.must_exist = !default_try; /* Process arguments */ if (args != NULL && args->type == UCL_OBJECT) { while ((param = ucl_object_iterate (args, &it, true)) != NULL) { if (param->type == UCL_BOOLEAN) { if (strncmp (param->key, "try", param->keylen) == 0) { params.must_exist = !ucl_object_toboolean (param); } else if (strncmp (param->key, "sign", param->keylen) == 0) { params.check_signature = ucl_object_toboolean (param); } else if (strncmp (param->key, "glob", param->keylen) == 0) { params.allow_glob = ucl_object_toboolean (param); } else if (strncmp (param->key, "url", param->keylen) == 0) { allow_url = ucl_object_toboolean (param); } else if (strncmp (param->key, "prefix", param->keylen) == 0) { params.use_prefix = ucl_object_toboolean (param); } } else if (param->type == UCL_STRING) { if (strncmp (param->key, "key", param->keylen) == 0) { params.prefix = ucl_object_tostring (param); } else if (strncmp (param->key, "target", param->keylen) == 0) { params.target = ucl_object_tostring (param); } else if (strncmp (param->key, "duplicate", param->keylen) == 0) { duplicate = ucl_object_tostring (param); if (strcmp (duplicate, "append") == 0) { params.strat = UCL_DUPLICATE_APPEND; } else if (strcmp (duplicate, "merge") == 0) { params.strat = UCL_DUPLICATE_MERGE; } else if (strcmp (duplicate, "rewrite") == 0) { params.strat = UCL_DUPLICATE_REWRITE; } else if (strcmp (duplicate, "error") == 0) { params.strat = UCL_DUPLICATE_ERROR; } } } else if (param->type == UCL_ARRAY) { if (strncmp (param->key, "path", param->keylen) == 0) { ucl_set_include_path (parser, __DECONST(ucl_object_t *, param)); } } else if (param->type == UCL_INT) { if (strncmp (param->key, "priority", param->keylen) == 0) { params.priority = ucl_object_toint (param); } } } } if (parser->includepaths == NULL) { if (allow_url && ucl_strnstr (data, "://", len) != NULL) { /* Globbing is not used for URL's */ return ucl_include_url (data, len, parser, ¶ms); } else if (data != NULL) { /* Try to load a file */ return ucl_include_file (data, len, parser, ¶ms); } } else { if (allow_url && ucl_strnstr (data, "://", len) != NULL) { /* Globbing is not used for URL's */ return ucl_include_url (data, len, parser, ¶ms); } ip = ucl_object_iterate_new (parser->includepaths); while ((param = ucl_object_iterate_safe (ip, true)) != NULL) { if (ucl_object_type(param) == UCL_STRING) { snprintf (ipath, sizeof (ipath), "%s/%.*s", ucl_object_tostring(param), (int)len, data); if ((search = ucl_include_file (ipath, strlen (ipath), parser, ¶ms))) { if (!params.allow_glob) { break; } } } } ucl_object_iterate_free (ip); if (search == true) { return true; } else { ucl_create_err (&parser->err, "cannot find file: %.*s in search path", (int)len, data); return false; } } return false; } /** * Handle include macro * @param data include data * @param len length of data * @param args UCL object representing arguments to the macro * @param ud user data * @return */ bool ucl_include_handler (const unsigned char *data, size_t len, const ucl_object_t *args, void* ud) { struct ucl_parser *parser = ud; return ucl_include_common (data, len, args, parser, false, false); } /** * Handle includes macro * @param data include data * @param len length of data * @param args UCL object representing arguments to the macro * @param ud user data * @return */ bool ucl_includes_handler (const unsigned char *data, size_t len, const ucl_object_t *args, void* ud) { struct ucl_parser *parser = ud; return ucl_include_common (data, len, args, parser, false, true); } /** * Handle tryinclude macro * @param data include data * @param len length of data * @param args UCL object representing arguments to the macro * @param ud user data * @return */ bool ucl_try_include_handler (const unsigned char *data, size_t len, const ucl_object_t *args, void* ud) { struct ucl_parser *parser = ud; return ucl_include_common (data, len, args, parser, true, false); } /** * Handle priority macro * @param data include data * @param len length of data * @param args UCL object representing arguments to the macro * @param ud user data * @return */ bool ucl_priority_handler (const unsigned char *data, size_t len, const ucl_object_t *args, void* ud) { struct ucl_parser *parser = ud; unsigned priority = 255; const ucl_object_t *param; bool found = false; char *value = NULL, *leftover = NULL; ucl_object_iter_t it = NULL; if (parser == NULL) { return false; } /* Process arguments */ if (args != NULL && args->type == UCL_OBJECT) { while ((param = ucl_object_iterate (args, &it, true)) != NULL) { if (param->type == UCL_INT) { if (strncmp (param->key, "priority", param->keylen) == 0) { priority = ucl_object_toint (param); found = true; } } } } if (len > 0) { value = malloc(len + 1); ucl_strlcpy(value, (const char *)data, len + 1); priority = strtol(value, &leftover, 10); if (*leftover != '\0') { ucl_create_err (&parser->err, "Invalid priority value in macro: %s", value); free(value); return false; } free(value); found = true; } if (found == true) { parser->chunks->priority = priority; return true; } ucl_create_err (&parser->err, "Unable to parse priority macro"); return false; } /** * Handle load macro * @param data include data * @param len length of data * @param args UCL object representing arguments to the macro * @param ud user data * @return */ bool ucl_load_handler (const unsigned char *data, size_t len, const ucl_object_t *args, void* ud) { struct ucl_parser *parser = ud; const ucl_object_t *param; ucl_object_t *obj, *old_obj; ucl_object_iter_t it = NULL; bool try_load, multiline, test; const char *target, *prefix; char *load_file, *tmp; unsigned char *buf; size_t buflen; unsigned priority; int64_t iv; ucl_object_t *container = NULL; enum ucl_string_flags flags; /* Default values */ try_load = false; multiline = false; test = false; target = "string"; prefix = NULL; load_file = NULL; buf = NULL; buflen = 0; priority = 0; obj = NULL; old_obj = NULL; flags = 0; if (parser == NULL) { return false; } /* Process arguments */ if (args != NULL && args->type == UCL_OBJECT) { while ((param = ucl_object_iterate (args, &it, true)) != NULL) { if (param->type == UCL_BOOLEAN) { if (strncmp (param->key, "try", param->keylen) == 0) { try_load = ucl_object_toboolean (param); } else if (strncmp (param->key, "multiline", param->keylen) == 0) { multiline = ucl_object_toboolean (param); } else if (strncmp (param->key, "escape", param->keylen) == 0) { test = ucl_object_toboolean (param); if (test) { flags |= UCL_STRING_ESCAPE; } } else if (strncmp (param->key, "trim", param->keylen) == 0) { test = ucl_object_toboolean (param); if (test) { flags |= UCL_STRING_TRIM; } } } else if (param->type == UCL_STRING) { if (strncmp (param->key, "key", param->keylen) == 0) { prefix = ucl_object_tostring (param); } else if (strncmp (param->key, "target", param->keylen) == 0) { target = ucl_object_tostring (param); } } else if (param->type == UCL_INT) { if (strncmp (param->key, "priority", param->keylen) == 0) { priority = ucl_object_toint (param); } } } } if (prefix == NULL || strlen (prefix) == 0) { ucl_create_err (&parser->err, "No Key specified in load macro"); return false; } if (len > 0) { load_file = malloc (len + 1); if (!load_file) { ucl_create_err (&parser->err, "cannot allocate memory for suffix"); return false; } snprintf (load_file, len + 1, "%.*s", (int)len, data); if (!ucl_fetch_file (load_file, &buf, &buflen, &parser->err, !try_load)) { free (load_file); return (try_load || false); } free (load_file); container = parser->stack->obj; old_obj = __DECONST (ucl_object_t *, ucl_object_lookup (container, prefix)); if (old_obj != NULL) { ucl_create_err (&parser->err, "Key %s already exists", prefix); if (buf) { ucl_munmap (buf, buflen); } return false; } if (strcasecmp (target, "string") == 0) { obj = ucl_object_fromstring_common (buf, buflen, flags); ucl_copy_value_trash (obj); if (multiline) { obj->flags |= UCL_OBJECT_MULTILINE; } } else if (strcasecmp (target, "int") == 0) { tmp = malloc (buflen + 1); if (tmp == NULL) { ucl_create_err (&parser->err, "Memory allocation failed"); if (buf) { ucl_munmap (buf, buflen); } return false; } snprintf (tmp, buflen + 1, "%.*s", (int)buflen, buf); iv = strtoll (tmp, NULL, 10); obj = ucl_object_fromint (iv); free (tmp); } if (buf) { ucl_munmap (buf, buflen); } if (obj != NULL) { obj->key = prefix; obj->keylen = strlen (prefix); ucl_copy_key_trash (obj); obj->prev = obj; obj->next = NULL; ucl_object_set_priority (obj, priority); ucl_object_insert_key (container, obj, obj->key, obj->keylen, false); } return true; } ucl_create_err (&parser->err, "Unable to parse load macro"); return false; } bool ucl_inherit_handler (const unsigned char *data, size_t len, const ucl_object_t *args, const ucl_object_t *ctx, void* ud) { const ucl_object_t *parent, *cur; ucl_object_t *target, *copy; ucl_object_iter_t it = NULL; bool replace = false; struct ucl_parser *parser = ud; parent = ucl_object_lookup_len (ctx, data, len); /* Some sanity checks */ if (parent == NULL || ucl_object_type (parent) != UCL_OBJECT) { ucl_create_err (&parser->err, "Unable to find inherited object %*.s", (int)len, data); return false; } if (parser->stack == NULL || parser->stack->obj == NULL || ucl_object_type (parser->stack->obj) != UCL_OBJECT) { ucl_create_err (&parser->err, "Invalid inherit context"); return false; } target = parser->stack->obj; if (args && (cur = ucl_object_lookup (args, "replace")) != NULL) { replace = ucl_object_toboolean (cur); } while ((cur = ucl_object_iterate (parent, &it, true))) { /* We do not replace existing keys */ if (!replace && ucl_object_lookup_len (target, cur->key, cur->keylen)) { continue; } copy = ucl_object_copy (cur); if (!replace) { copy->flags |= UCL_OBJECT_INHERITED; } ucl_object_insert_key (target, copy, copy->key, copy->keylen, false); } return true; } bool ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool need_expand) { char realbuf[PATH_MAX], *curdir; if (filename != NULL) { if (need_expand) { if (ucl_realpath (filename, realbuf) == NULL) { return false; } } else { ucl_strlcpy (realbuf, filename, sizeof (realbuf)); } /* Define variables */ ucl_parser_register_variable (parser, "FILENAME", realbuf); curdir = dirname (realbuf); ucl_parser_register_variable (parser, "CURDIR", curdir); } else { /* Set everything from the current dir */ curdir = getcwd (realbuf, sizeof (realbuf)); ucl_parser_register_variable (parser, "FILENAME", "undef"); ucl_parser_register_variable (parser, "CURDIR", curdir); } return true; } bool -ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename, - unsigned priority) +ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename, + unsigned priority, enum ucl_duplicate_strategy strat, + enum ucl_parse_type parse_type) { unsigned char *buf; size_t len; bool ret; char realbuf[PATH_MAX]; if (ucl_realpath (filename, realbuf) == NULL) { ucl_create_err (&parser->err, "cannot open file %s: %s", filename, strerror (errno)); return false; } if (!ucl_fetch_file (realbuf, &buf, &len, &parser->err, true)) { return false; } if (parser->cur_file) { free (parser->cur_file); } parser->cur_file = strdup (realbuf); ucl_parser_set_filevars (parser, realbuf, false); - ret = ucl_parser_add_chunk_priority (parser, buf, len, priority); + ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat, + parse_type); if (len > 0) { ucl_munmap (buf, len); } return ret; } bool +ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename, + unsigned priority) +{ + if (parser == NULL) { + return false; + } + + return ucl_parser_add_file_full(parser, filename, priority, + UCL_DUPLICATE_APPEND, UCL_PARSE_UCL); +} + +bool ucl_parser_add_file (struct ucl_parser *parser, const char *filename) { if (parser == NULL) { return false; } - return ucl_parser_add_file_priority(parser, filename, - parser->default_priority); + return ucl_parser_add_file_full(parser, filename, + parser->default_priority, UCL_DUPLICATE_APPEND, + UCL_PARSE_UCL); } bool ucl_parser_add_fd_priority (struct ucl_parser *parser, int fd, unsigned priority) { unsigned char *buf; size_t len; bool ret; struct stat st; if (fstat (fd, &st) == -1) { ucl_create_err (&parser->err, "cannot stat fd %d: %s", fd, strerror (errno)); return false; } if (st.st_size == 0) { return true; } if ((buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { ucl_create_err (&parser->err, "cannot mmap fd %d: %s", fd, strerror (errno)); return false; } if (parser->cur_file) { free (parser->cur_file); } parser->cur_file = NULL; len = st.st_size; ret = ucl_parser_add_chunk_priority (parser, buf, len, priority); if (len > 0) { ucl_munmap (buf, len); } return ret; } bool ucl_parser_add_fd (struct ucl_parser *parser, int fd) { if (parser == NULL) { return false; } return ucl_parser_add_fd_priority(parser, fd, parser->default_priority); } size_t ucl_strlcpy (char *dst, const char *src, size_t siz) { char *d = dst; const char *s = src; size_t n = siz; /* Copy as many bytes as will fit */ if (n != 0) { while (--n != 0) { if ((*d++ = *s++) == '\0') { break; } } } if (n == 0 && siz != 0) { *d = '\0'; } return (s - src - 1); /* count does not include NUL */ } size_t ucl_strlcpy_unsafe (char *dst, const char *src, size_t siz) { memcpy (dst, src, siz - 1); dst[siz - 1] = '\0'; return siz - 1; } size_t ucl_strlcpy_tolower (char *dst, const char *src, size_t siz) { char *d = dst; const char *s = src; size_t n = siz; /* Copy as many bytes as will fit */ if (n != 0) { while (--n != 0) { if ((*d++ = tolower (*s++)) == '\0') { break; } } } if (n == 0 && siz != 0) { *d = '\0'; } return (s - src); /* count does not include NUL */ } /* * Find the first occurrence of find in s */ char * ucl_strnstr (const char *s, const char *find, int len) { char c, sc; int mlen; if ((c = *find++) != 0) { mlen = strlen (find); do { do { if ((sc = *s++) == 0 || len-- == 0) return (NULL); } while (sc != c); } while (strncmp (s, find, mlen) != 0); s--; } return ((char *)s); } /* * Find the first occurrence of find in s, ignore case. */ char * ucl_strncasestr (const char *s, const char *find, int len) { char c, sc; int mlen; if ((c = *find++) != 0) { c = tolower (c); mlen = strlen (find); do { do { if ((sc = *s++) == 0 || len-- == 0) return (NULL); } while (tolower (sc) != c); } while (strncasecmp (s, find, mlen) != 0); s--; } return ((char *)s); } ucl_object_t * ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags flags) { ucl_object_t *obj; const char *start, *end, *p, *pos; char *dst, *d; size_t escaped_len; if (str == NULL) { return NULL; } obj = ucl_object_new (); if (obj) { if (len == 0) { len = strlen (str); } if (flags & UCL_STRING_TRIM) { /* Skip leading spaces */ for (start = str; (size_t)(start - str) < len; start ++) { if (!ucl_test_character (*start, UCL_CHARACTER_WHITESPACE_UNSAFE)) { break; } } /* Skip trailing spaces */ for (end = str + len - 1; end > start; end --) { if (!ucl_test_character (*end, UCL_CHARACTER_WHITESPACE_UNSAFE)) { break; } } end ++; } else { start = str; end = str + len; } obj->type = UCL_STRING; if (flags & UCL_STRING_ESCAPE) { for (p = start, escaped_len = 0; p < end; p ++, escaped_len ++) { if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE)) { escaped_len ++; } } dst = malloc (escaped_len + 1); if (dst != NULL) { for (p = start, d = dst; p < end; p ++, d ++) { if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE)) { switch (*p) { case '\n': *d++ = '\\'; *d = 'n'; break; case '\r': *d++ = '\\'; *d = 'r'; break; case '\b': *d++ = '\\'; *d = 'b'; break; case '\t': *d++ = '\\'; *d = 't'; break; case '\f': *d++ = '\\'; *d = 'f'; break; case '\\': *d++ = '\\'; *d = '\\'; break; case '"': *d++ = '\\'; *d = '"'; break; } } else { *d = *p; } } *d = '\0'; obj->value.sv = dst; obj->trash_stack[UCL_TRASH_VALUE] = dst; obj->len = escaped_len; } } else { dst = malloc (end - start + 1); if (dst != NULL) { ucl_strlcpy_unsafe (dst, start, end - start + 1); obj->value.sv = dst; obj->trash_stack[UCL_TRASH_VALUE] = dst; obj->len = end - start; } } if ((flags & UCL_STRING_PARSE) && dst != NULL) { /* Parse what we have */ if (flags & UCL_STRING_PARSE_BOOLEAN) { if (!ucl_maybe_parse_boolean (obj, dst, obj->len) && (flags & UCL_STRING_PARSE_NUMBER)) { ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos, flags & UCL_STRING_PARSE_DOUBLE, flags & UCL_STRING_PARSE_BYTES, flags & UCL_STRING_PARSE_TIME); } } else { ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos, flags & UCL_STRING_PARSE_DOUBLE, flags & UCL_STRING_PARSE_BYTES, flags & UCL_STRING_PARSE_TIME); } } } return obj; } static bool ucl_object_insert_key_common (ucl_object_t *top, ucl_object_t *elt, const char *key, size_t keylen, bool copy_key, bool merge, bool replace) { ucl_object_t *found, *tmp; const ucl_object_t *cur; ucl_object_iter_t it = NULL; const char *p; int ret = true; if (elt == NULL || key == NULL) { return false; } if (top == NULL) { return false; } if (top->type != UCL_OBJECT) { /* It is possible to convert NULL type to an object */ if (top->type == UCL_NULL) { top->type = UCL_OBJECT; } else { /* Refuse converting of other object types */ return false; } } if (top->value.ov == NULL) { top->value.ov = ucl_hash_create (false); } if (keylen == 0) { keylen = strlen (key); } for (p = key; p < key + keylen; p ++) { if (ucl_test_character (*p, UCL_CHARACTER_UCL_UNSAFE)) { elt->flags |= UCL_OBJECT_NEED_KEY_ESCAPE; break; } } /* workaround for some use cases */ if (elt->trash_stack[UCL_TRASH_KEY] != NULL && key != (const char *)elt->trash_stack[UCL_TRASH_KEY]) { /* Remove copied key */ free (elt->trash_stack[UCL_TRASH_KEY]); elt->trash_stack[UCL_TRASH_KEY] = NULL; elt->flags &= ~UCL_OBJECT_ALLOCATED_KEY; } elt->key = key; elt->keylen = keylen; if (copy_key) { ucl_copy_key_trash (elt); } found = __DECONST (ucl_object_t *, ucl_hash_search_obj (top->value.ov, elt)); if (found == NULL) { top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false); top->len ++; if (replace) { ret = false; } } else { if (replace) { ucl_hash_replace (top->value.ov, found, elt); ucl_object_unref (found); } else if (merge) { if (found->type != UCL_OBJECT && elt->type == UCL_OBJECT) { /* Insert old elt to new one */ ucl_object_insert_key_common (elt, found, found->key, found->keylen, copy_key, false, false); ucl_hash_delete (top->value.ov, found); top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false); } else if (found->type == UCL_OBJECT && elt->type != UCL_OBJECT) { /* Insert new to old */ ucl_object_insert_key_common (found, elt, elt->key, elt->keylen, copy_key, false, false); } else if (found->type == UCL_OBJECT && elt->type == UCL_OBJECT) { /* Mix two hashes */ while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) { tmp = ucl_object_ref (cur); ucl_object_insert_key_common (found, tmp, cur->key, cur->keylen, copy_key, false, false); } ucl_object_unref (elt); } else { /* Just make a list of scalars */ DL_APPEND (found, elt); } } else { DL_APPEND (found, elt); } } return ret; } bool ucl_object_delete_keyl (ucl_object_t *top, const char *key, size_t keylen) { ucl_object_t *found; if (top == NULL || key == NULL) { return false; } found = __DECONST (ucl_object_t *, ucl_object_lookup_len (top, key, keylen)); if (found == NULL) { return false; } ucl_hash_delete (top->value.ov, found); ucl_object_unref (found); top->len --; return true; } bool ucl_object_delete_key (ucl_object_t *top, const char *key) { return ucl_object_delete_keyl (top, key, strlen (key)); } ucl_object_t* ucl_object_pop_keyl (ucl_object_t *top, const char *key, size_t keylen) { const ucl_object_t *found; if (top == NULL || key == NULL) { return false; } found = ucl_object_lookup_len (top, key, keylen); if (found == NULL) { return NULL; } ucl_hash_delete (top->value.ov, found); top->len --; return __DECONST (ucl_object_t *, found); } ucl_object_t* ucl_object_pop_key (ucl_object_t *top, const char *key) { return ucl_object_pop_keyl (top, key, strlen (key)); } bool ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt, const char *key, size_t keylen, bool copy_key) { return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, false); } bool ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt, const char *key, size_t keylen, bool copy_key) { return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, true, false); } bool ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt, const char *key, size_t keylen, bool copy_key) { return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, true); } bool ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy) { ucl_object_t *cur = NULL, *cp = NULL, *found = NULL; ucl_object_iter_t iter = NULL; if (top == NULL || top->type != UCL_OBJECT || elt == NULL || elt->type != UCL_OBJECT) { return false; } /* Mix two hashes */ while ((cur = (ucl_object_t*)ucl_hash_iterate (elt->value.ov, &iter))) { if (copy) { cp = ucl_object_copy (cur); } else { cp = ucl_object_ref (cur); } found = __DECONST(ucl_object_t *, ucl_hash_search (top->value.ov, cp->key, cp->keylen)); if (found == NULL) { /* The key does not exist */ top->value.ov = ucl_hash_insert_object (top->value.ov, cp, false); top->len ++; } else { /* The key already exists, replace it */ ucl_hash_replace (top->value.ov, found, cp); ucl_object_unref (found); } } return true; } const ucl_object_t * ucl_object_lookup_len (const ucl_object_t *obj, const char *key, size_t klen) { const ucl_object_t *ret; ucl_object_t srch; if (obj == NULL || obj->type != UCL_OBJECT || key == NULL) { return NULL; } srch.key = key; srch.keylen = klen; ret = ucl_hash_search_obj (obj->value.ov, &srch); return ret; } const ucl_object_t * ucl_object_lookup (const ucl_object_t *obj, const char *key) { if (key == NULL) { return NULL; } return ucl_object_lookup_len (obj, key, strlen (key)); } const ucl_object_t* ucl_object_lookup_any (const ucl_object_t *obj, const char *key, ...) { va_list ap; const ucl_object_t *ret = NULL; const char *nk = NULL; if (obj == NULL || key == NULL) { return NULL; } ret = ucl_object_lookup_len (obj, key, strlen (key)); if (ret == NULL) { va_start (ap, key); while (ret == NULL) { nk = va_arg (ap, const char *); if (nk == NULL) { break; } else { ret = ucl_object_lookup_len (obj, nk, strlen (nk)); } } va_end (ap); } return ret; } const ucl_object_t* ucl_object_iterate (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values) { const ucl_object_t *elt = NULL; if (obj == NULL || iter == NULL) { return NULL; } if (expand_values) { switch (obj->type) { case UCL_OBJECT: return (const ucl_object_t*)ucl_hash_iterate (obj->value.ov, iter); break; case UCL_ARRAY: { unsigned int idx; UCL_ARRAY_GET (vec, obj); idx = (unsigned int)(uintptr_t)(*iter); if (vec != NULL) { while (idx < kv_size (*vec)) { if ((elt = kv_A (*vec, idx)) != NULL) { idx ++; break; } idx ++; } *iter = (void *)(uintptr_t)idx; } return elt; break; } default: /* Go to linear iteration */ break; } } /* Treat everything as a linear list */ elt = *iter; if (elt == NULL) { elt = obj; } else if (elt == obj) { return NULL; } *iter = __DECONST (void *, elt->next ? elt->next : obj); return elt; /* Not reached */ return NULL; } const char safe_iter_magic[4] = {'u', 'i', 't', 'e'}; struct ucl_object_safe_iter { char magic[4]; /* safety check */ const ucl_object_t *impl_it; /* implicit object iteration */ ucl_object_iter_t expl_it; /* explicit iteration */ }; #define UCL_SAFE_ITER(ptr) (struct ucl_object_safe_iter *)(ptr) #define UCL_SAFE_ITER_CHECK(it) do { \ assert (it != NULL); \ assert (memcmp (it->magic, safe_iter_magic, sizeof (it->magic)) == 0); \ } while (0) ucl_object_iter_t ucl_object_iterate_new (const ucl_object_t *obj) { struct ucl_object_safe_iter *it; it = UCL_ALLOC (sizeof (*it)); if (it != NULL) { memcpy (it->magic, safe_iter_magic, sizeof (it->magic)); it->expl_it = NULL; it->impl_it = obj; } return (ucl_object_iter_t)it; } ucl_object_iter_t ucl_object_iterate_reset (ucl_object_iter_t it, const ucl_object_t *obj) { struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it); UCL_SAFE_ITER_CHECK (rit); rit->impl_it = obj; rit->expl_it = NULL; return it; } const ucl_object_t* ucl_object_iterate_safe (ucl_object_iter_t it, bool expand_values) { struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it); const ucl_object_t *ret = NULL; UCL_SAFE_ITER_CHECK (rit); if (rit->impl_it == NULL) { return NULL; } if (rit->impl_it->type == UCL_OBJECT || rit->impl_it->type == UCL_ARRAY) { ret = ucl_object_iterate (rit->impl_it, &rit->expl_it, true); if (ret == NULL) { /* Need to switch to another implicit object in chain */ rit->impl_it = rit->impl_it->next; rit->expl_it = NULL; return ucl_object_iterate_safe (it, expand_values); } } else { /* Just iterate over the implicit array */ ret = rit->impl_it; rit->impl_it = rit->impl_it->next; if (expand_values) { /* We flatten objects if need to expand values */ if (ret->type == UCL_OBJECT || ret->type == UCL_ARRAY) { return ucl_object_iterate_safe (it, expand_values); } } } return ret; } void ucl_object_iterate_free (ucl_object_iter_t it) { struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it); UCL_SAFE_ITER_CHECK (rit); UCL_FREE (sizeof (*rit), it); } const ucl_object_t * ucl_object_lookup_path (const ucl_object_t *top, const char *path_in) { return ucl_object_lookup_path_char (top, path_in, '.'); } const ucl_object_t * ucl_object_lookup_path_char (const ucl_object_t *top, const char *path_in, const char sep) { const ucl_object_t *o = NULL, *found; const char *p, *c; char *err_str; unsigned index; if (path_in == NULL || top == NULL) { return NULL; } found = NULL; p = path_in; /* Skip leading dots */ while (*p == sep) { p ++; } c = p; while (*p != '\0') { p ++; if (*p == sep || *p == '\0') { if (p > c) { switch (top->type) { case UCL_ARRAY: /* Key should be an int */ index = strtoul (c, &err_str, 10); if (err_str != NULL && (*err_str != sep && *err_str != '\0')) { return NULL; } o = ucl_array_find_index (top, index); break; default: o = ucl_object_lookup_len (top, c, p - c); break; } if (o == NULL) { return NULL; } top = o; } if (*p != '\0') { c = p + 1; } } } found = o; return found; } ucl_object_t * ucl_object_new (void) { return ucl_object_typed_new (UCL_NULL); } ucl_object_t * ucl_object_typed_new (ucl_type_t type) { return ucl_object_new_full (type, 0); } ucl_object_t * ucl_object_new_full (ucl_type_t type, unsigned priority) { ucl_object_t *new; if (type != UCL_USERDATA) { new = UCL_ALLOC (sizeof (ucl_object_t)); if (new != NULL) { memset (new, 0, sizeof (ucl_object_t)); new->ref = 1; new->type = (type <= UCL_NULL ? type : UCL_NULL); new->next = NULL; new->prev = new; ucl_object_set_priority (new, priority); if (type == UCL_ARRAY) { new->value.av = UCL_ALLOC (sizeof (ucl_array_t)); if (new->value.av) { memset (new->value.av, 0, sizeof (ucl_array_t)); UCL_ARRAY_GET (vec, new); /* Preallocate some space for arrays */ kv_resize (ucl_object_t *, *vec, 8); } } } } else { new = ucl_object_new_userdata (NULL, NULL, NULL); ucl_object_set_priority (new, priority); } return new; } ucl_object_t* ucl_object_new_userdata (ucl_userdata_dtor dtor, ucl_userdata_emitter emitter, void *ptr) { struct ucl_object_userdata *new; size_t nsize = sizeof (*new); new = UCL_ALLOC (nsize); if (new != NULL) { memset (new, 0, nsize); new->obj.ref = 1; new->obj.type = UCL_USERDATA; new->obj.next = NULL; new->obj.prev = (ucl_object_t *)new; new->dtor = dtor; new->emitter = emitter; new->obj.value.ud = ptr; } return (ucl_object_t *)new; } ucl_type_t ucl_object_type (const ucl_object_t *obj) { if (obj == NULL) { return UCL_NULL; } return obj->type; } ucl_object_t* ucl_object_fromstring (const char *str) { return ucl_object_fromstring_common (str, 0, UCL_STRING_ESCAPE); } ucl_object_t * ucl_object_fromlstring (const char *str, size_t len) { return ucl_object_fromstring_common (str, len, UCL_STRING_ESCAPE); } ucl_object_t * ucl_object_fromint (int64_t iv) { ucl_object_t *obj; obj = ucl_object_new (); if (obj != NULL) { obj->type = UCL_INT; obj->value.iv = iv; } return obj; } ucl_object_t * ucl_object_fromdouble (double dv) { ucl_object_t *obj; obj = ucl_object_new (); if (obj != NULL) { obj->type = UCL_FLOAT; obj->value.dv = dv; } return obj; } ucl_object_t* ucl_object_frombool (bool bv) { ucl_object_t *obj; obj = ucl_object_new (); if (obj != NULL) { obj->type = UCL_BOOLEAN; obj->value.iv = bv; } return obj; } bool ucl_array_append (ucl_object_t *top, ucl_object_t *elt) { UCL_ARRAY_GET (vec, top); if (elt == NULL || top == NULL) { return false; } if (vec == NULL) { vec = UCL_ALLOC (sizeof (*vec)); if (vec == NULL) { return false; } kv_init (*vec); top->value.av = (void *)vec; } kv_push (ucl_object_t *, *vec, elt); top->len ++; return true; } bool ucl_array_prepend (ucl_object_t *top, ucl_object_t *elt) { UCL_ARRAY_GET (vec, top); if (elt == NULL || top == NULL) { return false; } if (vec == NULL) { vec = UCL_ALLOC (sizeof (*vec)); kv_init (*vec); top->value.av = (void *)vec; kv_push (ucl_object_t *, *vec, elt); } else { /* Slow O(n) algorithm */ kv_prepend (ucl_object_t *, *vec, elt); } top->len ++; return true; } bool ucl_array_merge (ucl_object_t *top, ucl_object_t *elt, bool copy) { unsigned i; ucl_object_t *cp = NULL; ucl_object_t **obj; if (elt == NULL || top == NULL || top->type != UCL_ARRAY || elt->type != UCL_ARRAY) { return false; } if (copy) { cp = ucl_object_copy (elt); } else { cp = ucl_object_ref (elt); } UCL_ARRAY_GET (v1, top); UCL_ARRAY_GET (v2, cp); if (v1 && v2) { kv_concat (ucl_object_t *, *v1, *v2); for (i = v2->n; i < v1->n; i ++) { obj = &kv_A (*v1, i); if (*obj == NULL) { continue; } top->len ++; } } return true; } ucl_object_t * ucl_array_delete (ucl_object_t *top, ucl_object_t *elt) { UCL_ARRAY_GET (vec, top); ucl_object_t *ret = NULL; unsigned i; if (vec == NULL) { return NULL; } for (i = 0; i < vec->n; i ++) { if (kv_A (*vec, i) == elt) { kv_del (ucl_object_t *, *vec, i); ret = elt; top->len --; break; } } return ret; } const ucl_object_t * ucl_array_head (const ucl_object_t *top) { UCL_ARRAY_GET (vec, top); if (vec == NULL || top == NULL || top->type != UCL_ARRAY || top->value.av == NULL) { return NULL; } return (vec->n > 0 ? vec->a[0] : NULL); } const ucl_object_t * ucl_array_tail (const ucl_object_t *top) { UCL_ARRAY_GET (vec, top); if (top == NULL || top->type != UCL_ARRAY || top->value.av == NULL) { return NULL; } return (vec->n > 0 ? vec->a[vec->n - 1] : NULL); } ucl_object_t * ucl_array_pop_last (ucl_object_t *top) { UCL_ARRAY_GET (vec, top); ucl_object_t **obj, *ret = NULL; if (vec != NULL && vec->n > 0) { obj = &kv_A (*vec, vec->n - 1); ret = *obj; kv_del (ucl_object_t *, *vec, vec->n - 1); top->len --; } return ret; } ucl_object_t * ucl_array_pop_first (ucl_object_t *top) { UCL_ARRAY_GET (vec, top); ucl_object_t **obj, *ret = NULL; if (vec != NULL && vec->n > 0) { obj = &kv_A (*vec, 0); ret = *obj; kv_del (ucl_object_t *, *vec, 0); top->len --; } return ret; } const ucl_object_t * ucl_array_find_index (const ucl_object_t *top, unsigned int index) { UCL_ARRAY_GET (vec, top); if (vec != NULL && vec->n > 0 && index < vec->n) { return kv_A (*vec, index); } return NULL; } unsigned int ucl_array_index_of (ucl_object_t *top, ucl_object_t *elt) { UCL_ARRAY_GET (vec, top); unsigned i; if (vec == NULL) { return (unsigned int)(-1); } for (i = 0; i < vec->n; i ++) { if (kv_A (*vec, i) == elt) { return i; } } return (unsigned int)(-1); } ucl_object_t * ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt, unsigned int index) { UCL_ARRAY_GET (vec, top); ucl_object_t *ret = NULL; if (vec != NULL && vec->n > 0 && index < vec->n) { ret = kv_A (*vec, index); kv_A (*vec, index) = elt; } return ret; } ucl_object_t * ucl_elt_append (ucl_object_t *head, ucl_object_t *elt) { if (head == NULL) { elt->next = NULL; elt->prev = elt; head = elt; } else { elt->prev = head->prev; head->prev->next = elt; head->prev = elt; elt->next = NULL; } return head; } bool ucl_object_todouble_safe (const ucl_object_t *obj, double *target) { if (obj == NULL || target == NULL) { return false; } switch (obj->type) { case UCL_INT: *target = obj->value.iv; /* Probaly could cause overflow */ break; case UCL_FLOAT: case UCL_TIME: *target = obj->value.dv; break; default: return false; } return true; } double ucl_object_todouble (const ucl_object_t *obj) { double result = 0.; ucl_object_todouble_safe (obj, &result); return result; } bool ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target) { if (obj == NULL || target == NULL) { return false; } switch (obj->type) { case UCL_INT: *target = obj->value.iv; break; case UCL_FLOAT: case UCL_TIME: *target = obj->value.dv; /* Loosing of decimal points */ break; default: return false; } return true; } int64_t ucl_object_toint (const ucl_object_t *obj) { int64_t result = 0; ucl_object_toint_safe (obj, &result); return result; } bool ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target) { if (obj == NULL || target == NULL) { return false; } switch (obj->type) { case UCL_BOOLEAN: *target = (obj->value.iv == true); break; default: return false; } return true; } bool ucl_object_toboolean (const ucl_object_t *obj) { bool result = false; ucl_object_toboolean_safe (obj, &result); return result; } bool ucl_object_tostring_safe (const ucl_object_t *obj, const char **target) { if (obj == NULL || target == NULL) { return false; } switch (obj->type) { case UCL_STRING: if (!(obj->flags & UCL_OBJECT_BINARY)) { *target = ucl_copy_value_trash (obj); } break; default: return false; } return true; } const char * ucl_object_tostring (const ucl_object_t *obj) { const char *result = NULL; ucl_object_tostring_safe (obj, &result); return result; } const char * ucl_object_tostring_forced (const ucl_object_t *obj) { /* TODO: For binary strings we might encode string here */ if (!(obj->flags & UCL_OBJECT_BINARY)) { return ucl_copy_value_trash (obj); } return NULL; } bool ucl_object_tolstring_safe (const ucl_object_t *obj, const char **target, size_t *tlen) { if (obj == NULL || target == NULL) { return false; } switch (obj->type) { case UCL_STRING: *target = obj->value.sv; if (tlen != NULL) { *tlen = obj->len; } break; default: return false; } return true; } const char * ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen) { const char *result = NULL; ucl_object_tolstring_safe (obj, &result, tlen); return result; } const char * ucl_object_key (const ucl_object_t *obj) { return ucl_copy_key_trash (obj); } const char * ucl_object_keyl (const ucl_object_t *obj, size_t *len) { if (len == NULL || obj == NULL) { return NULL; } *len = obj->keylen; return obj->key; } ucl_object_t * ucl_object_ref (const ucl_object_t *obj) { ucl_object_t *res = NULL; if (obj != NULL) { if (obj->flags & UCL_OBJECT_EPHEMERAL) { /* * Use deep copy for ephemeral objects, note that its refcount * is NOT increased, since ephemeral objects does not need refcount * at all */ res = ucl_object_copy (obj); } else { res = __DECONST (ucl_object_t *, obj); #ifdef HAVE_ATOMIC_BUILTINS (void)__sync_add_and_fetch (&res->ref, 1); #else res->ref ++; #endif } } return res; } static ucl_object_t * ucl_object_copy_internal (const ucl_object_t *other, bool allow_array) { ucl_object_t *new; ucl_object_iter_t it = NULL; const ucl_object_t *cur; new = malloc (sizeof (*new)); if (new != NULL) { memcpy (new, other, sizeof (*new)); if (other->flags & UCL_OBJECT_EPHEMERAL) { /* Copied object is always non ephemeral */ new->flags &= ~UCL_OBJECT_EPHEMERAL; } new->ref = 1; /* Unlink from others */ new->next = NULL; new->prev = new; /* deep copy of values stored */ if (other->trash_stack[UCL_TRASH_KEY] != NULL) { new->trash_stack[UCL_TRASH_KEY] = strdup (other->trash_stack[UCL_TRASH_KEY]); if (other->key == (const char *)other->trash_stack[UCL_TRASH_KEY]) { new->key = new->trash_stack[UCL_TRASH_KEY]; } } if (other->trash_stack[UCL_TRASH_VALUE] != NULL) { new->trash_stack[UCL_TRASH_VALUE] = strdup (other->trash_stack[UCL_TRASH_VALUE]); if (new->type == UCL_STRING) { new->value.sv = new->trash_stack[UCL_TRASH_VALUE]; } } if (other->type == UCL_ARRAY || other->type == UCL_OBJECT) { /* reset old value */ memset (&new->value, 0, sizeof (new->value)); while ((cur = ucl_object_iterate (other, &it, true)) != NULL) { if (other->type == UCL_ARRAY) { ucl_array_append (new, ucl_object_copy_internal (cur, false)); } else { ucl_object_t *cp = ucl_object_copy_internal (cur, true); if (cp != NULL) { ucl_object_insert_key (new, cp, cp->key, cp->keylen, false); } } } } else if (allow_array && other->next != NULL) { LL_FOREACH (other->next, cur) { ucl_object_t *cp = ucl_object_copy_internal (cur, false); if (cp != NULL) { DL_APPEND (new, cp); } } } } return new; } ucl_object_t * ucl_object_copy (const ucl_object_t *other) { return ucl_object_copy_internal (other, true); } void ucl_object_unref (ucl_object_t *obj) { if (obj != NULL) { #ifdef HAVE_ATOMIC_BUILTINS unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1); if (rc == 0) { #else if (--obj->ref == 0) { #endif ucl_object_free_internal (obj, true, ucl_object_dtor_unref); } } } int ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2) { const ucl_object_t *it1, *it2; ucl_object_iter_t iter = NULL; int ret = 0; if (o1->type != o2->type) { return (o1->type) - (o2->type); } switch (o1->type) { case UCL_STRING: if (o1->len == o2->len && o1->len > 0) { ret = strcmp (ucl_object_tostring(o1), ucl_object_tostring(o2)); } else { ret = o1->len - o2->len; } break; case UCL_FLOAT: case UCL_INT: case UCL_TIME: ret = ucl_object_todouble (o1) - ucl_object_todouble (o2); break; case UCL_BOOLEAN: ret = ucl_object_toboolean (o1) - ucl_object_toboolean (o2); break; case UCL_ARRAY: if (o1->len == o2->len && o1->len > 0) { UCL_ARRAY_GET (vec1, o1); UCL_ARRAY_GET (vec2, o2); unsigned i; /* Compare all elements in both arrays */ for (i = 0; i < vec1->n; i ++) { it1 = kv_A (*vec1, i); it2 = kv_A (*vec2, i); if (it1 == NULL && it2 != NULL) { return -1; } else if (it2 == NULL && it1 != NULL) { return 1; } else if (it1 != NULL && it2 != NULL) { ret = ucl_object_compare (it1, it2); if (ret != 0) { break; } } } } else { ret = o1->len - o2->len; } break; case UCL_OBJECT: if (o1->len == o2->len && o1->len > 0) { while ((it1 = ucl_object_iterate (o1, &iter, true)) != NULL) { it2 = ucl_object_lookup (o2, ucl_object_key (it1)); if (it2 == NULL) { ret = 1; break; } ret = ucl_object_compare (it1, it2); if (ret != 0) { break; } } } else { ret = o1->len - o2->len; } break; default: ret = 0; break; } return ret; } int ucl_object_compare_qsort (const ucl_object_t **o1, const ucl_object_t **o2) { return ucl_object_compare (*o1, *o2); } void ucl_object_array_sort (ucl_object_t *ar, int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2)) { UCL_ARRAY_GET (vec, ar); if (cmp == NULL || ar == NULL || ar->type != UCL_ARRAY) { return; } qsort (vec->a, vec->n, sizeof (ucl_object_t *), (int (*)(const void *, const void *))cmp); } #define PRIOBITS 4 unsigned int ucl_object_get_priority (const ucl_object_t *obj) { if (obj == NULL) { return 0; } return (obj->flags >> ((sizeof (obj->flags) * NBBY) - PRIOBITS)); } void ucl_object_set_priority (ucl_object_t *obj, unsigned int priority) { if (obj != NULL) { priority &= (0x1 << PRIOBITS) - 1; priority <<= ((sizeof (obj->flags) * NBBY) - PRIOBITS); priority |= obj->flags & ((1 << ((sizeof (obj->flags) * NBBY) - PRIOBITS)) - 1); obj->flags = priority; } } bool ucl_object_string_to_type (const char *input, ucl_type_t *res) { if (strcasecmp (input, "object") == 0) { *res = UCL_OBJECT; } else if (strcasecmp (input, "array") == 0) { *res = UCL_ARRAY; } else if (strcasecmp (input, "integer") == 0) { *res = UCL_INT; } else if (strcasecmp (input, "number") == 0) { *res = UCL_FLOAT; } else if (strcasecmp (input, "string") == 0) { *res = UCL_STRING; } else if (strcasecmp (input, "boolean") == 0) { *res = UCL_BOOLEAN; } else if (strcasecmp (input, "null") == 0) { *res = UCL_NULL; } else if (strcasecmp (input, "userdata") == 0) { *res = UCL_USERDATA; } else { return false; } return true; } const char * ucl_object_type_to_string (ucl_type_t type) { const char *res = "unknown"; switch (type) { case UCL_OBJECT: res = "object"; break; case UCL_ARRAY: res = "array"; break; case UCL_INT: res = "integer"; break; case UCL_FLOAT: case UCL_TIME: res = "number"; break; case UCL_STRING: res = "string"; break; case UCL_BOOLEAN: res = "boolean"; break; case UCL_USERDATA: res = "userdata"; break; case UCL_NULL: res = "null"; break; } return res; } const ucl_object_t * ucl_parser_get_comments (struct ucl_parser *parser) { if (parser && parser->comments) { return parser->comments; } return NULL; } const ucl_object_t * ucl_comments_find (const ucl_object_t *comments, const ucl_object_t *srch) { if (comments && srch) { return ucl_object_lookup_len (comments, (const char *)&srch, sizeof (void *)); } return NULL; } bool ucl_comments_move (ucl_object_t *comments, const ucl_object_t *from, const ucl_object_t *to) { const ucl_object_t *found; ucl_object_t *obj; if (comments && from && to) { found = ucl_object_lookup_len (comments, (const char *)&from, sizeof (void *)); if (found) { /* Replace key */ obj = ucl_object_ref (found); ucl_object_delete_keyl (comments, (const char *)&from, sizeof (void *)); ucl_object_insert_key (comments, obj, (const char *)&to, sizeof (void *), true); return true; } } return false; } void ucl_comments_add (ucl_object_t *comments, const ucl_object_t *obj, const char *comment) { if (comments && obj && comment) { ucl_object_insert_key (comments, ucl_object_fromstring (comment), (const char *)&obj, sizeof (void *), true); } } Index: head/contrib/libucl/tests/schema/patternProperties.json =================================================================== --- head/contrib/libucl/tests/schema/patternProperties.json (revision 306543) +++ head/contrib/libucl/tests/schema/patternProperties.json (revision 306544) @@ -1,110 +1,115 @@ [ { "description": "patternProperties validates properties matching a regex", "schema": { "patternProperties": { "f.*o": {"type": "integer"} } }, "tests": [ { "description": "a single valid match is valid", "data": {"foo": 1}, "valid": true }, { "description": "multiple valid matches is valid", "data": {"foo": 1, "foooooo" : 2}, "valid": true }, { "description": "a single invalid match is invalid", "data": {"foo": "bar", "fooooo": 2}, "valid": false }, { + "description": "a single invalid match is invalid", + "data": {"fooooo": 2, "foo": "bar"}, + "valid": false + }, + { "description": "multiple invalid matches is invalid", "data": {"foo": "bar", "foooooo" : "baz"}, "valid": false }, { "description": "ignores non-objects", "data": 12, "valid": true } ] }, { "description": "multiple simultaneous patternProperties are validated", "schema": { "patternProperties": { "a*": {"type": "integer"}, "aaa*": {"maximum": 20} } }, "tests": [ { "description": "a single valid match is valid", "data": {"a": 21}, "valid": true }, { "description": "a simultaneous match is valid", "data": {"aaaa": 18}, "valid": true }, { "description": "multiple matches is valid", "data": {"a": 21, "aaaa": 18}, "valid": true }, { "description": "an invalid due to one is invalid", "data": {"a": "bar"}, "valid": false }, { "description": "an invalid due to the other is invalid", "data": {"aaaa": 31}, "valid": false }, { "description": "an invalid due to both is invalid", "data": {"aaa": "foo", "aaaa": 31}, "valid": false } ] }, { "description": "regexes are not anchored by default and are case sensitive", "schema": { "patternProperties": { "[0-9]{2,}": { "type": "boolean" }, "X_": { "type": "string" } } }, "tests": [ { "description": "non recognized members are ignored", "data": { "answer 1": "42" }, "valid": true }, { "description": "recognized members are accounted for", "data": { "a31b": null }, "valid": false }, { "description": "regexes are case sensitive", "data": { "a_x_3": 3 }, "valid": true }, { "description": "regexes are case sensitive, 2", "data": { "a_X_3": 3 }, "valid": false } ] } ] Index: head/contrib/libucl/tests/schema/refRemote.json =================================================================== --- head/contrib/libucl/tests/schema/refRemote.json (revision 306543) +++ head/contrib/libucl/tests/schema/refRemote.json (revision 306544) @@ -1,76 +1,76 @@ [ { "description": "remote ref", "schema": {"$ref": "http://highsecure.ru/ucl-schema/remotes/integer.json"}, "tests": [ { "description": "remote ref valid", "data": 1, "valid": true }, { "description": "remote ref invalid", "data": "a", "valid": false } ] }, { "description": "fragment within remote ref", "schema": {"$ref": "http://highsecure.ru/ucl-schema/remotes/subSchemas.json#/integer"}, "tests": [ { "description": "remote fragment valid", "data": 1, "valid": true }, { "description": "remote fragment invalid", "data": "a", "valid": false } ] }, { "description": "ref within remote ref", "schema": { "$ref": "http://highsecure.ru/ucl-schema/remotes/subSchemas.json#/refToInteger" }, "tests": [ { "description": "ref within ref valid", "data": 1, "valid": true }, { "description": "ref within ref invalid", "data": "a", "valid": false } ] - }, + } /* { "description": "change resolution scope", "schema": { "id": "http://highsecure.ru/ucl-schema/remotes/", "items": { "id": "folder/", "items": {"$ref": "folderInteger.json"} } }, "tests": [ { "description": "changed scope ref valid", "data": [[1]], "valid": true }, { "description": "changed scope ref invalid", "data": [["a"]], "valid": false } ] } */ ] Index: head/contrib/libucl =================================================================== --- head/contrib/libucl (revision 306543) +++ head/contrib/libucl (revision 306544) Property changes on: head/contrib/libucl ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /vendor/libucl/dist:r301337-306543