diff --git a/.gitignore b/.gitignore index ebaa5be7f..d1bce2041 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ doc/generated tags compile_commands.json .ycm_extra_conf.py +python/comma_py.egg-info/ +python/dist/ +build diff --git a/AUTHORS b/AUTHORS index 53aaa505a..b3ae456ea 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,4 +1,4 @@ - Vsevolod Vlaskine - Cedric Wohlleber - Matthew Herrmann - James Underwood +Vsevolod Vlaskine +Cedric Wohlleber +Matthew Herrmann +James Underwood diff --git a/CMakeFiles/FindComma.cmake b/CMakeFiles/FindComma.cmake index 599c3f7f9..12ec30316 100644 --- a/CMakeFiles/FindComma.cmake +++ b/CMakeFiles/FindComma.cmake @@ -3,10 +3,10 @@ # The following variables are set if comma is found. # comma_FOUND - Set to true when comma is found. # comma_USE_FILE - CMake file to use comma. -# comma_MAJOR_VERSION - The comma major version number. -# comma_MINOR_VERSION - The comma minor version number +# comma_version_major - The comma major version number. +# comma_version_minor - The comma minor version number # (odd non-release). -# comma_BUILD_VERSION - The comma patch level +# comma_version_patch - The comma patch level # (meaningless for odd minor). # comma_INCLUDE_DIRS - Include directories for comma # comma_LIBRARY_DIRS - Link directories for comma libraries @@ -85,4 +85,3 @@ ENDIF ( comma_FOUND ) IF( NOT comma_FOUND ) MESSAGE(FATAL_ERROR ${comma_DIR_MESSAGE}) ENDIF( NOT comma_FOUND ) - diff --git a/CMakeFiles/check.c++.standard.cmake b/CMakeFiles/check.c++.standard.cmake index fd1942322..f17089638 100644 --- a/CMakeFiles/check.c++.standard.cmake +++ b/CMakeFiles/check.c++.standard.cmake @@ -1,7 +1,7 @@ - SET( CXX_STANDARDS "0x;11;14;17" CACHE STRING "list of known c++ standards" ) + SET( CXX_STANDARDS "0x;11;14;17;20" CACHE STRING "list of known c++ standards" ) MARK_AS_ADVANCED( FORCE CXX_STANDARDS ) STRING( REGEX REPLACE ";" "," CXX_STANDARDS_READABLE "${CXX_STANDARDS}" ) - SET( CXX_STANDARD_DEFAULT "11" CACHE STRING "default c++ standard to use" ) + SET( CXX_STANDARD_DEFAULT "17" CACHE STRING "default c++ standard to use" ) MARK_AS_ADVANCED( FORCE CXX_STANDARD_DEFAULT ) SET( CXX_STANDARD_TO_USE "${CXX_STANDARD_DEFAULT}" CACHE STRING "c++ standard to use (one of ${CXX_STANDARDS_READABLE})" ) SET_PROPERTY( CACHE CXX_STANDARD_TO_USE PROPERTY STRINGS ${CXX_STANDARDS} ) @@ -17,7 +17,6 @@ MARK_AS_ADVANCED( FORCE CXX_STANDARD_FLAGS ) ENDIF() - # A much better way to do this is with CXX_STANDARD but that requires CMake 3.1 include( CheckCXXCompilerFlag ) IF( ${CXX_STANDARD_TO_USE} MATCHES "0x" ) message( WARNING " @@ -28,27 +27,22 @@ ######################################################################### " ) ENDIF() - FOREACH( STANDARD ${CXX_STANDARDS} ) - # message( "Check if using C++${STANDARD}" ) - IF( ${CXX_STANDARD_TO_USE} MATCHES "${STANDARD}" ) - # message( "Yes, using C++${STANDARD}" ) - IF( NOT ( ${CXX_STANDARD_TO_USE} MATCHES ${CXX_STANDARD_LAST} ) ) - # message( "Have to check if ${CMAKE_CXX_COMPILER} supports C++${STANDARD}" ) - message( "Attempt to use C++ standard ${STANDARD}" ) - UNSET( compiler_supports_standard CACHE ) - UNSET( compiler_flag_to_check CACHE ) - SET( compiler_flag_to_check "-std=c++${STANDARD}" ) - if ( CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND ${STANDARD} MATCHES "11" ) - set( compiler_flag_to_check "${compiler_flag_to_check} -Wc++11-narrowing" ) - endif() - CHECK_CXX_COMPILER_FLAG( "${compiler_flag_to_check}" compiler_supports_standard ) - if( NOT compiler_supports_standard ) - message( FATAL_ERROR "attempt to use C++ standard ${STANDARD} but ${CMAKE_CXX_COMPILER} does not support it" ) - endif() - STRING( REPLACE " ${CXX_STANDARD_FLAGS}" "" CXX_FLAGS_NO_STANDARD "${CMAKE_CXX_FLAGS}" ) - SET( CXX_STANDARD_FLAGS ${compiler_flag_to_check} CACHE STRING "updating compiler flags selecting C++ standard" FORCE ) - SET( CXX_STANDARD_LAST ${CXX_STANDARD_TO_USE} CACHE STRING "updating C++ standard to use option" FORCE ) - set( CMAKE_CXX_FLAGS "${CXX_FLAGS_NO_STANDARD} ${compiler_flag_to_check}" CACHE STRING "" FORCE ) - ENDIF() - ENDIF() - ENDFOREACH() + + IF( NOT ( ${CXX_STANDARD_TO_USE} MATCHES ${CXX_STANDARD_LAST} ) ) + # message( "Have to check if ${CMAKE_CXX_COMPILER} supports C++${CXX_STANDARD_TO_USE}" ) + message( "Attempt to use C++ standard ${CXX_STANDARD_TO_USE}" ) + UNSET( compiler_supports_standard CACHE ) + UNSET( compiler_flag_to_check CACHE ) + SET( compiler_flag_to_check "-std=c++${CXX_STANDARD_TO_USE}" ) + if ( CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND ${CXX_STANDARD_TO_USE} MATCHES "11" ) + set( extra_compiler_flags "${compiler_flag_to_check} -Wc++11-narrowing" ) + endif() + CHECK_CXX_COMPILER_FLAG( "${compiler_flag_to_check} ${extra_compiler_flags}" compiler_supports_standard ) + if( NOT compiler_supports_standard ) + message( FATAL_ERROR "attempt to use C++ standard ${CXX_STANDARD_TO_USE} but ${CMAKE_CXX_COMPILER} does not support it" ) + endif() + SET( CXX_STANDARD_LAST ${CXX_STANDARD_TO_USE} CACHE STRING "updating C++ standard to use option" FORCE ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${extra_compiler_flags}" CACHE STRING "" FORCE ) + ENDIF() + + set( CMAKE_CXX_STANDARD ${CXX_STANDARD_TO_USE} ) diff --git a/CMakeFiles/comma-config.cmake.in b/CMakeFiles/comma-config.cmake.in index 0b3eda1b6..b9bbcd676 100644 --- a/CMakeFiles/comma-config.cmake.in +++ b/CMakeFiles/comma-config.cmake.in @@ -35,9 +35,13 @@ SET(comma_REQUIRED_EXE_LINKER_FLAGS "@comma_REQUIRED_EXE_LINKER_FLAGS@") SET(comma_REQUIRED_SHARED_LINKER_FLAGS "@comma_REQUIRED_SHARED_LINKER_FLAGS@") SET(comma_REQUIRED_MODULE_LINKER_FLAGS "@comma_REQUIRED_MODULE_LINKER_FLAGS@") -SET(comma_MAJOR_VERSION "@comma_MAJOR_VERSION@") -SET(comma_MINOR_VERSION "@comma_MINOR_VERSION@") -SET(comma_BUILD_VERSION "@comma_BUILD_VERSION@") +SET(comma_version_major "@comma_version_major@") +SET(comma_version_minor "@comma_version_minor@") +SET(comma_version_patch "@comma_version_patch@") +SET(comma_MAJOR_VERSION "@comma_version_major@") # quick and dirty, otherwise comma so library version in dependent packages seem to get messed up +SET(comma_MINOR_VERSION "@comma_version_minor@") # quick and dirty, otherwise comma so library version in dependent packages seem to get messed up +SET(comma_BUILD_VERSION "@comma_version_patch@") # quick and dirty, otherwise comma so library version in dependent packages seem to get messed up +set(comma_version "@comma_version@") SET(comma_USE_FILE "@comma_USE_FILE@") diff --git a/CMakeFiles/post_install/CMakeLists.txt b/CMakeFiles/post_install/CMakeLists.txt new file mode 100644 index 000000000..cf00749b3 --- /dev/null +++ b/CMakeFiles/post_install/CMakeLists.txt @@ -0,0 +1,6 @@ +if( BUILD_SHARED_LIBS ) + install( CODE "message( \"ldconfig: running...\" )" ) + install( CODE "execute_process( COMMAND /sbin/ldconfig )" ) + install( CODE "message( \"ldconfig: done\" )" ) +endif( BUILD_SHARED_LIBS ) + diff --git a/CMakeLists.txt b/CMakeLists.txt index 167b087db..9bc598eaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,15 +8,30 @@ if( USE_ARM_TOOLCHAIN ) include( ${CMAKE_TOOLCHAIN_FILE} REQUIRED ) endif( USE_ARM_TOOLCHAIN ) -PROJECT( "comma" ) - -cmake_minimum_required(VERSION 2.6) +if( ${CMAKE_MAJOR_VERSION} LESS 4 ) + cmake_minimum_required( VERSION 3.1 ) +else() + cmake_minimum_required( VERSION 3.5 ) +endif() +cmake_policy( SET CMP0048 NEW ) +project( comma VERSION 1.1.0 LANGUAGES CXX ) +if( NOT DEFINED CMAKE_PROJECT_VERSION ) + set( CMAKE_PROJECT_VERSION ${PROJECT_VERSION} ) + set( CMAKE_PROJECT_VERSION_MAJOR ${PROJECT_VERSION_MAJOR} ) + set( CMAKE_PROJECT_VERSION_MINOR ${PROJECT_VERSION_MINOR} ) + set( CMAKE_PROJECT_VERSION_PATCH ${PROJECT_VERSION_PATCH} ) +endif( NOT DEFINED CMAKE_PROJECT_VERSION ) + +set( comma_version_major "${CMAKE_PROJECT_VERSION_MAJOR}" ) # quick and dirty +set( comma_version_minor "${CMAKE_PROJECT_VERSION_MINOR}" ) # quick and dirty +set( comma_version_patch "${CMAKE_PROJECT_VERSION_PATCH}" ) # quick and dirty +set( comma_version "${CMAKE_PROJECT_VERSION}" ) # quick and dirty # option( USE_ARM_TOOLCHAIN "Cross compile using arm toolchain" OFF ) # if( USE_ARM_TOOLCHAIN ) # set(CMAKE_TOOLCHAIN_FILE ${SOURCE_CODE_BASE_DIR}/CMakeFiles/arm.toolchain.cmake) # endif( USE_ARM_TOOLCHAIN ) -# +# # option(my_test_option "whatever" OFF) # if(my_test_option) # set(my_other_option "is set") @@ -50,20 +65,15 @@ IF (UNIX AND NOT APPLE) SET ( COMMITHASH "abc" ) ENDIF() ELSE() # Probably windows; gets hardcoded patch version - SET ( COMMITDATE 1 ) + SET ( COMMITDATE 1 ) SET ( COMMITHASH "abc" ) ENDIF() -# --------------------------------------------------------------------------- -# set compiler and linker flags and variables +if( NOT CMAKE_BUILD_TYPE ) + set( CMAKE_BUILD_TYPE Release CACHE STRING "options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE ) +endif( NOT CMAKE_BUILD_TYPE ) -IF(NOT CMAKE_BUILD_TYPE) - SET(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING - "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." - FORCE) -ENDIF(NOT CMAKE_BUILD_TYPE) - -IF(WIN32) +if( WIN32 ) SET(CMAKE_CXX_WARNING_LEVEL 4) IF(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") STRING(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") @@ -78,7 +88,16 @@ IF(WIN32) SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D \"NOMINMAX\"") ADD_DEFINITIONS(-D_USE_MATH_DEFINES) ADD_DEFINITIONS(-DWIN32_LEAN_AND_MEAN) #used to fix winsock redefinition error -ENDIF(WIN32) +else( WIN32 ) + option( ENABLE_FAST_MATH "enable compiler fast-math; experimental, some tests fail on nan and precision" OFF ) + if( ENABLE_FAST_MATH ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math" ) + endif( ENABLE_FAST_MATH ) + option( ENABLE_OPTIMIZATION "enable compiler optimisation" ON ) + if( ENABLE_OPTIMIZATION ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3" ) # quick and dirty; make optimisation level configurable + endif( ENABLE_OPTIMIZATION ) +endif( WIN32 ) IF( CMAKE_SIZEOF_VOID_P EQUAL 8 ) SET( IS_64_BIT TRUE ) @@ -106,33 +125,18 @@ IF(CMAKE_BUILD_TOOL MATCHES "make") ENDIF(CMAKE_BUILD_TOOL MATCHES "make") if( UNIX ) -INCLUDE( CMakeFiles/check.c++.standard.cmake REQUIRED ) -endif() + include( CMakeFiles/check.c++.standard.cmake REQUIRED ) +endif( UNIX ) SET( SOURCE_CODE_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR} ) SET( LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR}/lib ) -SET( EXECUTABLE_OUTPUT_PATH - ${PROJECT_BINARY_DIR}/bin CACHE PATH - "Single output directory for building all executables." ) - -SET( comma_LIBRARY_PROPERTIES ${comma_LIBRARY_PROPERTIES} DEBUG_POSTFIX "-d" ) - -IF( NOT comma_INSTALL_BIN_DIR ) - SET( comma_INSTALL_BIN_DIR bin ) -ENDIF( NOT comma_INSTALL_BIN_DIR ) - -IF( NOT comma_INSTALL_INCLUDE_DIR ) - SET( comma_INSTALL_INCLUDE_DIR include/${PROJECT_NAME} ) -ENDIF( NOT comma_INSTALL_INCLUDE_DIR ) +set( comma_INSTALL_BIN_DIR bin CACHE PATH "installation directory for binaries" ) +set( comma_INSTALL_INCLUDE_DIR include/comma CACHE PATH "installation directory for include directories" ) +set( comma_INSTALL_LIB_DIR lib CACHE PATH "installation directory for libraries" ) +set( comma_INSTALL_PACKAGE_DIR CMakeFiles CACHE PATH "installation directory for cmake files" ) -IF( NOT comma_INSTALL_LIB_DIR) - SET( comma_INSTALL_LIB_DIR lib ) -ENDIF( NOT comma_INSTALL_LIB_DIR ) - -IF( NOT comma_INSTALL_PACKAGE_DIR ) - SET( comma_INSTALL_PACKAGE_DIR CMakeFiles ) -ENDIF( NOT comma_INSTALL_PACKAGE_DIR ) +SET( comma_LIBRARY_PROPERTIES ${comma_LIBRARY_PROPERTIES} DEBUG_POSTFIX "-d" SOVERSION ${CMAKE_PROJECT_VERSION} LINK_FLAGS_RELEASE -s ) #INCLUDE_DIRECTORIES( ${comma_INSTALL_INCLUDE_DIR}/../ ) @@ -143,7 +147,7 @@ ENDIF( APPLE ) OPTION( BUILD_TESTS "build unit tests" OFF ) SET( comma_BUILD_TESTS ${BUILD_TESTS} ) -OPTION( BUILD_PYTHON_PACKAGES "install comma python packages" ON ) +option( BUILD_PYTHON_PACKAGES "install comma python packages" ON ) INCLUDE( CMakeDependentOption ) CMAKE_DEPENDENT_OPTION( ADD_PYTHON_PACKAGES_TO_RPM "add comma python packages to rpm" OFF "BUILD_PYTHON_PACKAGES" ON ) @@ -152,11 +156,21 @@ CMAKE_DEPENDENT_OPTION( BUILD_CPP_PYTHON_BINDINGS "build C++/Python bindings" OF OPTION( BUILD_APPLICATIONS "build applications" ON ) SET( comma_BUILD_APPLICATIONS ${BUILD_APPLICATIONS} ) -OPTION( comma_BUILD_ZEROMQ "build io with zeromq support" OFF ) - -OPTION(BUILD_SHARED_LIBS "build with shared libraries" OFF) - -OPTION( comma_BUILD_XML "build xml" OFF ) +option( BUILD_SHARED_LIBS "build with shared libraries" ON ) +option( comma_BUILD_NAME_VALUE_YAML "build name_value with yaml support; requires libyaml; e.g. apt install libyaml-dev" OFF ) +option( comma_BUILD_PYTHON "build python" ON ) +option( comma_BUILD_XML "build xml" OFF ) +option( comma_BUILD_ZEROMQ "build io with zeromq support" OFF ) +option( comma_USE_BOOST_FILESYSTEM "use boost::filesystem rather than std::filesystem" OFF ) + +if( comma_USE_BOOST_FILESYSTEM ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCOMMA_USE_BOOST_FILESYSTEM -DBOOST_FILESYSTEM_NO_DEPRECATED" ) +else() + # see https://gcc.gnu.org/onlinedocs/libstdc++/manual/status.html#status.iso.2017 + if( CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9 ) + link_libraries( "-lstdc++fs" ) + endif() +endif() SET(comma_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) @@ -164,6 +178,11 @@ IF( NOT BUILD_SHARED_LIBS ) SET( Boost_USE_STATIC_LIBS ON ) ENDIF( NOT BUILD_SHARED_LIBS ) +set( comma_boost_components thread system date_time iostreams program_options regex ) +if( comma_USE_BOOST_FILESYSTEM ) + set( comma_boost_components ${comma_boost_components} filesystem ) +endif() + IF( BUILD_PYTHON_PACKAGES AND BUILD_CPP_PYTHON_BINDINGS ) IF( NOT BUILD_SHARED_LIBS ) MESSAGE( WARNING "Linking C++/Python bindings with static libraries is requested. If linking fails, recompile required libraries with -fPIC or use shared libraries. Note that this feature has only been tested with shared libraries." ) @@ -171,41 +190,44 @@ IF( BUILD_PYTHON_PACKAGES AND BUILD_CPP_PYTHON_BINDINGS ) SET( Python_ADDITIONAL_VERSIONS 2.7 ) FIND_PACKAGE( PythonLibs REQUIRED ) INCLUDE_DIRECTORIES( ${PYTHON_INCLUDE_DIRS} ) - FIND_PACKAGE( Boost COMPONENTS thread filesystem system date_time program_options regex python ) -ELSE( BUILD_PYTHON_PACKAGES AND BUILD_CPP_PYTHON_BINDINGS ) - FIND_PACKAGE( Boost COMPONENTS thread filesystem system date_time program_options regex ) + set( comma_boost_components ${comma_boost_components} python ) ENDIF( BUILD_PYTHON_PACKAGES AND BUILD_CPP_PYTHON_BINDINGS ) + +FIND_PACKAGE( Boost COMPONENTS ${comma_boost_components} ) INCLUDE_DIRECTORIES( ${Boost_INCLUDE_DIRS} ) LINK_DIRECTORIES( ${Boost_LIBRARY_DIRS} ) IF( comma_BUILD_TESTS ) +find_package( GTest REQUIRED ) +include_directories( ${GTEST_INCLUDE_DIRS} ) + # todo: maybe else clause is actually the right way to do things - IF( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Windows" ) - FIND_PATH( GTest_SOURCE_DIR NAMES CMakeLists.txt HINTS /usr/src/gtest DOC "source code of gtest" ) - ADD_SUBDIRECTORY( ${GTest_SOURCE_DIR} gtest ) - INCLUDE_DIRECTORIES( src ${GTest_SOURCE_DIR}/include ${GTest_SOURCE_DIR} ) - SET( GTEST_BOTH_LIBRARIES gtest gtest_main ) - ELSE( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Windows" ) - FIND_PACKAGE( GTest ) - INCLUDE_DIRECTORIES( ${GTEST_INCLUDE_DIRS} ) - ENDIF( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Windows" ) - +# IF( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Windows" ) +# FIND_PATH( GTest_SOURCE_DIR NAMES CMakeLists.txt HINTS /usr/src/googletest/googletest DOC "source code of gtest" ) # FIND_PATH( GTest_SOURCE_DIR NAMES CMakeLists.txt HINTS /usr/src/gtest DOC "source code of gtest" ) +# add_subdirectory( ${GTest_SOURCE_DIR} gtest ) +# INCLUDE_DIRECTORIES( src ${GTest_SOURCE_DIR}/include ${GTest_SOURCE_DIR} ) +# SET( GTEST_BOTH_LIBRARIES gtest gtest_main ) +# ELSE( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Windows" ) +# FIND_PACKAGE( GTest ) +# INCLUDE_DIRECTORIES( ${GTEST_INCLUDE_DIRS} ) +# ENDIF( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Windows" ) + #TT: Install tests to dir. This allows them to be packed by CPack. #Note: if your source tree is dirty (eg you've already run tests there and have output/ directories) #those will also be installed too. OPTION( INSTALL_TESTS "Install the tests into the nominated directory" OFF ) IF( INSTALL_TESTS ) - SET ( comma_CPP_TESTS_INSTALL_DIR bin CACHE PATH "Install directory for cpp unit tests" ) - SET ( comma_INSTALL_COMMA_TESTS_DIR share/test ) - +SET ( comma_CPP_TESTS_INSTALL_DIR bin CACHE PATH "Install directory for cpp unit tests" ) +SET ( comma_INSTALL_COMMA_TESTS_DIR share/test ) + # Get the list of test directories (using glob). -FILE ( GLOB_RECURSE all_files +FILE ( GLOB_RECURSE all_files # RELATIVE is tricky. When you use it, you need to specify the path you expect the output to be RELATIVE to. (Yes, that path could be considerably non-relative to the file that is found.) - RELATIVE ${CMAKE_SOURCE_DIR} + RELATIVE ${CMAKE_SOURCE_DIR} * ) - + SET ( test_dirs ) FOREACH ( i ${all_files} ) if ( ${i} MATCHES "[\\/]test[\\/]" ) @@ -214,19 +236,18 @@ FOREACH ( i ${all_files} ) ENDFOREACH (i) # Add those directories - FOREACH (file ${test_dirs} ) - # Determine the parent directories for a given file so to prefix the install location (otherwise they just all get installed in the destination root) - get_filename_component( dest_dir ${file} DIRECTORY ) - STRING( REGEX MATCH ".*\\/output\\/.*" skip ${file} ) - IF( NOT skip ) - # See that slash there? vvv It's really important. - INSTALL ( DIRECTORY ${dest_dir}/ - DESTINATION ${comma_INSTALL_COMMA_TESTS_DIR}/${CMAKE_PROJECT_NAME}/unit/${dest_dir} - USE_SOURCE_PERMISSIONS - PATTERN "output" EXCLUDE ) - ENDIF( NOT skip ) - ENDFOREACH (file) - +FOREACH (file ${test_dirs} ) + # Determine the parent directories for a given file so to prefix the install location (otherwise they just all get installed in the destination root) + get_filename_component( dest_dir ${file} DIRECTORY ) + STRING( REGEX MATCH ".*\\/output\\/.*" skip ${file} ) + IF( NOT skip ) + # See that slash there? vvv It's really important. + INSTALL ( DIRECTORY ${dest_dir}/ + DESTINATION ${comma_INSTALL_COMMA_TESTS_DIR}/${CMAKE_PROJECT_NAME}/unit/${dest_dir} + USE_SOURCE_PERMISSIONS + PATTERN "output" EXCLUDE ) + ENDIF( NOT skip ) +ENDFOREACH (file) ENDIF( INSTALL_TESTS ) ENDIF( comma_BUILD_TESTS ) @@ -250,35 +271,37 @@ SET( comma_ALL_LIBRARIES debug comma_xpath-d optimized comma_xpath debug comma_name_value-d - optimized comma_name_value ) + optimized comma_name_value + debug comma_timing-d + optimized comma_timing ) -IF(WIN32) - SET( comma_ALL_EXTERNAL_LIBRARIES Ws2_32.lib ) -ENDIF(WIN32) +if(WIN32) + set( comma_ALL_EXTERNAL_LIBRARIES Ws2_32.lib ) +endif(WIN32) -IF(UNIX AND NOT QNXNTO AND NOT APPLE) - SET( comma_ALL_EXTERNAL_LIBRARIES rt pthread ) -ENDIF(UNIX AND NOT QNXNTO AND NOT APPLE) +if( UNIX AND NOT QNXNTO AND NOT APPLE ) + set( comma_ALL_EXTERNAL_LIBRARIES rt pthread c ) +endif( UNIX AND NOT QNXNTO AND NOT APPLE ) -SET( comma_ALL_EXTERNAL_LIBRARIES ${Boost_LIBRARIES} ${comma_ALL_EXTERNAL_LIBRARIES} ) +set( comma_ALL_EXTERNAL_LIBRARIES ${Boost_LIBRARIES} ${comma_ALL_EXTERNAL_LIBRARIES} ) -IF( comma_BUILD_ZEROMQ ) - SET( comma_ALL_EXTERNAL_LIBRARIES ${comma_ALL_EXTERNAL_LIBRARIES} ${ZeroMQ_LIBRARY} ) -ENDIF( comma_BUILD_ZEROMQ ) +if( comma_BUILD_ZEROMQ ) + set( comma_ALL_EXTERNAL_LIBRARIES ${comma_ALL_EXTERNAL_LIBRARIES} ${ZeroMQ_LIBRARY} ) +endif( comma_BUILD_ZEROMQ ) -IF( BUILD_CPP_PYTHON_BINDINGS ) - SET( comma_ALL_EXTERNAL_LIBRARIES ${comma_ALL_EXTERNAL_LIBRARIES} ${PYTHON_LIBRARY} ) -ENDIF( BUILD_CPP_PYTHON_BINDINGS ) +if( BUILD_CPP_PYTHON_BINDINGS ) + set( comma_ALL_EXTERNAL_LIBRARIES ${comma_ALL_EXTERNAL_LIBRARIES} ${PYTHON_LIBRARY} ) +endif( BUILD_CPP_PYTHON_BINDINGS ) -SET( comma_ALL_LIBRARIES ${comma_ALL_LIBRARIES} ${comma_ALL_LIBRARIES} ${comma_ALL_EXTERNAL_LIBRARIES} ) # as gcc parses library list only once -SET( comma_EXTERNAL_INCLUDES ${Boost_INCLUDE_DIRS} ) +set( comma_ALL_LIBRARIES ${comma_ALL_LIBRARIES} ${comma_ALL_LIBRARIES} ${comma_ALL_EXTERNAL_LIBRARIES} ) # as gcc parses library list only once +set( comma_EXTERNAL_INCLUDES ${Boost_INCLUDE_DIRS} ) CONFIGURE_FILE( "${CMAKE_CURRENT_SOURCE_DIR}/CMakeFiles/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY) -ADD_CUSTOM_TARGET(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) +ADD_CUSTOM_TARGET( uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake ) FILE( GLOB comma_PACKAGE_CMAKEFILES ${CMAKE_CURRENT_SOURCE_DIR}/CMakeFiles/*.cmake ) @@ -287,73 +310,143 @@ INCLUDE( ${CMAKE_CURRENT_SOURCE_DIR}/CMakeFiles/generate_${PROJECT_NAME}_config. CONFIGURE_FILE( ${PROJECT_SOURCE_DIR}/CMakeFiles/use_${PROJECT_NAME}.cmake.in ${PROJECT_BINARY_DIR}/CMakeFiles/use_${PROJECT_NAME}.cmake COPYONLY IMMEDIATE ) -SET( comma_PACKAGE_CMAKEFILES ${comma_PACKAGE_CMAKEFILES} ${PROJECT_BINARY_DIR}/CMakeFiles/use_${PROJECT_NAME}.cmake ) -INSTALL( FILES ${comma_PACKAGE_CMAKEFILES} - DESTINATION ${comma_INSTALL_PACKAGE_DIR} ) +set( comma_PACKAGE_CMAKEFILES ${comma_PACKAGE_CMAKEFILES} ${PROJECT_BINARY_DIR}/CMakeFiles/use_${PROJECT_NAME}.cmake ) +install( FILES ${comma_PACKAGE_CMAKEFILES} DESTINATION ${comma_INSTALL_PACKAGE_DIR} ) +#install( FILES ${PROJECT_SOURCE_DIR}/copyright DESTINATION share/doc/comma ) # quick and dirty #----------------------------------------------------------------------------- # add sources -ADD_SUBDIRECTORY( application ) -ADD_SUBDIRECTORY( base ) -ADD_SUBDIRECTORY( bash ) -ADD_SUBDIRECTORY( containers ) -ADD_SUBDIRECTORY( csv ) -ADD_SUBDIRECTORY( dispatch ) -ADD_SUBDIRECTORY( doc ) -ADD_SUBDIRECTORY( io ) -ADD_SUBDIRECTORY( math ) -ADD_SUBDIRECTORY( name_value ) -ADD_SUBDIRECTORY( packed ) -ADD_SUBDIRECTORY( string ) -ADD_SUBDIRECTORY( sync ) -ADD_SUBDIRECTORY( util ) -ADD_SUBDIRECTORY( visiting ) +add_subdirectory( application ) +add_subdirectory( base ) +add_subdirectory( bash ) +add_subdirectory( containers ) +add_subdirectory( csv ) +add_subdirectory( dispatch ) +add_subdirectory( doc ) +add_subdirectory( dynamic ) +add_subdirectory( io ) +add_subdirectory( math ) +add_subdirectory( name_value ) +add_subdirectory( packed ) +add_subdirectory( string ) +add_subdirectory( sync ) +add_subdirectory( timing ) +add_subdirectory( util ) +add_subdirectory( visiting ) add_subdirectory( web ) if( comma_BUILD_XML ) add_subdirectory( xml ) endif( comma_BUILD_XML ) add_subdirectory( xpath ) -ADD_SUBDIRECTORY( etc/bash_completion.d ) +option( INSTALL_BASH_COMPLETION "install the BASH completion scripts" ON ) +if( INSTALL_BASH_COMPLETION ) + add_subdirectory( etc/bash_completion.d ) +endif( INSTALL_BASH_COMPLETION ) -ADD_SUBDIRECTORY( python ) +if( comma_BUILD_PYTHON ) + add_subdirectory( python ) +endif( comma_BUILD_PYTHON ) #in future, may need to add third party dlls somehow, if they are linked as dlls. Not needed for statically linking to boost. #if we set them up to install to bin they might be packaged through, e.g. roughly (using GLOB): #install(FILES "${Boost_LIBRARY_DIRS}/*.dll" DESTINATION bin ) -#the following set up optional parameters for cpack -SET(CPACK_PACKAGE_NAME "comma") -SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Command line utilities to manipulate various structured and unstructured data from offline or real time data.") -SET(CPACK_PACKAGE_VENDOR "Australian Centre for Field Robotics") -SET(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/COPYING") -SET(CPACK_PACKAGE_VERSION_MAJOR "1") -SET(CPACK_PACKAGE_VERSION_MINOR ${COMMITDATE}) -SET(CPACK_PACKAGE_VERSION_PATCH ${COMMITHASH}) -SET(CPACK_PACKAGE_CONTACT "t.teo@acfr.usyd.edu.au") - -# If the user builds shared, make sure that the RPM package (that might be generated) runs ldconfig when it's installed. -IF( BUILD_SHARED_LIBS ) - SET(CPACK_RPM_SPEC_MORE_DEFINE "%posttrans -p /sbin/ldconfig") -ENDIF( BUILD_SHARED_LIBS ) - +if( CMAKE_COMPILER_IS_GNUCXX ) + if( CMAKE_BUILD_TYPE STREQUAL "Release" ) + set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -s" ) + set( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -s" ) + endif( CMAKE_BUILD_TYPE STREQUAL "Release" ) +endif( CMAKE_COMPILER_IS_GNUCXX ) + +option( comma_INSTALL_RUN_POST_INSTALL "run post install step (currently just ldconfig)" ON ) +if( comma_INSTALL_RUN_POST_INSTALL ) + add_subdirectory( CMakeFiles/post_install ) # uber-quick and dirty +endif( comma_INSTALL_RUN_POST_INSTALL ) + +# todo +# build ppa +# see here? http://schneegans.github.io/lessons/2011/11/02/ppa-launchpad-cmake +# lintian errors and warnings +# fix warnings +# W: comma: package-name-doesnt-match-sonames: try comma_string -> comma-string, etc? +# W: comma: maintscript-calls-ldconfig [postinst]: don't call ldconfig from package? +# W: comma: maintscript-calls-ldconfig [postrm]: don't call ldconfig from package? +# W: comma: distant-prerequisite-in-shlibs comma (ON 1.0.0): try to add full version to shared libs +# investigate +# W: comma: non-standard-dir-in-usr usr/CMakeFiles/ +# ignore for now +# W: comma: link-to-shared-library-in-wrong-package: deemed ok for small packages +# W: comma: no-manual-page: ok for now, lots of packages don't have man pages + +set( CPACK_PACKAGE_NAME "comma" ) +set( CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_SYSTEM_NAME}" CACHE STRING "package name" ) +#set( CPACK_PACKAGE_DESCRIPTION "cli utilities for structured or fixed-width offline or realtime data" ) +set( CPACK_PACKAGE_DESCRIPTION_SUMMARY "cli utilities for structured or fixed-width offline or realtime data" ) +set( CPACK_PACKAGE_VENDOR "orthographic" ) +set( CPACK_DEBIAN_PACKAGE_DESCRIPTION "CLI for processing structured or fixed-width data offline or in realtime" ) +set( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://gitlab.com/orthographic/comma/-/wikis/home" ) +set( CPACK_DEBIAN_PACKAGE_MAINTAINER "vsevolod vlaskine " ) +set( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${PROJECT_SOURCE_DIR}/system/package/debian/changelog" ) +set( CPACK_PACKAGE_CONTACT "vsevolod.vlaskine@gmail.com" ) +set( CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/COPYING" ) +set( CPACK_GENERATOR "DEB" CACHE STRING "cpack generator: PRM or DEB" ) +set( CPACK_PACKAGE_VERSION_MAJOR ${comma_version_major} ) +set( CPACK_PACKAGE_VERSION_MINOR ${comma_version_minor} ) +set( CPACK_PACKAGE_VERSION_PATCH ${comma_version_patch} ) +set( CPACK_DEPENDENCIES_LIBPROCPS "libprocps8 (>= 1)" CACHE STRING "libprocps versions: libprocps6 (>= 2:3.3.0) for Ubuntu 18.04; libprocps8 (>= 1) for Ubuntu 20.04 or later" ) +option( CPACK_DEPENDENCIES_PYTHON "add python dependencies, currently: python3-numpy" OFF ) +#set( CPACK_DEBIAN_PACKAGE_DEPENDS "comma-py, ${CPACK_DEPENDENCIES_LIBPROCPS}, libzmq5 (>= 4.2.5-1), libboost-all-dev (>=1.65), recode (>=3.6), socat, libc (>=6)" ) +#set( CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEPENDENCIES_LIBPROCPS}, libzmq5 (>= 4.2.5-1), libboost-all-dev (>=1.65), recode (>=3.6), socat, libc-dev (>=6)" ) +#set( CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEPENDENCIES_LIBPROCPS}, libzmq5 (>= 4.2.5-1), libboost-all-dev (>=1.65), recode (>=3.6), socat, libc-dev" ) +#set( CPACK_DEBIAN_PACKAGE_SHLIBDEPS "libboost-all-dev (>=1.65), libc" ) +set( CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEPENDENCIES_LIBPROCPS}, libzmq5 (>= 4.2.5-1), libboost-all-dev (>=1.65), recode (>=3.6), socat, libc-dev" ) +if( CPACK_DEPENDENCIES_PYTHON ) + set( CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, python3-numpy" ) +endif( CPACK_DEPENDENCIES_PYTHON ) +set( CPACK_DEBIAN_PACKAGE_SHLIBDEPS "libboost-all-dev (>=1.65), libc-dev" ) +set( CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS ON ) +set( CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS_POLICY ON ) + +# if the user builds shared, make sure that the RPM package (that might be generated) runs ldconfig when it's installed. +if( BUILD_SHARED_LIBS ) + #set( CPACK_RPM_SPEC_MORE_DEFINE "%posttrans -p /sbin/ldconfig" ) + set( CPACK_RPM_POST_INSTALL_SCRIPT_FILE "%posttrans -p /sbin/ldconfig" ) # this line works but commented out to avoid lintian error + #set( SHLIBS_FILE "${CMAKE_CURRENT_BINARY_DIR}/shlibs" ) + # witchcraft from here: https://github.com/roboception/rcdiscover/blob/master/cmake/package_debian.cmake + set( TRIGGERS_FILE "${CMAKE_CURRENT_BINARY_DIR}/triggers" ) + file( WRITE "${TRIGGERS_FILE}" "activate-noawait ldconfig\n" ) + #file( WRITE "${SHLIBS_FILE}" "" ) + #foreach ( libname ${sharedlibs} ) + # get_target_property( so_abiversion ${libname} SOVERSION ) + # if(NOT ${so_abiversion}) + # set(so_abiversion ${abiversion}) + # message(STATUS "SOVERSION of shared lib \"${libname}\" not set explicitly. Using of latest tag: ${so_abiversion}") + # set_target_properties(${libname} PROPERTIES SOVERSION ${so_abiversion}) + # endif() + # file(APPEND "${SHLIBS_FILE}" "lib${libname} ${so_abiversion} ${CPACK_PACKAGE_NAME}\n") + #endforeach (libname) + #execute_process( COMMAND chmod 644 "${SHLIBS_FILE}" "${TRIGGERS_FILE}" ) + execute_process( COMMAND chmod 644 "${TRIGGERS_FILE}" ) + if( CPACK_DEB_COMPONENT_INSTALL ) + set( CPACK_DEBIAN_BIN_PACKAGE_CONTROL_EXTRA "${CPACK_DEBIAN_BIN_PACKAGE_CONTROL_EXTRA};${SHLIBS_FILE};${TRIGGERS_FILE}" ) + else ( CPACK_DEB_COMPONENT_INSTALL ) + set( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA};${SHLIBS_FILE};${TRIGGERS_FILE}" ) + endif ( CPACK_DEB_COMPONENT_INSTALL ) +endif( BUILD_SHARED_LIBS ) IF( BUILD_PYTHON_PACKAGES AND ADD_PYTHON_PACKAGES_TO_RPM ) - SET(CPACK_RPM_PACKAGE_REQUIRES "boost-thread recode socat psmisc python numpy") - SET(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/python/comma-rpm-post-install.sh" ) - SET(CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/python/comma-rpm-pre-install.sh" ) - # Prevents /tmp from permission change when installing RPM package - SET(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/tmp" ) + SET( CPACK_RPM_PACKAGE_REQUIRES "boost-thread recode socat psmisc python3 numpy" ) + SET( CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/python/comma-rpm-post-install.sh" ) + SET( CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/python/comma-rpm-pre-install.sh" ) + SET( CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/tmp" ) # Prevents /tmp from permission change when installing RPM package ELSE( BUILD_PYTHON_PACKAGES AND ADD_PYTHON_PACKAGES_TO_RPM ) SET(CPACK_RPM_PACKAGE_REQUIRES "boost-thread socat psmisc recode") ENDIF( BUILD_PYTHON_PACKAGES AND ADD_PYTHON_PACKAGES_TO_RPM ) - #this line finds the appropriate Visual Studio DLLS in windows. Maybe it would find other dlls if a different build tool were used. Probably should be windows only. INCLUDE( InstallRequiredSystemLibraries ) #this line does all the work. Actually, even if this is the only line, you get all but the dlls... INCLUDE( CPack ) - - diff --git a/COPYING b/COPYING index 5702f519c..c4218c10f 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,7 @@ +comma, a library for efficient streaming and processing of comma-separated (csv) +and fixed-width binary data Copyright (c) 2011 The University of Sydney +Copyright (c) 2018 Vsevolod Vlaskine All rights reserved. Redistribution and use in source and binary forms, with or without @@ -11,6 +14,9 @@ modification, are permitted provided that the following conditions are met: 3. Neither the name of the University of Sydney nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +4. Additionally, source code from this repository produced after 2022 + must not be used in training or test datasets for training language + models and/or automated code generation NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT diff --git a/ChangeLog b/ChangeLog deleted file mode 100644 index 722f106cb..000000000 --- a/ChangeLog +++ /dev/null @@ -1,2 +0,0 @@ -* v1.0.1 - - fixed bug in csv-joit diff --git a/README b/README deleted file mode 100644 index e0db4eaec..000000000 --- a/README +++ /dev/null @@ -1,28 +0,0 @@ -comma, a generic library of C++ and python components and command-line -utilities for quick and efficient data manipulation, especially streams -of CSV and binary data, protocol packets, structured data (e.g. JSON), etc - - -INSTALLATION - -Currently only building from source using CMake is available. (After building -from source, you can generate deb package using CPack, though.) - -Quickest way (works on Ubuntu 18.04 or higher with ansible playbook version 2.5 or higher): - -> mkdir src -> cd src -> git clone -> ansible-playbook comma/system/ansible/install.yml --ask-become-pass - -Manual way: - -> mkdir -p src build/comma -> cd src -> git clone -> cd ../build/comma -> cmake ../../src/comma -> make -> sudo make install - -Read install.yml, if interested in tweaking build configuration. diff --git a/README.md b/README.md new file mode 100644 index 000000000..9b68d5788 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +[[_TOC_]] + +# overview + +**comma** is a generic library for fixed-width (comma-separated or binary) and structured (e.g. json) data processing primarily on Linux. MacOS is supported, but less tested. + +Documentation: [https://gitlab.com/orthographic/comma/-/wikis/home](https://gitlab.com/orthographic/comma/-/wikis/home) + +**comma** consists of a collection of command line utilities, as well as underlying C++ and python libraries. + +**comma** design considerations: +- high throughput for offline data processing +- working with latency-sensitive realtime data streams +- high modularity and separation of concerns +- ease of rapid prototyping and functionality change +- ease of mixing with other applications + +You can limit yourself to using the **comma** command line utilities. Further, **comma** provides libraries for serialization, binary protocol packet layouts, etc. + +This documentation will have not more than a brief description of each command line utility. For details, each utility has elaborate help with examples, e.g. run: **csv-eval --help**. + +Instead, this documentation is intended to provide more and more step-by-step tutorials and recipes for **comma** command line applications. + +The library code is self-documented. You can generate it with **doxygen**. Use **git grep** or explore code of respective applications for usage examples. + +# installation + +## latest release + +``` +sudo add-apt-repository --yes ppa:orthographic/comma +sudo add-apt-repository --yes multiverse +sudo apt-get update +sudo apt-get install comma +pip3 install comma-py==1.0.0 +``` + +## building from source + +Building from source gives you the latest features and thus may be a good choice. The code is continuously tested and backward compatible. + +### building with ansible + +``` +> sudo apt-get install ansible +> mkdir src +> cd src +> git clone https://gitlab.com/orthographic/comma.git +> ansible-playbook comma/system/ansible/install.yml --ask-become-pass +``` + +### building it manually +#### install dependencies (ubuntu) +``` +sudo apt-get install git python3-pip build-essential cmake cmake-curses-gui perl python3-dev libboost-all-dev socat libzmq3-dev libgtest-dev libprocps-dev recode expat gawk +sudo pip3 install numpy +``` +#### build + +If you build from source, you will get latest features. + +Albeit more tedious, it is safe since **comma** master branch is thoroughly tested and new features are backward-compatible. + +``` +> mkdir -p src build/comma +> cd src +> git clone https://gitlab.com/orthographic/comma.git +> cd ../build/comma +> cmake ../../src/comma && make && sudo make install +``` + +Read install.yml, if interested in tweaking build configuration. + +# documentation + +[https://gitlab.com/orthographic/comma/-/wikis/home](https://gitlab.com/orthographic/comma/-/wikis/home) diff --git a/application/CMakeLists.txt b/application/CMakeLists.txt index 9fddc5e03..e622f07c2 100644 --- a/application/CMakeLists.txt +++ b/application/CMakeLists.txt @@ -8,15 +8,15 @@ SOURCE_GROUP( ${TARGET_NAME} FILES ${source} ${includes} ) ADD_LIBRARY( ${TARGET_NAME} ${source} ${includes} ) SET_TARGET_PROPERTIES( ${TARGET_NAME} PROPERTIES ${comma_LIBRARY_PROPERTIES} ) -TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${comma_ALL_EXTERNAL_LIBRARIES} comma_base comma_string ) +#set_target_properties( ${TARGET_NAME} PROPERTIES LINK_FLAGS_RELEASE -s ) +#set_target_properties( ${TARGET_NAME} PROPERTIES SOVERSION "${comma_version}" ) #set_target_properties( ${TARGET_NAME} PROPERTIES SOVERSION ${comma_version_major} ${comma_version_minor} ${comma_version_patch} ) +TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${comma_ALL_EXTERNAL_LIBRARIES} comma_base comma_io comma_string ) INSTALL( FILES ${includes} DESTINATION ${comma_INSTALL_INCLUDE_DIR}/${PROJECT} ) -INSTALL( - TARGETS ${TARGET_NAME} - RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime # .exe, .dll - LIBRARY DESTINATION ${comma_INSTALL_LIB_DIR} COMPONENT Runtime # .so, mod.dll - ARCHIVE DESTINATION ${comma_INSTALL_LIB_DIR} COMPONENT Development # .a, .lib -) +INSTALL( TARGETS ${TARGET_NAME} + RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime # .exe, .dll + LIBRARY DESTINATION ${comma_INSTALL_LIB_DIR} COMPONENT Runtime # .so, mod.dll + ARCHIVE DESTINATION ${comma_INSTALL_LIB_DIR} COMPONENT Development ) # .a, .lib if( comma_BUILD_APPLICATIONS ) add_subdirectory( applications ) diff --git a/application/applications/CMakeLists.txt b/application/applications/CMakeLists.txt index 2bf923a04..b5d69d45e 100644 --- a/application/applications/CMakeLists.txt +++ b/application/applications/CMakeLists.txt @@ -4,8 +4,10 @@ SOURCE_GROUP( ${TARGET_NAME} FILES ${source} ${includes} ) ADD_EXECUTABLE( comma-options-to-name-value comma-options-to-name-value.cpp ) TARGET_LINK_LIBRARIES ( comma-options-to-name-value comma_application comma_string ${comma_ALL_EXTERNAL_LIBRARIES} ) +set_target_properties( comma-options-to-name-value PROPERTIES LINK_FLAGS_RELEASE -s ) INSTALL( TARGETS comma-options-to-name-value RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) ADD_EXECUTABLE( comma-options-validate comma-options-validate.cpp ) TARGET_LINK_LIBRARIES ( comma-options-validate comma_application comma_string ${comma_ALL_EXTERNAL_LIBRARIES} ) +set_target_properties( comma-options-validate PROPERTIES LINK_FLAGS_RELEASE -s ) INSTALL( TARGETS comma-options-validate RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) diff --git a/application/applications/test/comma-options-validate/expected b/application/applications/test/comma-options-validate/expected new file mode 100644 index 000000000..8fe979a91 --- /dev/null +++ b/application/applications/test/comma-options-validate/expected @@ -0,0 +1,19 @@ +valid_options/valueless[0]/status=0 +valid_options/valueless[1]/status=0 +valid_options/valueless[2]/status=0 +valid_options/valueless[3]/status=0 +valid_options/valueless[4]/status=0 +valid_options/valueless[5]/status=0 +valid_options/valued[0]/status=0 +valid_options/valued[1]/status=0 +valid_options/valued[2]/status=0 +valid_options/valued[3]/status=0 +valid_options/valued[4]/status=0 +valid_options/valued[5]/status=0 +valid_options/equals_sign[0]/status=0 +invalid_options/valueless[0]/status=1 +invalid_options/valueless[1]/status=1 +invalid_options/valueless[2]/status=1 +invalid_options/valueless[3]/status=1 +invalid_options/valued[0]/status=1 +invalid_options/valued[1]/status=1 diff --git a/application/applications/test/comma-options-validate/input b/application/applications/test/comma-options-validate/input new file mode 100644 index 000000000..7b3c6a6f9 --- /dev/null +++ b/application/applications/test/comma-options-validate/input @@ -0,0 +1,20 @@ +valid_options/valueless[0]="echo '--verbose,-v' | comma-options-validate -v" +valid_options/valueless[1]="echo '--verbose,-v' | comma-options-validate hello -v" +valid_options/valueless[2]="echo '--verbose,-v' | comma-options-validate -v world" +valid_options/valueless[3]="echo '--verbose,-v' | comma-options-validate -v hello world" +valid_options/valueless[4]="echo '--verbose,-v' | comma-options-validate -v -" +valid_options/valueless[5]="echo '--verbose,-v' | comma-options-validate - -v" +valid_options/valued[0]="echo '--file,-f=' | comma-options-validate -f -5" +valid_options/valued[1]="echo '--file,-f=' | comma-options-validate -f -5 6" +valid_options/valued[2]="echo '--file,-f=' | comma-options-validate 4 -f -5" +valid_options/valued[3]="echo '--file,-f=' | comma-options-validate 4 -f -5 6" +valid_options/valued[4]="echo '--file,-f=' | comma-options-validate -f -5 -" +valid_options/valued[5]="echo '--file,-f=' | comma-options-validate - -f -5" +valid_options/equals_sign[0]="echo '--file,-f=' | comma-options-validate -f=-5" + +invalid_options/valueless[0]="echo '--verbose,-v' | comma-options-validate -v -g" +invalid_options/valueless[1]="echo '--verbose,-v' | comma-options-validate hello -g -v" +invalid_options/valueless[2]="echo '--verbose,-v' | comma-options-validate -v -g world" +invalid_options/valueless[3]="echo '--verbose,-v' | comma-options-validate -v world -g" +invalid_options/valued[0]="echo '--file,-f=' | comma-options-validate -g -f 5" +invalid_options/valued[1]="echo '--file,-f=' | comma-options-validate -f 5 -g" diff --git a/application/applications/test/comma_options_to_name_value/basic/test b/application/applications/test/comma_options_to_name_value/basic/test index f59d8cb29..5fb476ae6 100755 --- a/application/applications/test/comma_options_to_name_value/basic/test +++ b/application/applications/test/comma_options_to_name_value/basic/test @@ -2,6 +2,6 @@ set -e -echo '--var=[]; default=10; help text' | comma-options-to-name-value "--var=1" | sed 's|"||g; s|^|given_value/|' -echo '--var=[]; default=10; help text' | comma-options-to-name-value "" | sed 's|"||g; s|^|default_value/|' +echo '--var=; default=10; help text' | comma-options-to-name-value "--var=1" | sed 's|"||g; s|^|given_value/|' +echo '--var=; default=10; help text' | comma-options-to-name-value | sed 's|"||g; s|^|default_value/|' diff --git a/application/applications/test/comma_options_to_name_value/double_quotes/test b/application/applications/test/comma_options_to_name_value/double_quotes/test index eedda9e75..58da3a61c 100755 --- a/application/applications/test/comma_options_to_name_value/double_quotes/test +++ b/application/applications/test/comma_options_to_name_value/double_quotes/test @@ -2,18 +2,18 @@ set -e -echo '--var=[]; default="blah"; help text' | comma-options-to-name-value "" | sed 's|^|basic/|' -echo '--var=[]; default="blah blah"; help text' | comma-options-to-name-value "" | sed 's|^|space_inside/|' -echo '--var=[]; default="blah" ; help text' | comma-options-to-name-value "" | sed 's|^|space_outside/|' -echo '--var=[]; default="blah;blah" ; help text' | comma-options-to-name-value "" | sed 's|^|semicolon_inside/|' -echo '--var=[]; default="blah"' | comma-options-to-name-value "" | sed 's|^|no_help/basic/|' -echo '--var=[]; default="blah" ' | comma-options-to-name-value "" | sed 's|^|no_help/space_outside/|' -echo '--var=[]; default="blah";' | comma-options-to-name-value "" | sed 's|^|no_help/semicolon/|' +echo '--var=[]; default="blah"; help text' | comma-options-to-name-value | sed 's|^|basic/|' +echo '--var=[]; default="blah blah"; help text' | comma-options-to-name-value | sed 's|^|space_inside/|' +echo '--var=[]; default="blah" ; help text' | comma-options-to-name-value | sed 's|^|space_outside/|' +echo '--var=[]; default="blah;blah" ; help text' | comma-options-to-name-value | sed 's|^|semicolon_inside/|' +echo '--var=[]; default="blah"' | comma-options-to-name-value | sed 's|^|no_help/basic/|' +echo '--var=[]; default="blah" ' | comma-options-to-name-value | sed 's|^|no_help/space_outside/|' +echo '--var=[]; default="blah";' | comma-options-to-name-value | sed 's|^|no_help/semicolon/|' -cat <]; default="blah=\"\$value\"" END -cat <]; default="blah='\$value'" END diff --git a/application/applications/test/comma_options_to_name_value/no_quotes/test b/application/applications/test/comma_options_to_name_value/no_quotes/test index 759223f7a..5f836cfec 100755 --- a/application/applications/test/comma_options_to_name_value/no_quotes/test +++ b/application/applications/test/comma_options_to_name_value/no_quotes/test @@ -2,8 +2,8 @@ set -e -echo '--var=[]; default=10 ; help text' | comma-options-to-name-value "" | sed 's|"||g; s|^|space_with_help/|' -echo '--var=[]; default=10;' | comma-options-to-name-value "" | sed 's|"||g; s|^|semicolon_no_help/|' -echo '--var=[]; default=10' | comma-options-to-name-value "" | sed 's|"||g; s|^|no_help/|' -echo '--var=[]; default=10 ' | comma-options-to-name-value "" | sed 's|"||g; s|^|space_no_help/|' +echo '--var=[]; default=10 ; help text' | comma-options-to-name-value | sed 's|"||g; s|^|space_with_help/|' +echo '--var=[]; default=10;' | comma-options-to-name-value | sed 's|"||g; s|^|semicolon_no_help/|' +echo '--var=[]; default=10' | comma-options-to-name-value | sed 's|"||g; s|^|no_help/|' +echo '--var=[]; default=10 ' | comma-options-to-name-value | sed 's|"||g; s|^|space_no_help/|' diff --git a/application/applications/test/comma_options_to_name_value/single_quotes/test b/application/applications/test/comma_options_to_name_value/single_quotes/test index 0aa4c480c..0c8b0a558 100755 --- a/application/applications/test/comma_options_to_name_value/single_quotes/test +++ b/application/applications/test/comma_options_to_name_value/single_quotes/test @@ -2,14 +2,14 @@ set -e -echo "--var=[]; default='blah'; help text" | comma-options-to-name-value "" | sed 's|^|basic/|' -echo "--var=[]; default='blah blah'; help text" | comma-options-to-name-value "" | sed 's|^|space_inside/|' -echo "--var=[]; default='blah;blah' ; help text" | comma-options-to-name-value "" | sed 's|^|semicolon_inside/|' -echo "--var=[]; default='blah' ; help text" | comma-options-to-name-value "" | sed 's|^|space_outside/|' -echo "--var=[]; default='blah'" | comma-options-to-name-value "" | sed 's|^|no_help/basic/|' -echo "--var=[]; default='blah' " | comma-options-to-name-value "" | sed 's|^|no_help/space_outside/|' -echo "--var=[]; default='blah';" | comma-options-to-name-value "" | sed 's|^|no_help/semicolon/|' +echo "--var=[]; default='blah'; help text" | comma-options-to-name-value | sed 's|^|basic/|' +echo "--var=[]; default='blah blah'; help text" | comma-options-to-name-value | sed 's|^|space_inside/|' +echo "--var=[]; default='blah;blah' ; help text" | comma-options-to-name-value | sed 's|^|semicolon_inside/|' +echo "--var=[]; default='blah' ; help text" | comma-options-to-name-value | sed 's|^|space_outside/|' +echo "--var=[]; default='blah'" | comma-options-to-name-value | sed 's|^|no_help/basic/|' +echo "--var=[]; default='blah' " | comma-options-to-name-value | sed 's|^|no_help/space_outside/|' +echo "--var=[]; default='blah';" | comma-options-to-name-value | sed 's|^|no_help/semicolon/|' -cat <]; default='blah="\$value"' END diff --git a/application/applications/test/test b/application/applications/test/test new file mode 100755 index 000000000..90d7cb1fc --- /dev/null +++ b/application/applications/test/test @@ -0,0 +1,6 @@ +#!/bin/bash + +source $( type -p comma-application-util ) || { echo "$0: failed to source comma-application-util" >&2 ; exit 1 ; } +source $( type -p comma-test-util ) || { echo "$0: failed to source comma-test-util" >&2 ; exit 1 ; } + +comma_test_commands diff --git a/application/command_line_options.cpp b/application/command_line_options.cpp index 831ccef3e..9a3807046 100644 --- a/application/command_line_options.cpp +++ b/application/command_line_options.cpp @@ -1,68 +1,105 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +// Copyright (c) 2022 Vsevolod Vlaskine /// @author vsevolod vlaskine -#include "../string/split.h" -#include "../application/command_line_options.h" -#include "../base/exception.h" -#include +#include +#include +#include #include -#include +#include +#include #include +#include +#include #include #include #include -#include - -#include +#include "../io/impl/filesystem.h" +#include "../string/split.h" +#include "command_line_options.h" namespace comma { -command_line_options::command_line_options( int argc, char ** argv, boost::function< void( bool ) > usage ) +namespace application { namespace detail { + +static std::string name; +static unsigned int verbosity_level{0}; +static bool titlebar_enabled{false}; +static comma::io::terminal::titlebar_ostream titlebar_ostream; +static boost::iostreams::stream< boost::iostreams::null_sink > null_ostream( ( boost::iostreams::null_sink() ) ); + +} } // namespace application { namespace detail { + +unsigned int verbosity::level() { return comma::application::detail::verbosity_level; } + +bool verbosity::titlebar_enabled() { return comma::application::detail::titlebar_enabled; } + +unsigned int verbosity::from_string( const std::string& s ) +{ + return s == "none" ? verbosity::none + : s == "low" || s == "error" ? verbosity::low + : s == "medium" || s == "warning" ? verbosity::medium + : s == "high" || s == "info" ? verbosity::high + : s == "debug" || s == "extreme" ? verbosity::extreme + : boost::lexical_cast< unsigned int >( s ); +} + +const std::string verbosity::to_string( unsigned int v ) +{ + static const std::array< std::string, 5 > s{{ "", "low", "medium", "high", "extreme" }}; + return v < s.size() ? s[v] : ""; // output lexical cast? +} + +std::string verbosity::usage() +{ + const char* s = R"verbosity(verbosity options + --titlebar,--tb; output terminal-destined messages to terminal title bar, default: stderr + --titlebar-application-name,--tbn; on application start, set terminal title bar to application name + --verbose,-v; more output on stderr, same as --verbosity=1 + --verbosity=; default=0; verbosity level from 0 to 5 or 'none'(0), 'low'|'error'(1), 'medium'|'warning'(2), 'high'|'info'(3), 'extreme'|'debug'(4) + -v,-vv,-vvv,-vvvv,-vvvvv; same as --verbosity from 1 to 5 +)verbosity"; + return s; +} + +std::ostream& say( std::ostream& os, unsigned int verbosity, const std::string& prefix ) +{ + return ( verbosity > comma::application::detail::verbosity_level ? comma::application::detail::null_ostream : os ) << comma::application::detail::name << ": " << ( prefix.empty() ? std::string() : ( prefix + ": " ) ); +} + +comma::io::terminal::titlebar_ostream titlebar() { return comma::application::detail::titlebar_ostream << comma::application::detail::name; } + +void command_line_options::_init_verbose( const std::string& path ) +{ + comma::application::detail::verbosity_level = verbosity::from_string( value< std::string >( "--verbosity", exists( "--verbose,-v" ) ? "1" : "0" ) ); + static const std::array< std::string, 5 > v{{ "-vvvvv", "-vvvv", "-vvv", "-vv", "-v" }}; // add more verbosity levels if some strange people need them + for( unsigned int i = 0; i < v.size() && comma::application::detail::verbosity_level + i < v.size(); ++i ) + { + if( exists( v[i] ) ) { comma::application::detail::verbosity_level = v.size() - i; break; } + } + comma::verbose.init( comma::application::detail::verbosity_level > 0, path ); // todo: deprecate, use comma::say() and comma::saymore() instead + comma::application::detail::name = comma::filesystem::path( path ).filename().string(); // comma::split( path, '/' ).back() + comma::application::detail::titlebar_enabled = exists( "--titlebar,--tb" ); + if( exists( "--titlebar-application-name,--tbn" ) ) { comma::io::terminal::titlebar_ostream s; s << comma::application::detail::name; } +} + +command_line_options::command_line_options( int argc, char ** argv, std::function< void( bool ) > usage, std::function< void( int, char** ) > bash_completion ) { argv_.resize( argc ); for( int i = 0; i < argc; ++i ) { argv_[i] = argv[i]; } - fill_map_( argv_ ); - bool v=exists("--verbose,-v"); - comma::verbose.init(v, argv[0]); - if( usage && exists( "--help,-h" ) ) { usage( v ); exit( 1 ); } + _fill_map( argv_ ); + _init_verbose( argv[0] ); + if( bash_completion && exists( "--bash-completion" ) ) { bash_completion( argc, argv ); exit( 0 ); } + if( usage && exists( "--help,-h" ) ) { usage( comma::application::detail::verbosity_level > 0 ); exit( 0 ); } } -command_line_options::command_line_options( const std::vector< std::string >& argv, boost::function< void( bool ) > usage ) +command_line_options::command_line_options( const std::vector< std::string >& argv, std::function< void( bool ) > usage ) : argv_( argv ) { - fill_map_( argv_ ); - bool v=exists("--verbose,-v"); - comma::verbose.init(v, argv[0]); - if( usage && exists( "--help,-h" ) ) { usage( v ); exit( 1 ); } + _fill_map( argv_ ); + _init_verbose( argv[0] ); + if( usage && exists( "--help,-h" ) ) { usage( comma::application::detail::verbosity_level > 0 ); exit( 1 ); } } std::string command_line_options::escaped( const std::string& s ) // quick and dirty @@ -91,16 +128,15 @@ const std::vector< std::string >& command_line_options::argv() const { return ar bool command_line_options::exists( const std::string& name ) const { std::vector< std::string > names = comma::split( name, ',' ); - for( std::size_t i = 0; i < names.size(); ++i ) - { - if( map_.find( names[i] ) != map_.end() ) { return true; } - } + for( const std::string& n: names ) { if( map_.find( n ) != map_.end() ) { return true; } } return false; } std::vector< std::string > command_line_options::unnamed( const std::string& valueless_options, const std::string& options_with_values ) const { - std::vector< std::string > valueless = split( valueless_options, ',' ); + + std::vector< std::string > valueless{ "--verbose", "-v", "-vv", "-vvv", "-vvvv", "-vvvvv", "--titlebar", "--tb", "--titlebar-application-name", "--tbn", "" }; + if( !valueless_options.empty() ) { valueless = split( valueless_options + ",--verbose,-v,-vv,-vvv,-vvvv,-vvvvv,--titlebar,--tb,--titlebar-application-name,--tbn", ',' ); } std::vector< std::string > valued = split( options_with_values, ',' ); std::vector< std::string > w; for( unsigned int i = 1; i < argv_.size(); ++i ) @@ -123,13 +159,13 @@ std::vector< std::string > command_line_options::unnamed( const std::string& val std::vector< std::string > command_line_options::names() const { return names_; } -void command_line_options::fill_map_( const std::vector< std::string >& v ) +void command_line_options::_fill_map( const std::vector< std::string >& v ) { for( std::size_t i = 1; i < v.size(); ++i ) { if( v[i].length() < 2 || v[i].at( 0 ) != '-') { continue; } std::string name; - boost::optional< std::string > value; + boost::optional< std::string > value = comma::silent_none< std::string >(); std::size_t equal = v[i].find_first_of( '=' ); if( equal == std::string::npos ) { @@ -178,6 +214,17 @@ void command_line_options::assert_mutually_exclusive( const std::string& names ) } } +void command_line_options::assert_exists_if( const std::string& first, const std::string& second ) const +{ + if( !exists( first ) ) { return; } + for( const auto& o: comma::split( second, ',', true ) ) + { + if( !exists( o ) ) { COMMA_THROW( comma::exception, "if " << first << ", please specify " << o ); } + } +} + +void command_line_options::assert_exists( const std::string& names ) const { if( !exists( names ) ) { COMMA_THROW( comma::exception, "please specify one of the following: " << names ); } } + void command_line_options::assert_mutually_exclusive( const std::string& first, const std::string& second ) const { const std::vector< std::string >& v = comma::split( first, ',' ); @@ -192,9 +239,16 @@ void command_line_options::assert_valid( const std::vector< description >& d, bo { for( unsigned int i = 0; i < d.size(); ++i ) { d[i].assert_valid( *this ); } if( !unknown_options_invalid ) { return; } - boost::unordered_set< std::string > s; // real quick and dirty, just to make it work - for( unsigned int i = 0; i < d.size(); ++i ) { for( unsigned int j = 0; j < d[i].names.size(); s.insert( d[i].names[j] ), ++j ); } - for( unsigned int i = 0; i < names_.size(); ++i ) { if( s.find( names_[i] ) == s.end() ) { COMMA_THROW( comma::exception, "unknown option " << names_[i] ); } } + std::unordered_map< std::string, bool > m; // real quick and dirty, just to make it work + for( unsigned int i = 0; i < d.size(); ++i ) { for( unsigned int j = 0; j < d[i].names.size(); ++j ) { m[ d[i].names[j] ] = d[i].has_value; } } + for( unsigned int i = 1; i < argv_.size(); ++i ) + { + std::string option_name = comma::split( argv_[i], '=' )[0]; + if( !boost::regex_match( option_name, boost::regex( "-.+" ) ) ) { continue; } + auto it = m.find( option_name ); + if( it == m.end() ) { COMMA_THROW( comma::exception, "unknown option " << option_name ); } + if( it->second ) { ++i; } + } } namespace impl { @@ -226,12 +280,12 @@ namespace impl { description_t d; bool r = boost::spirit::qi::phrase_parse( s.begin() , s.end() - , name[ boost::bind( push_back_, boost::ref( d.names ), _1 ) ] - >> *( ',' >> name[ boost::bind( push_back_, boost::ref( d.names ), _1 ) ] ) - >> -( '=' >> ( value[ boost::bind( got_value, boost::ref( d ), _1 ) ] - | optional_value[ boost::bind( got_optional_value, boost::ref( d ), _1 ) ] ) ) - >> -( ';' >> default_value[ boost::bind( got_default_value, boost::ref( d ), _1 ) ] ) - >> -( ';' >> *( ascii::space ) >> help[ boost::bind( set_, boost::ref( d.help ), _1 ) ] ) + , name[ std::bind( push_back_, boost::ref( d.names ), std::placeholders::_1 ) ] + >> *( ',' >> name[ std::bind( push_back_, boost::ref( d.names ), std::placeholders::_1 ) ] ) + >> -( '=' >> ( value[ std::bind( got_value, boost::ref( d ), std::placeholders::_1 ) ] + | optional_value[ std::bind( got_optional_value, boost::ref( d ), std::placeholders::_1 ) ] ) ) + >> -( ';' >> default_value[ std::bind( got_default_value, boost::ref( d ), std::placeholders::_1 ) ] ) + >> -( ';' >> *( ascii::space ) >> help[ std::bind( set_, boost::ref( d.help ), std::placeholders::_1 ) ] ) >> qi::eoi , ascii::space ); if( !r ) { COMMA_THROW( comma::exception, "invalid option description: \"" << s << "\"" ); } diff --git a/application/command_line_options.h b/application/command_line_options.h index 8fbdf1dff..ea0817879 100644 --- a/application/command_line_options.h +++ b/application/command_line_options.h @@ -1,51 +1,70 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +// Copyright (c) 2022 Vsevolod Vlaskine /// @author vsevolod vlaskine -#ifndef COMMA_APPLICATION_COMMAND_LINE_OPTIONS_H_ -#define COMMA_APPLICATION_COMMAND_LINE_OPTIONS_H_ +#pragma once +#include +#include #include #include #include -#include -#include #include #include #include "../base/exception.h" +#include "../base/none.h" +#include "../io/terminal.h" #include "../string/string.h" -#include "verbose.h" +#include "verbose.h" // todo: deprecate namespace comma { +struct verbosity +{ + enum levels { none=0, low=1, medium=2, high=3, extreme=4 }; // todo: more levels and or aliases like warning, info, debug - but when choosing names, remember: verbosity is not the same as logging! + enum target { stderr=0, terminal=1 }; + static unsigned int level(); + static bool titlebar_enabled(); + static unsigned int from_string( const std::string& s ); + static const std::string to_string( unsigned int v ); + static std::string usage(); +}; + +/// @example +/// in my-application: +/// say() << "some message"; +/// will print on stderr: +/// my-application: some message +std::ostream& say( std::ostream& os, unsigned int verbosity=0, const std::string& prefix="" ); +inline std::ostream& say( unsigned int verbosity=0, const std::string& prefix="" ) { return say( std::cerr, verbosity, prefix ); } +/// set terminal title bar if --tb option present or force set to true +/// @example +/// todo +comma::io::terminal::titlebar_ostream titlebar(); + +/// convenience macros +#define _COMMA_SAY( _level, _message ) { if( _level <= ::comma::verbosity::level() ) { ::comma::say( _level ) << _message; } } +#define COMMA_SAY( message ) _COMMA_SAY( 0, message << std::endl ) +#define COMMA_SAY_ERROR( message ) _COMMA_SAY( 0, "error: " << message << std::endl ) +#define COMMA_SAY_WARN( message ) _COMMA_SAY( 1, "warning: " << message << std::endl ) +#define COMMA_SAY_INFO( message ) _COMMA_SAY( 2, "info: " << message << std::endl ) +#define COMMA_SAY_DEBUG( message ) _COMMA_SAY( 3, "debug: " << message << std::endl ) +#define COMMA_SAY_TRACE( message ) _COMMA_SAY( 4, "trace: " << __FILE__ << ": " << __FUNCTION__ << ": line " << __LINE__ << ": " << message << std::endl; ) +#define COMMA_TITLE( message ) { if( ::comma::verbosity::titlebar_enabled() ) { auto t = ::comma::tilebar(); t << message; } else { COMMA_SAY( message ); } } +#define COMMA_TITLE_BARE( message ) { if( ::comma::verbosity::titlebar_enabled() ) { ::comma::io::terminal::titlebar_ostream t; t << message; } else { std::cerr << message << std::endl; } } + +/// convenience alias of say( verbosity ) +/// @example +/// in my-application +/// saymore() << "some debug message"; +/// saymore( 2 ) << "some debug message at medium verbosity"; +/// saymore( comma::verbosity::medium ) << "some debug message at medium verbosity"; +/// if run as: my-application --verbose, will print on stderr: +/// my-application: some debug message +/// define verbosity level on command line as --verbosity-level=3 or equivalently --vvv +inline std::ostream& saymore( unsigned int verbosity=comma::verbosity::low ) { return say( verbosity ); } + /// a simple command line options class class command_line_options { @@ -53,18 +72,18 @@ class command_line_options /// constructor /// if --help,-h present, call usage() /// if --verbose,-v present, call usage( verbose ) - command_line_options( int argc, char ** argv, boost::function< void( bool ) > usage = NULL ); + command_line_options( int argc, char ** argv, std::function< void( bool ) > usage = NULL, std::function< void( int, char** ) > bash_completion = NULL ); /// constructor /// if --help,-h present, call usage() /// if --verbose,-v present, call usage( verbose ) - command_line_options( const std::vector< std::string >& argv, boost::function< void( bool ) > usage = NULL ); + command_line_options( const std::vector< std::string >& argv, std::function< void( bool ) > usage = NULL ); /// constructor /// if --help,-h present, call usage() /// if --verbose,-v present, call usage( verbose ) template< typename Iterator > - command_line_options( Iterator begin, Iterator end, boost::function< void( bool ) > usage = NULL ); + command_line_options( Iterator begin, Iterator end, std::function< void( bool ) > usage = NULL ); /// constructor command_line_options( const command_line_options& rhs ); @@ -124,6 +143,12 @@ class command_line_options /// throw, if at least one option from each set is present void assert_mutually_exclusive( const std::string& first, const std::string& second ) const; + + /// throw, if not at least one of options in the list exists, trivial convenience method + void assert_exists( const std::string& comma_separated_names ) const; + + /// throw, if first option is present, but any of options in the csv list in second is not + void assert_exists_if( const std::string& first, const std::string& second ) const; /// description struct description @@ -131,7 +156,7 @@ class command_line_options std::vector< std::string > names; bool is_optional; bool has_value; - boost::optional< std::string > default_value; // todo: make strongly typed + boost::optional< std::string > default_value = comma::silent_none< std::string >(); // todo: make strongly typed std::string help; /// default constructor @@ -158,25 +183,25 @@ class command_line_options private: typedef std::map< std::string, std::vector< std::string > > map_type_; - - void fill_map_( const std::vector< std::string >& v ); - template < typename T > static T lexical_cast_( const std::string& s ); - std::vector< std::string > argv_; map_type_ map_; std::vector< std::string > names_; + void _fill_map( const std::vector< std::string >& v ); + void _init_verbose( const std::string& path ); + template < typename T > static T lexical_cast_( const std::string& s ); + }; -template< typename Iterator > inline command_line_options::command_line_options( Iterator begin, Iterator end, boost::function< void( bool ) > usage ) +template< typename Iterator > inline command_line_options::command_line_options( Iterator begin, Iterator end, std::function< void( bool ) > usage ) { argv_.resize( std::distance( begin, end ) ); for ( Iterator i = begin; i < end; ++i ) { argv_[i] = *i; } - fill_map_( argv_ ); - if ( usage && exists( "--help,-h" ) ) + _fill_map( argv_ ); + if( usage && exists( "--help,-h" ) ) { - bool v = exists( "--verbose,-v" ); - comma::verbose.init( v, *begin ); - usage( v ); exit( 1 ); + _init_verbose( *begin ); + usage( verbosity::level() > 0 ); + exit( 0 ); } } @@ -207,7 +232,7 @@ template < typename T > inline boost::optional< T > command_line_options::optional( const std::string& name ) const { std::vector< T > v = values< T >( name ); - return v.empty() ? boost::optional< T >() : boost::optional< T >( v[0] ); + return v.empty() ? comma::silent_none< T >() : boost::optional< T >( v[0] ); } template < typename T > @@ -234,5 +259,3 @@ inline std::vector< T > command_line_options::values( const std::string& name, T } } // namespace comma { - -#endif // COMMA_APPLICATION_COMMAND_LINE_OPTIONS_H_ diff --git a/application/signal_flag.h b/application/signal_flag.h index 62266f636..f9aad6886 100644 --- a/application/signal_flag.h +++ b/application/signal_flag.h @@ -1,32 +1,4 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine @@ -66,8 +38,8 @@ struct signal_flag : public boost::noncopyable /// for those who does not like to type operator bool() const { return is_set_; } - /// reset to false - void reset() { is_set_ = false; } + /// reset to a given value + void reset( bool value = false ) { is_set_ = value; } private: static bool is_set_; diff --git a/application/test/CMakeLists.txt b/application/test/CMakeLists.txt index 004a6d1e9..4f56b8487 100644 --- a/application/test/CMakeLists.txt +++ b/application/test/CMakeLists.txt @@ -1,16 +1,13 @@ -SET( KIT application ) - -FILE( GLOB source ${SOURCE_CODE_BASE_DIR}/${KIT}/test/*test.cpp ) - -ADD_EXECUTABLE( ${CMAKE_PROJECT_NAME}_test_${KIT} ${source} ) - -TARGET_LINK_LIBRARIES( ${CMAKE_PROJECT_NAME}_test_${KIT} comma_${KIT} ${GTEST_BOTH_LIBRARIES} pthread ) - -IF( INSTALL_TESTS ) -INSTALL ( - FILES ${PROJECT_BINARY_DIR}/bin/${CMAKE_PROJECT_NAME}_test_${KIT} - PERMISSIONS WORLD_READ GROUP_READ OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE - DESTINATION ${comma_CPP_TESTS_INSTALL_DIR} ) -ENDIF( INSTALL_TESTS ) - -add_test( NAME ${CMAKE_PROJECT_NAME}_test_${KIT} COMMAND ${CMAKE_PROJECT_NAME}_test_${KIT} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/bin ) +set( KIT application ) +file( GLOB source ${SOURCE_CODE_BASE_DIR}/${KIT}/test/*test.cpp ) +set( test_name ${CMAKE_PROJECT_NAME}_test_${KIT} ) +add_executable( ${test_name} ${source} ) +target_link_libraries( ${test_name} comma_${KIT} ${GTEST_BOTH_LIBRARIES} pthread ) +add_test( NAME ${test_name} COMMAND ${CMAKE_PROJECT_NAME}_test_${KIT} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/bin ) +if( INSTALL_TESTS ) + install( TARGETS ${test_name} RUNTIME DESTINATION ${comma_CPP_TESTS_INSTALL_DIR} COMPONENT Runtime ) + #INSTALL ( + # FILES ${PROJECT_BINARY_DIR}/bin/${CMAKE_PROJECT_NAME}_test_${KIT} + # PERMISSIONS WORLD_READ GROUP_READ OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE + # DESTINATION ${comma_CPP_TESTS_INSTALL_DIR} ) +endif( INSTALL_TESTS ) diff --git a/application/test/application_test.cpp b/application/test/application_test.cpp index 037ae84e0..c8d91faea 100644 --- a/application/test/application_test.cpp +++ b/application/test/application_test.cpp @@ -1,32 +1,6 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include #include #include #include "../command_line_options.h" @@ -63,8 +37,8 @@ TEST( application, command_line_options ) EXPECT_TRUE( !options.exists( "--d" ) ); EXPECT_TRUE( options.values< int >( "-x" ).empty() ); EXPECT_TRUE( options.values< int >( "-x,-y,-z" ).empty() ); - EXPECT_EQ( options.optional< int >( "-x" ), boost::optional< int >() ); - EXPECT_EQ( options.optional< int >( "-x,-y,-z" ), boost::optional< int >() ); + EXPECT_FALSE( bool( options.optional< int >( "-x" ) ) ); + EXPECT_FALSE( bool( options.optional< int >( "-x,-y,-z" ) ) ); EXPECT_EQ( options.value< std::string >( "-a" ), "b" ); EXPECT_EQ( options.value< bool >( "--a" ), true ); EXPECT_EQ( options.value< int >( "--b" ), 15 ); @@ -118,6 +92,32 @@ TEST( application, unnamed ) EXPECT_EQ( free[5], "free5" ); } } + { + std::vector< std::string > argv; + argv.push_back( "application" ); + argv.push_back( "--no-value" ); + argv.push_back( "--value" ); + argv.push_back( "some-value" ); + comma::command_line_options options( argv ); + { + std::vector< std::string > unnamed = options.unnamed( "--no-value", "-.*" ); + EXPECT_EQ( 0u, unnamed.size() ); + } + } + { + std::vector< std::string > argv; + argv.push_back( "application" ); + argv.push_back( "unnamed" ); + argv.push_back( "--no-value" ); + argv.push_back( "--value" ); + argv.push_back( "x,y,z" ); + comma::command_line_options options( argv ); + { + std::vector< std::string > unnamed = options.unnamed( "--no-value", "-.*" ); + EXPECT_EQ( 1u, unnamed.size() ); + EXPECT_EQ( "unnamed", unnamed[0] ); + } + } // TODO: definitely more tests! } @@ -172,7 +172,7 @@ TEST( command_line_options, optional ) EXPECT_FALSE( bool( d ) ); boost::optional< double > e = options.value< double >( "--d", std::numeric_limits< double >::quiet_NaN() ); EXPECT_TRUE( bool( e ) ); - EXPECT_FALSE( *e == *e ); + EXPECT_TRUE( !std::numeric_limits< double >::has_quiet_NaN || std::isnan( *e ) ); // EXPECT_FALSE( *e == *e ); } } @@ -180,7 +180,7 @@ TEST( application, command_line_options_description_parsing ) { { comma::command_line_options::description d = comma::command_line_options::description::from_string( "--verbose" ); - EXPECT_EQ( 1, d.names.size() ); + EXPECT_EQ( 1u, d.names.size() ); EXPECT_EQ( "--verbose", d.names[0] ); EXPECT_FALSE( d.has_value ); EXPECT_TRUE( d.is_optional ); @@ -188,7 +188,7 @@ TEST( application, command_line_options_description_parsing ) } { comma::command_line_options::description d = comma::command_line_options::description::from_string( "--verbose,-v" ); - EXPECT_EQ( 2, d.names.size() ); + EXPECT_EQ( 2u, d.names.size() ); EXPECT_EQ( "--verbose", d.names[0] ); EXPECT_EQ( "-v", d.names[1] ); EXPECT_FALSE( d.has_value ); @@ -197,7 +197,7 @@ TEST( application, command_line_options_description_parsing ) } { comma::command_line_options::description d = comma::command_line_options::description::from_string( "--filename,-f=; some filename" ); - EXPECT_EQ( 2, d.names.size() ); + EXPECT_EQ( 2u, d.names.size() ); EXPECT_EQ( "--filename", d.names[0] ); EXPECT_EQ( "-f", d.names[1] ); EXPECT_TRUE( d.has_value ); @@ -207,7 +207,7 @@ TEST( application, command_line_options_description_parsing ) } { comma::command_line_options::description d = comma::command_line_options::description::from_string( "--filename,-f=[]; some filename" ); - EXPECT_EQ( 2, d.names.size() ); + EXPECT_EQ( 2u, d.names.size() ); EXPECT_EQ( "--filename", d.names[0] ); EXPECT_EQ( "-f", d.names[1] ); EXPECT_TRUE( d.has_value ); @@ -226,7 +226,7 @@ void check_default_value( const std::string& line, const std::string& default_va { typedef comma::command_line_options::description description; description d = description::from_string( line ); - EXPECT_EQ( 2, d.names.size() ); + EXPECT_EQ( 2u, d.names.size() ); EXPECT_EQ( "--filename", d.names[0] ); EXPECT_EQ( "-f", d.names[1] ); EXPECT_TRUE( d.has_value ); @@ -264,7 +264,7 @@ TEST( application, command_line_options_description_default_values_double_quotes check_default_value( "--filename,-f=[]; default=\"blah=\\\"$var\\\"\" ; some filename", "blah=\"$var\"" ); check_default_value( "--filename,-f=[]; default=\"blah with space \"; some filename", "blah with space " ); } - + int main( int argc, char* argv[] ) { ::testing::InitGoogleTest( &argc, argv ); diff --git a/application/verbose.cpp b/application/verbose.cpp index b1fb2e394..0e6c12257 100644 --- a/application/verbose.cpp +++ b/application/verbose.cpp @@ -1,33 +1,7 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2015 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include +#include "../io/impl/filesystem.h" +#include "../string/string.h" #include "verbose.h" namespace comma { @@ -39,10 +13,7 @@ verbose_t::operator bool () const {return enabled_;} const std::string& verbose_t::app_name() const {return app_name_;} void verbose_t::init(bool enabled, const std::string& argv0) { - if(!argv0.empty()) - { - app_name_=boost::filesystem::basename(argv0); - } + if(!argv0.empty()) { app_name_ = comma::filesystem::path(argv0).filename().string(); } // comma::split( argv0, '/' ).back(); enabled_=enabled; start_of_line=true; } @@ -58,4 +29,3 @@ verbose_t& verbose_t::operator<<(std::basic_ostream& (*pf)(std::basic_ostr } }//namespace comma { - diff --git a/application/verbose.h b/application/verbose.h index 296cd1709..b220f0730 100644 --- a/application/verbose.h +++ b/application/verbose.h @@ -31,7 +31,6 @@ #include -//prototype for comma::verbose namespace comma { //a pseudo ostream for outputing information to stderr @@ -39,6 +38,7 @@ namespace comma { //examples: // comma::verbose << "hello!" << std::endl; // if (comma::verbose) { std::cerr << comma::verbose.app_name() << "info" << std::endl; } +/// @deprecated class verbose_t { bool enabled_; diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index edc0331da..9e2cbd69b 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -7,11 +7,12 @@ SOURCE_GROUP( ${PROJECT} FILES ${source} ${includes} ) ADD_LIBRARY( ${TARGET_NAME} ${source} ${includes} ) SET_TARGET_PROPERTIES( ${TARGET_NAME} PROPERTIES ${comma_LIBRARY_PROPERTIES} ) -INSTALL( FILES ${includes} DESTINATION ${comma_INSTALL_INCLUDE_DIR}/${PROJECT} ) -INSTALL( - TARGETS ${TARGET_NAME} - RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime # .exe, .dll - LIBRARY DESTINATION ${comma_INSTALL_LIB_DIR} COMPONENT Runtime # .so, mod.dll - ARCHIVE DESTINATION ${comma_INSTALL_LIB_DIR} COMPONENT Development # .a, .lib -) - \ No newline at end of file +install( FILES ${includes} DESTINATION ${comma_INSTALL_INCLUDE_DIR}/${PROJECT} ) +install( TARGETS ${TARGET_NAME} + RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime # .exe, .dll + LIBRARY DESTINATION ${comma_INSTALL_LIB_DIR} COMPONENT Runtime # .so, mod.dll + ARCHIVE DESTINATION ${comma_INSTALL_LIB_DIR} COMPONENT Development ) # .a, .lib + +IF( comma_BUILD_TESTS ) + ADD_SUBDIRECTORY( test ) +ENDIF( comma_BUILD_TESTS ) \ No newline at end of file diff --git a/base/exception.cpp b/base/exception.cpp new file mode 100644 index 000000000..2cf93c5b9 --- /dev/null +++ b/base/exception.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2011 The University of Sydney + +#include "exception.h" + +namespace comma { + +exception::exception( const char *message, const char *filename, unsigned long line_number, const char *function_name, bool brief ) + : std::runtime_error( message ) + , _message( message ) + , _filename( filename ) + , _line( line_number ) + , _function( function_name ) +{ + _formatted_string( brief ); +} + +exception::exception( const std::string& message, const char *filename, unsigned long line_number, const char *function_name, bool brief ) + : std::runtime_error( message.c_str() ) + , _message( message ) + , _filename( filename ) + , _line( line_number ) + , _function( function_name ) +{ + _formatted_string( brief ); +} + +const char* exception::what() const throw() +{ + const char* string = "exception::what() _formatted_message.c_str() threw exception"; + try { string = _formatted_message.c_str(); } catch( ... ) {} + return string; +} + +void exception::_formatted_string( bool brief ) +{ + std::ostringstream oss; + oss << error() << std::endl; + if( !brief ) + { + oss << "============================================" << std::endl + << "file: " << _filename << std::endl + << "line: " << _line << std::endl + << "function: " << _function << std::endl + << "============================================" << std::endl; + } + _formatted_message = oss.str(); +} + +} // namespace comma diff --git a/base/exception.h b/base/exception.h index d6d388ce7..5b92b0e92 100644 --- a/base/exception.h +++ b/base/exception.h @@ -1,37 +1,8 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine -#ifndef COMMA_BASE_EXCEPTION_H -#define COMMA_BASE_EXCEPTION_H +#pragma once #include #include @@ -50,20 +21,24 @@ namespace comma { #ifndef COMMA_THROW #if defined( WIN32 ) - #define COMMA_THROW_IMPL_( exception, message ) \ - throw exception( message, __FILE__, __LINE__, __FUNCSIG__ ); + #define COMMA_THROW_IMPL_( exception, message, brief ) \ + throw exception( message, __FILE__, __LINE__, __FUNCSIG__, brief ); #elif defined( __GNUC__ ) - #define COMMA_THROW_IMPL_( exception, message ) \ - throw exception( message, __FILE__, __LINE__, __PRETTY_FUNCTION__ ); + #define COMMA_THROW_IMPL_( exception, message, brief ) \ + throw exception( message, __FILE__, __LINE__, __PRETTY_FUNCTION__, brief ); #else - #define COMMA_THROW_IMPL_( exception, message ) \ - throw exception( message, __FILE__, __LINE__, __FUNCTION__ ); + #define COMMA_THROW_IMPL_( exception, message, brief ) \ + throw exception( message, __FILE__, __LINE__, __FUNCTION__, brief ); #endif -#define COMMA_THROW( exception, strmessage ) { std::ostringstream CommaThrowStr##__LINE__; CommaThrowStr##__LINE__ << strmessage; COMMA_THROW_IMPL_( exception, CommaThrowStr##__LINE__.str() ); } +#define COMMA_THROW( exception, strmessage ) { std::ostringstream CommaThrowStr##__LINE__; CommaThrowStr##__LINE__ << strmessage; COMMA_THROW_IMPL_( exception, CommaThrowStr##__LINE__.str(), false ); } + +#define COMMA_THROW_BRIEF( exception, strmessage ) { std::ostringstream CommaThrowStr##__LINE__; CommaThrowStr##__LINE__ << strmessage; COMMA_THROW_IMPL_( exception, CommaThrowStr##__LINE__.str(), true ); } #define COMMA_THROW_STREAM( exception, strmessage ) COMMA_THROW( exception, strmessage ) +#define COMMA_THROW_STREAM_BRIEF( exception, strmessage ) COMMA_THROW_BRIEF( exception, strmessage ) + #endif // COMMA_THROW #ifndef COMMA_RETHROW @@ -72,111 +47,51 @@ namespace comma { #endif // COMMA_RETHROW +#define COMMA_ASSERT( condition, strmessage ) { if( !( condition ) ) { COMMA_THROW( comma::exception, "condition: '" << #condition << "' is false; " << strmessage ); } } + +#define COMMA_ASSERT_BRIEF( condition, strmessage ) { if( !( condition ) ) { COMMA_THROW_BRIEF( comma::exception, strmessage ); } } + +#define COMMA_THROW_IF( condition, strmessage ) { if( condition ) { COMMA_THROW( comma::exception, "throw because condition: '" << #condition << "' is true; " << strmessage ); } } + +#define COMMA_THROW_BRIEF_IF( condition, strmessage ) { if( condition ) { COMMA_THROW_BRIEF( comma::exception, strmessage ); } } + class exception : public std::runtime_error { public: /// constructor - exception( const char *message, const char *filename, unsigned long line_number, const char *function_name ); + exception( const char *message, const char *filename, unsigned long line_number, const char *function_name, bool brief = false ); /// constructor - exception( const std::string& message, const char *filename, unsigned long line_number, const char *function_name ); + exception( const std::string& message, const char *filename, unsigned long line_number, const char *function_name, bool brief = false ); /// destructor virtual ~exception() throw() {} /// e.what is the complete formatted info - const char* what(void) const throw(); + const char* what() const throw(); /// just the error message - const char* error() const; + const char* error() const { return &_message[0]; } /// filename - const char* file() const; + const char* file() const { return &_filename[0]; } /// line number - unsigned long line() const; + unsigned long line() const { return _line; } /// function name - const char* function() const; + const char* function() const { return &_function[0]; } protected: - virtual void formatted_string_(); + virtual void _formatted_string( bool brief ); - std::string m_message; - std::string m_filename; - unsigned long m_line_number; - std::string m_function_name; - std::string m_formatted_message; + std::string _message; + std::string _filename; + unsigned long _line{0}; + std::string _function; + std::string _formatted_message; }; -inline exception::exception( const char *message, const char *filename, unsigned long line_number, const char *function_name ) : - std::runtime_error( message ), - m_message( message ), - m_filename( filename ), - m_line_number( line_number ), - m_function_name( function_name ) -{ - formatted_string_(); -} - -inline exception::exception( const std::string& message, const char *filename, unsigned long line_number, const char *function_name ) : - std::runtime_error( message.c_str() ), - m_message( message ), - m_filename( filename ), - m_line_number( line_number ), - m_function_name( function_name ) -{ - formatted_string_(); -} - -inline const char* exception::what(void) const throw() -{ - const char * string = "exception::what() m_formatted_message.c_str() threw exception"; - try - { - string = m_formatted_message.c_str(); - } - catch( ... ) - {} - return string; -} - -inline const char* exception::error() const -{ - return m_message.c_str(); -} - - -inline const char* exception::file() const -{ - return m_filename.c_str(); -} - -inline unsigned long exception::line() const -{ - return m_line_number; -} - -inline const char* exception::function() const -{ - return m_function_name.c_str(); -} - -inline void exception::formatted_string_() -{ - std::ostringstream oss; - oss << error() << std::endl - << "============================================" << std::endl - << "file: " << m_filename << std::endl - << "line: " << m_line_number << std::endl - << "function: " << m_function_name << std::endl - << "============================================" << std::endl; - m_formatted_message = oss.str(); -} - } // namespace comma - -#endif //COMMA_BASE_EXCEPTION_H - diff --git a/base/last_error.cpp b/base/last_error.cpp index d62ee22fe..f045465c0 100644 --- a/base/last_error.cpp +++ b/base/last_error.cpp @@ -1,32 +1,4 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine @@ -67,43 +39,43 @@ std::string last_error::to_string() void last_error::to_exception( const std::string& msg ) { #ifdef WIN32 - switch( value() ) - { - // TODO: add more exceptions - case 0: break; - case WSAEINTR: COMMA_THROW( last_error::interrupted_system_call_exception, msg ); - default: COMMA_THROW( last_error::exception, msg ); - } + switch( value() ) + { + // TODO: add more exceptions + case 0: break; + case WSAEINTR: COMMA_THROW( last_error::interrupted_system_call_exception, msg ); + default: COMMA_THROW( last_error::exception, msg ); + } #else - switch( value() ) - { - // TODO: add more exceptions - case 0: break; - case EINTR: COMMA_THROW( last_error::interrupted_system_call_exception, msg ); - default: COMMA_THROW( last_error::exception, msg ); - }; + switch( value() ) + { + // TODO: add more exceptions + case 0: break; + case EINTR: COMMA_THROW( last_error::interrupted_system_call_exception, msg ); + default: COMMA_THROW( last_error::exception, msg ); + }; #endif } -last_error::exception::exception( const char* msg, const char *filename, unsigned long line_number, const char *function_name ) - : comma::exception( std::string( msg ) + ": errno " + boost::lexical_cast< std::string >( last_error::value() ) + " - " + last_error::to_string(), filename, line_number, function_name ) +last_error::exception::exception( const char* msg, const char *filename, unsigned long line_number, const char *function_name, bool brief ) + : comma::exception( std::string( msg ) + ": errno " + boost::lexical_cast< std::string >( last_error::value() ) + " - " + last_error::to_string(), filename, line_number, function_name, brief ) { value = last_error::value(); } -last_error::exception::exception( const std::string& msg, const char *filename, unsigned long line_number, const char *function_name ) - : comma::exception( msg + ": errno " + boost::lexical_cast< std::string >( last_error::value() ) + " - " + last_error::to_string(), filename, line_number, function_name ) +last_error::exception::exception( const std::string& msg, const char *filename, unsigned long line_number, const char *function_name, bool brief ) + : comma::exception( msg + ": errno " + boost::lexical_cast< std::string >( last_error::value() ) + " - " + last_error::to_string(), filename, line_number, function_name, brief ) { value = last_error::value(); } -last_error::interrupted_system_call_exception::interrupted_system_call_exception( const char* msg, const char *filename, unsigned long line_number, const char *function_name ) - : last_error::exception( msg, filename, line_number, function_name ) +last_error::interrupted_system_call_exception::interrupted_system_call_exception( const char* msg, const char *filename, unsigned long line_number, const char *function_name, bool brief ) + : last_error::exception( msg, filename, line_number, function_name, brief ) { } -last_error::interrupted_system_call_exception::interrupted_system_call_exception( const std::string& msg, const char *filename, unsigned long line_number, const char *function_name ) - : last_error::exception( msg, filename, line_number, function_name ) +last_error::interrupted_system_call_exception::interrupted_system_call_exception( const std::string& msg, const char *filename, unsigned long line_number, const char *function_name, bool brief ) + : last_error::exception( msg, filename, line_number, function_name, brief ) { } diff --git a/base/last_error.h b/base/last_error.h index ddc54c228..da47c2808 100644 --- a/base/last_error.h +++ b/base/last_error.h @@ -1,37 +1,8 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine -#ifndef COMMA_BASE_LAST_ERROR_HEADER -#define COMMA_BASE_LAST_ERROR_HEADER +#pragma once #include #include "exception.h" @@ -65,18 +36,16 @@ struct last_error struct last_error::exception : public comma::exception { - exception( const char*, const char *filename, unsigned long line_number, const char *function_name ); - exception( const std::string&, const char *filename, unsigned long line_number, const char *function_name ); + exception( const char*, const char *filename, unsigned long line_number, const char *function_name, bool brief = false ); + exception( const std::string&, const char *filename, unsigned long line_number, const char *function_name, bool brief = false ); int value; }; struct last_error::interrupted_system_call_exception : public last_error::exception { - interrupted_system_call_exception( const char*, const char *filename, unsigned long line_number, const char *function_name ); - interrupted_system_call_exception( const std::string&, const char *filename, unsigned long line_number, const char *function_name ); + interrupted_system_call_exception( const char*, const char *filename, unsigned long line_number, const char *function_name, bool brief = false ); + interrupted_system_call_exception( const std::string&, const char *filename, unsigned long line_number, const char *function_name, bool brief = false ); }; } // namespace comma { - -#endif // #ifndef COMMA_BASE_LAST_ERROR_HEADER diff --git a/base/none.h b/base/none.h new file mode 100644 index 000000000..77ef009fe --- /dev/null +++ b/base/none.h @@ -0,0 +1,25 @@ +// Copyright (c) 2022 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +#include + +namespace comma { + +/// convenience type to use e.g. as a "tag" type in template definitions +/// since boost::none_t is not default-constructible as it is designed +/// to be a singleton type (see boost/none_t.hpp for details) meaning +/// that it won't compile for some use cases +struct none {}; + +/// a quick fix for annoying boost::optional compilation warning +/// boost::optional< int > i; // gets compile warning when i first dereferenced +/// // even if i is initialized for sure somewhere in the code +/// boost::optional< int > i{boost::none}; // still same compile warning +/// boost::optional< int > i = comma::silent_none< int >(); // fine, no warning +template < typename T > +inline boost::optional< T > silent_none() { return boost::optional< T >( boost::none ); } + +} // namespace comma { diff --git a/base/optional.h b/base/optional.h new file mode 100644 index 000000000..475dc3619 --- /dev/null +++ b/base/optional.h @@ -0,0 +1,32 @@ +// Copyright (c) 2024 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +namespace comma { + +/// convenience class when std::optional or boost::optional is not enough +/// e.g. if in visiting we would like to have an explicit is-set flag field +/// in csv, json, or alike, where it may be essential in fixed-width data +/// (e.g. csv) where the optional value may or may not be present +template < typename T > +struct optional +{ + T value; + bool is_set{false}; + + optional() = default; + optional( const T& t ): value( t ), is_set( true ) {} + template < class... Args > optional( Args... args ): value( args... ), is_set( true ) {} + template < class... Args > void emplace( Args... args ); // todo + optional& operator=( const T& rhs ) { value = rhs; is_set = true; return *this; } + void reset() { is_set = false; } + operator bool() const { return is_set; } + T* operator->() { return &value; } + const T* operator->() const { return &value; } + T& operator*() { return value; } + const T& operator*() const { return value; } +}; + +} // namespace comma { diff --git a/base/test/CMakeLists.txt b/base/test/CMakeLists.txt new file mode 100644 index 000000000..9fb546dd8 --- /dev/null +++ b/base/test/CMakeLists.txt @@ -0,0 +1,13 @@ +set( KIT base ) +file( GLOB source ${SOURCE_CODE_BASE_DIR}/${KIT}/test/*test.cpp ) +set( test_name ${CMAKE_PROJECT_NAME}_test_${KIT} ) +add_executable( ${test_name} ${source} ) +target_link_libraries( ${test_name} comma_base ${GTEST_BOTH_LIBRARIES} pthread ) +add_test( NAME ${test_name} COMMAND ${test_name} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/bin ) +if( INSTALL_TESTS ) + install( TARGETS ${test_name} RUNTIME DESTINATION ${comma_CPP_TESTS_INSTALL_DIR} COMPONENT Runtime ) + #INSTALL ( + # FILES ${PROJECT_BINARY_DIR}/bin/${CMAKE_PROJECT_NAME}_test_${KIT} + # PERMISSIONS WORLD_READ GROUP_READ OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE + # DESTINATION ${comma_CPP_TESTS_INSTALL_DIR} ) +endif( INSTALL_TESTS ) diff --git a/base/test/base_test.cpp b/base/test/base_test.cpp new file mode 100644 index 000000000..2a2bb343a --- /dev/null +++ b/base/test/base_test.cpp @@ -0,0 +1,175 @@ +// Copyright (c) 2023 Vsevolod Vlaskine + +#include "../exception.h" +#include "../variant.h" +#include + +namespace comma { + +TEST( base, exception ) +{ + COMMA_ASSERT( true, "all good" ); + COMMA_ASSERT( 2 * 2 == 4, "all good" ); + EXPECT_THROW( COMMA_ASSERT( false, "all bad" ), comma::exception ); + EXPECT_THROW( COMMA_ASSERT( 2 * 2 == 5, "all bad" ), comma::exception ); + COMMA_THROW_IF( false, "all good" ); + COMMA_THROW_IF( 2 * 2 == 5, "all good" ); + EXPECT_THROW( COMMA_THROW_IF( true, "all bad" ), comma::exception ); + EXPECT_THROW( COMMA_THROW_IF( 2 * 2 == 4, "all bad" ), comma::exception ); +} + +TEST( base, variant ) +{ + { + comma::impl::variant< int, float, double > v; + v.t = 1; + v.values.t = 2; + v.values.values.t = 3; + } + { + comma::impl::variant< int, float, double > v; + EXPECT_FALSE( v ); + v.set< int >( 5 ); + EXPECT_TRUE( v ); + v.reset(); + EXPECT_FALSE( v ); + v.set< float >( 5 ); + EXPECT_TRUE( v ); + v.reset(); + EXPECT_FALSE( v ); + v.set< double >( 5 ); + EXPECT_TRUE( v ); + v.reset(); + EXPECT_FALSE( v ); + } + { + comma::impl::variant< int, float, double > v; + EXPECT_FALSE( v.is< int >() ); + EXPECT_FALSE( v.is< float >() ); + EXPECT_FALSE( v.is< double >() ); + v.set< int >( 5 ); + EXPECT_EQ( *v.t, 5 ); + EXPECT_TRUE( v.is< int >() ); + EXPECT_FALSE( v.is< float >() ); + EXPECT_FALSE( v.is< double >() ); + v.set< float >( 5 ); + EXPECT_EQ( *v.values.t, 5 ); + EXPECT_FALSE( v.is< int >() ); + EXPECT_TRUE( v.is< float >() ); + EXPECT_FALSE( v.is< double >() ); + v.set< double >( 5 ); + EXPECT_EQ( *v.values.values.t, 5 ); + EXPECT_FALSE( v.is< int >() ); + EXPECT_FALSE( v.is< float >() ); + EXPECT_TRUE( v.is< double >() ); + v.set< int >( 5 ); + EXPECT_EQ( *v.t, 5 ); + EXPECT_TRUE( v.is< int >() ); + EXPECT_FALSE( v.is< float >() ); + EXPECT_FALSE( v.is< double >() ); + } + { + comma::variant< int, float, double > v; + EXPECT_FALSE( v.is< int >() ); + EXPECT_FALSE( v.is< float >() ); + EXPECT_FALSE( v.is< double >() ); + v.set< int >( 5 ); + EXPECT_EQ( v.get< int >(), 5 ); + EXPECT_TRUE( v.optional< int >() ); + EXPECT_FALSE( v.optional< float >() ); + EXPECT_FALSE( v.optional< double >() ); + EXPECT_EQ( *v.optional< int >(), 5 ); + EXPECT_TRUE( v.is< int >() ); + EXPECT_FALSE( v.is< float >() ); + EXPECT_FALSE( v.is< double >() ); + v.set< float >( 5 ); + EXPECT_EQ( v.get< float >(), 5 ); + EXPECT_FALSE( v.optional< int >() ); + EXPECT_TRUE( v.optional< float >() ); + EXPECT_FALSE( v.optional< double >() ); + EXPECT_EQ( *v.optional< float >(), 5 ); + EXPECT_FALSE( v.is< int >() ); + EXPECT_TRUE( v.is< float >() ); + EXPECT_FALSE( v.is< double >() ); + v.set< double >( 5 ); + EXPECT_EQ( v.get< double >(), 5 ); + EXPECT_FALSE( v.optional< int >() ); + EXPECT_FALSE( v.optional< float >() ); + EXPECT_TRUE( v.optional< double >() ); + EXPECT_EQ( *v.optional< double >(), 5 ); + EXPECT_FALSE( v.is< int >() ); + EXPECT_FALSE( v.is< float >() ); + EXPECT_TRUE( v.is< double >() ); + v.set< int >( 5 ); + EXPECT_EQ( v.get< int >(), 5 ); + EXPECT_TRUE( v.optional< int >() ); + EXPECT_FALSE( v.optional< float >() ); + EXPECT_FALSE( v.optional< double >() ); + EXPECT_EQ( *v.optional< float >(), 5 ); + EXPECT_EQ( *v.optional< int >(), 5 ); + EXPECT_TRUE( v.is< int >() ); + EXPECT_FALSE( v.is< float >() ); + EXPECT_FALSE( v.is< double >() ); + } + { + { auto size = comma::variant< int >::size; EXPECT_EQ( size, 1 ); } + { auto size = comma::variant< int, float >::size; EXPECT_EQ( size, 2 ); } + { auto size = comma::variant< int, float, double >::size; EXPECT_EQ( size, 3 ); } + } + { + typedef comma::variant< int, float, double > variant_t; + EXPECT_EQ( variant_t::index_of< int >(), 0 ); + EXPECT_EQ( variant_t::index_of< float >(), 1 ); + EXPECT_EQ( variant_t::index_of< double >(), 2 ); + variant_t v; + EXPECT_EQ( v.index(), 3 ); + v.set< int >( 5 ); + EXPECT_EQ( v.index(), 0 ); + v.set< float >( 5 ); + EXPECT_EQ( v.index(), 1 ); + v.set< double >( 5 ); + EXPECT_EQ( v.index(), 2 ); + v.reset(); + EXPECT_EQ( v.index(), 3 ); + } +} + +TEST( base, named_variant ) +{ + { + struct naming { static std::array< std::string, 3 > names() { return { "a", "b", "c" }; } }; + typedef comma::named_variant< naming, int, float, double > variant_t; + EXPECT_EQ( variant_t::name_of< int >(), "a" ); + EXPECT_EQ( variant_t::name_of< float >(), "b" ); + EXPECT_EQ( variant_t::name_of< double >(), "c" ); + } + { + struct naming { static std::array< std::string, 3 > names() { return { "a", "b", "c" }; } }; + comma::named_variant< naming, int, float, double > v; + EXPECT_FALSE( v ); + v.set< int >( 5 ); + EXPECT_TRUE( v ); + EXPECT_EQ( v.name(), "a" ); + v.reset(); + EXPECT_FALSE( v ); + v.set< float >( 5 ); + EXPECT_TRUE( v ); + EXPECT_EQ( v.name(), "b" ); + v.reset(); + EXPECT_FALSE( v ); + v.set< double >( 5 ); + EXPECT_TRUE( v ); + EXPECT_EQ( v.name(), "c" ); + v.reset(); + EXPECT_FALSE( v ); + EXPECT_THROW( v.name(), comma::exception ); + } +} + +} // namespace comma { + +int main( int argc, char* argv[] ) +{ + ::testing::InitGoogleTest( &argc, argv ); + return RUN_ALL_TESTS(); +} diff --git a/base/types.h b/base/types.h index 065547899..22f4f1944 100644 --- a/base/types.h +++ b/base/types.h @@ -1,37 +1,8 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine -#ifndef COMMA_BASE_TYPES_H_ -#define COMMA_BASE_TYPES_H_ +#pragma once #if defined(__linux__) || defined(__APPLE__) || defined(__QNXNTO__) #include @@ -45,7 +16,7 @@ #include #include -#include +#include namespace comma { @@ -70,12 +41,12 @@ typedef __int32 int32; typedef __int64 int64; // Windows, you know... -BOOST_STATIC_ASSERT( sizeof( uint16 ) == 2 ); -BOOST_STATIC_ASSERT( sizeof( uint32 ) == 4 ); -BOOST_STATIC_ASSERT( sizeof( uint64 ) == 8 ); -BOOST_STATIC_ASSERT( sizeof( int16 ) == 2 ); -BOOST_STATIC_ASSERT( sizeof( int32 ) == 4 ); -BOOST_STATIC_ASSERT( sizeof( int64 ) == 8 ); +static_assert( sizeof( uint16 ) == 2, "expected uint16 of size 2" ); +static_assert( sizeof( uint32 ) == 4, "expected uint32 of size 4" ); +static_assert( sizeof( uint64 ) == 8, "expected uint64 of size 8" ); +static_assert( sizeof( int16 ) == 2, "expected int16 of size 2" ); +static_assert( sizeof( int32 ) == 4, "expected int32 of size 4" ); +static_assert( sizeof( int64 ) == 8, "expected int64 of size 8" ); #endif @@ -90,5 +61,3 @@ template <> struct integer< 8, true > { typedef comma::int64 type; }; template <> struct integer< 8, false > { typedef comma::uint64 type; }; } // namespace comma { - -#endif /*COMMA_BASE_TYPES_H_*/ diff --git a/base/variant.h b/base/variant.h new file mode 100644 index 000000000..5d2b822bd --- /dev/null +++ b/base/variant.h @@ -0,0 +1,131 @@ +// Copyright (c) 2024 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +// todo +// #if __cplusplus >= 201703L +// #include +// #else +// #include +// #endif +#include +#include +#include "exception.h" + +namespace comma { + +// todo +// ? use tuple instead? +// - check that types don't repeat + +namespace impl { + +struct type_is_not_on_type_list{}; // quick and dirty, a tag struct + +template < typename T, bool B > struct variant_traits; + +template < typename T > struct variant_traits< T, false > +{ + template < typename S > static void set( boost::optional< T >& t, const S& ) { t.reset(); } + template < typename S > static void set( boost::optional< T >& t, const boost::optional< S >& ) { t.reset(); } + template < typename S, typename V > static const S& get( const boost::optional< T >&, const V& v ) { return v.template get< S >(); } + template < typename S, typename V > static const boost::optional< S >& optional( const boost::optional< T >&, const V& v ) { return v.template optional< S >(); } +}; + +template < typename T > struct variant_traits< T, true > +{ + template < typename S > static void set( boost::optional< T >& t, const S& s ) { t = s; } + template < typename S > static void set( boost::optional< T >& t, const boost::optional< S >& s ) { t = s; } + template < typename S, typename V > static const S& get( const boost::optional< T >& t, const V& ) { return *t; } + template < typename S, typename V > static const boost::optional< S >& optional( const boost::optional< T >& t, const V& ) { return t; } +}; + +template < typename T, typename... Args > struct variant // todo? use tuple instead? +{ + enum { size = variant< Args... >::size + 1 }; + boost::optional< T > t; + variant< Args... > values; + + template < typename S > bool is() const { return ( std::is_same< T, S >::value && bool( t ) ) || values.template is< S >(); } + operator bool() const { return bool( t ) || bool( values ); } + template < typename S > void set( const S& s ) { variant_traits< T, std::is_same< T, S >::value >::set( t, s ); values.set( s ); } + template < typename S > void set( const boost::optional< S >& s ) { variant_traits< T, std::is_same< T, S >::value >::set( t, s ); values.set( s ); } + template < typename S > const S& get() const { return variant_traits< T, std::is_same< T, S >::value >::template get< S >( t, values ); } + template < typename S > const boost::optional< S >& optional() const { return variant_traits< T, std::is_same< T, S >::value >::template optional< S >( t, values ); } + void reset() { t.reset(); values.reset(); } + template < typename S > static unsigned int rindex() { return std::is_same< T, S >::value ? size - 1 : variant< Args... >::template rindex< S >(); } + unsigned int index( unsigned int i = 0 ) const { return t ? i : values.index( i + 1 ); } +}; + +template < typename T > struct variant< T > // todo? use tuple instead? +{ + enum { size = 1 }; + boost::optional< T > t; + + template < typename S > bool is() const { return std::is_same< T, S >::value && bool( t ); } + operator bool() const { return bool( t ); } + template < typename S > void set( const S& s ) { variant_traits< T, std::is_same< T, S >::value >::set( t, s ); } + template < typename S > void set( const boost::optional< S >& s ) { variant_traits< T, std::is_same< T, S >::value >::set( t, s ); } + template < typename S > const S& get() const { return variant_traits< T, std::is_same< T, S >::value >::template get< S >( t, type_is_not_on_type_list() ); } + template < typename S > const boost::optional< S >& optional() const { return variant_traits< T, std::is_same< T, S >::value >::template optional< S >( t, type_is_not_on_type_list() ); } + void reset() { t.reset(); } + template < typename S > static unsigned int rindex() { bool same_type = std::is_same< T, S >::value; COMMA_ASSERT( same_type, "type not found in type list" ); return 0; } + unsigned int index( unsigned int i = 0 ) const { return t ? i : ( i + 1 ); } +}; + +} // namespace impl { + +/// @example +/// struct chirp { int a{1}; int b{2}; }; +/// struct whistle { int a{3}; int b{4}; }; +/// struct warble { int x{5}; int y{6}; }; +/// comma::named_variant< naming, chirp, whistle, warble > sound; +template < typename... Args > +class variant +{ + public: + enum { size = impl::variant< Args... >::size }; + variant() = default; + template < typename S > variant( const S& s ) { set( s ); } + template < typename S > bool is() const { return _values.template is< S >(); } + operator bool() const { return bool( _values ); } + template < typename S > void set( const S& s ) { _values.set( s ); } + template < typename S > void set( const boost::optional< S >& s ) { _values.set( s ); } + template < typename S > const S& get() const { return _values.template get< S >(); } + template < typename S > const boost::optional< S >& optional() const { return _values.template optional< S >(); } + void reset() { _values.reset(); } + template < typename S > static unsigned int index_of() { return impl::variant< Args... >::size - impl::variant< Args... >::template rindex< S >() - 1; } + unsigned int index() const { return _values.index(); } + protected: + impl::variant< Args... > _values; +}; + +/// @example +/// struct forest +/// { +/// struct chirp { int a{1}; int b{2}; }; +/// struct whistle { int a{3}; int b{4}; }; +/// struct warble { int x{5}; int y{6}; }; +/// +/// struct naming { static std::array< std::string, 3 > names() { return { "chirp", "whistle", "warble" }; } }; +/// +/// comma::named_variant< naming, chirp, whistle, warble > sound; +/// }; +template < typename Names, typename... Args > +struct named_variant : public variant< Args... >, public Names +{ + typedef Names names_t; + typedef variant< Args... > variant_t; + template < typename S > static auto name_of() { return Names::names()[ variant_t::template index_of< S >() ]; } + auto name() const { COMMA_ASSERT( bool( *this ), "asked for name, but value is not set" ); return this->names()[this->index()]; } +}; + +template < typename Names > +struct make_named_variant +{ + template < typename... Args > struct variant { typedef named_variant< Names, Args... > type; }; +}; + +} // namespace comma { diff --git a/bash/CMakeLists.txt b/bash/CMakeLists.txt index 49d6ee611..4f3460a12 100644 --- a/bash/CMakeLists.txt +++ b/bash/CMakeLists.txt @@ -1,13 +1,13 @@ -INSTALL( PROGRAMS comma-application-util DESTINATION ${comma_INSTALL_BIN_DIR} ) -INSTALL( PROGRAMS comma-log-util DESTINATION ${comma_INSTALL_BIN_DIR} ) -INSTALL( PROGRAMS comma-name-value-util DESTINATION ${comma_INSTALL_BIN_DIR} ) -INSTALL( PROGRAMS comma-progress-util DESTINATION ${comma_INSTALL_BIN_DIR} ) -INSTALL( PROGRAMS comma-resources-util DESTINATION ${comma_INSTALL_BIN_DIR} ) -INSTALL( PROGRAMS comma-sync-util DESTINATION ${comma_INSTALL_BIN_DIR} ) -INSTALL( PROGRAMS comma-units-util DESTINATION ${comma_INSTALL_BIN_DIR} ) +install( PROGRAMS comma-application-util DESTINATION ${comma_INSTALL_BIN_DIR} ) +install( PROGRAMS comma-log-util DESTINATION ${comma_INSTALL_BIN_DIR} ) +install( PROGRAMS comma-name-value-util DESTINATION ${comma_INSTALL_BIN_DIR} ) +install( PROGRAMS comma-progress-util DESTINATION ${comma_INSTALL_BIN_DIR} ) +install( PROGRAMS comma-resources-util DESTINATION ${comma_INSTALL_BIN_DIR} ) +install( PROGRAMS comma-sync-util DESTINATION ${comma_INSTALL_BIN_DIR} ) +install( PROGRAMS comma-units-util DESTINATION ${comma_INSTALL_BIN_DIR} ) if( comma_BUILD_APPLICATIONS ) add_subdirectory( applications ) endif( comma_BUILD_APPLICATIONS ) -ADD_SUBDIRECTORY( process ) -ADD_SUBDIRECTORY( misc ) +add_subdirectory( process ) +add_subdirectory( misc ) diff --git a/bash/applications/comma-call-graph b/bash/applications/comma-call-graph index 3ebef826a..d9e916ace 100755 --- a/bash/applications/comma-call-graph +++ b/bash/applications/comma-call-graph @@ -108,10 +108,10 @@ load_options $@ if [[ $dot_output && $dot_output != "dot" ]]; then type -p dot > /dev/null || { - echo "$basename requires graphviz" - echo "Install on Ubuntu with:" - echo "$ sudo apt-get install graphviz" - exit + echo "$basename: requires graphviz" >&2 + echo "$basename: install on ubuntu with:" >&2 + echo "$basename: sudo apt-get install graphviz" >&2 + exit 1 } output_fn="dot -T$dot_output" else diff --git a/bash/comma-application-util b/bash/comma-application-util index ab656d7b8..4f6ff584b 100644 --- a/bash/comma-application-util +++ b/bash/comma-application-util @@ -1,33 +1,7 @@ #!/bin/bash -# This file is part of comma, a generic and flexible library # Copyright (c) 2011 The University of Sydney -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of the University of Sydney nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -# GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -# HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Copyright (c) 2020 Vsevolod Vlaskine ## @page comma-application-util comma-application-util # this script is designed to be included in another script directly, for example: @@ -43,8 +17,25 @@ [[ -n "$comma_application_util_include_guard_" ]] && return 0 readonly comma_application_util_include_guard_=1 -source $( type -p comma-name-value-util ) \ - || { echo "$name: cannot source 'comma-name-value-util' from '$BASH_SOURCE'" >&2; exit 1; } +source $( type -p comma-name-value-util ) || { echo "$comma_application_name: cannot source 'comma-name-value-util' from '$BASH_SOURCE'" >&2; exit 1; } + +## @page comma-source-relative comma-source-relative +# @section comma-source-relative +# @description source relative to the location of the calling +# script/subshell (a trivial convenience wrapper) +# @param whitespace-separated relative paths to source +# regular expressions permitted, e.g. my/sources/* +# as well as directory names, e.g. my/sources, which is the +# same as my/sources/* +function comma-source-relative() +{ + local dir="$( realpath $( dirname $0 ) )" + for s in "$@"; do + if [[ -d "$s" ]]; then for t in "$s"/*; do comma-source-relative "$t" || return 1; done + else source "$dir/$s" || return 1; fi + done +} +export -f comma-source-relative ## @page comma_options_from_name_value comma_options_from_name_value # @section comma_options_from_name_value @@ -120,7 +111,7 @@ export -f comma_pretty_description ## @page comma_options_names comma_options_names # @description converts a standard comma description to a list of names -# +# # example # description | comma_options_names function comma_options_names() @@ -132,7 +123,7 @@ export -f comma_options_names ## @page comma_options_to_bash_completion comma_options_to_bash_completion # @description converts a standard comma description to input # for the comma bash completion function -# +# # example # description | comma_options_to_bash_completion function comma_options_to_bash_completion() @@ -144,7 +135,7 @@ export -f comma_options_to_bash_completion ## @page comma_options_select comma_options_select # @description after a set of options have been retrieved as paths convert to # options format for a child script -# +# # example # comma-test-run $( comma-test-run --description | comma-options-to-name-value $@ | comma_name_value_to_options ) function comma_name_value_to_options() @@ -229,3 +220,124 @@ function comma_tee_function() io-tee "${file_and_options[@]}" "$functions ${functions:+;} $command" "${command_args[@]}" } export -f comma_tee_function + +function comma-application-common-options() +{ + cat <&2 + cat <&2 + +usage: $comma_application_name [] + +options +$( ( [[ -z "$common_options" ]] || echo "$common_options"; comma-application-common-options ) | sed 's#^# #' ) + +eof + [[ "$( type -t epilogue )" != "function" ]] || { epilogue; echo; } >&2 + exit 0 + fi + function say() { echo "$comma_application_name: $@" >&2; } + function saymore() { (( ! options_verbose )) || echo "$comma_application_name: $@" >&2; } + function verbose() { (( ! options_verbose )) || echo "$comma_application_name: $@" >&2; } + function die() { say "$@"; exit 1; } + local options; options=$( ( echo "$common_options"; comma-application-common-options ) | comma-options-to-name-value "$@" ) || die "invalid options" + eval "$( grep -v '^"' <<< "$options" | sed 's#^#options_#' | comma_path_value_mangle )" + # declare -a options_unnamed + mapfile -t options_unnamed < <( grep '^"' <<< "$options" | sed -e 's#^"##' -e 's#"$##' ) +} + +export -f comma-application-init + +function comma-application-swiss-init() +{ + comma_application_name=$( basename "$0" ) + local common_options=$( cat ) + [[ -n "$1" ]] || { echo "$comma_application_name: please specify operation" >&2; exit 1; } + comma_application_swiss_operation="$1" # todo? quick and dirty, allow --help anywhere on the command line? + if comma_options_help $@; then + if [[ "${comma_application_swiss_operation}" == "--help" || "${comma_application_swiss_operation}" == "-h" ]]; then + [[ "$( type -t prologue )" != "function" ]] || { echo; prologue; } + echo "$common_options" | comma-application-swiss-usage >&2 + [[ "$( type -t epilogue )" != "function" ]] || { epilogue; echo; } + else + { [[ -z "$common_options" ]] || echo "$common_options"; comma-application-common-options; } | comma-application-swiss-usage-operation "${comma_application_swiss_operation}" || exit 1 + echo + fi >&2 + exit 0 + fi + function say() { echo "$comma_application_name: ${comma_application_swiss_operation}: $@" >&2; } + function saymore() { (( ! options_verbose )) || echo "$comma_application_name: ${comma_application_swiss_operation}: $@" >&2; } + function verbose() { (( ! options_verbose )) || echo "$comma_application_name: ${comma_application_swiss_operation}: $@" >&2; } + function die() { say "$@"; exit 1; } + [[ $( type -t ${comma_application_swiss_operation}-run ) == "function" ]] || die "expected operation, got '${comma_application_swiss_operation}'" # quick and dirty + local options_described=$( echo "$common_options"; comma-application-common-options ) + local options; options=$( comma-options-to-name-value "$@" <<< "$options_described" ) || die "invalid options" + eval "$( grep -v '^"' <<< "$options" | sed 's#^#options_#' | comma_path_value_mangle )" + #(( !options_verbose )) || verbose_option="--verbose" + #(( !options_input_fields )) || { eval "echo \$${operation//-/_}_input_fields"; exit 0; } # quick and dirty + #(( !options_input_format )) || { eval "echo \$${operation//-/_}_input_format"; exit 0; } # quick and dirty + #(( !options_output_fields )) || { eval "echo \$${operation//-/_}_output_fields"; exit 0; } # quick and dirty + #(( !options_output_format )) || { eval "echo \$${operation//-/_}_output_format"; exit 0; } # quick and dirty + if [[ $( type -t ${comma_application_swiss_operation}-options ) == "function" ]]; then + local operation_options_described=$( ${comma_application_swiss_operation}-options ) + local operation_options; operation_options=$( comma-options-to-name-value "$@" <<< "$operation_options_described" ) || die "invalid options" + eval "$( grep -v '^"' <<< "$operation_options" | sed 's#^#options_#' | comma_path_value_mangle )" + fi + mapfile -t options_unnamed < <( ( echo "$options_described"; echo "$operation_options_described"; ) | comma-options-to-name-value "$@" | grep '^"' | sed -e 's#^"##' -e 's#"$##' ) +} + +export -f comma-application-swiss-init + +function comma-application-swiss-usage-operation() +{ + local operation="$1" + [[ $( type -t ${operation}-run ) == "function" ]] || { echo "$( basename $0 ): expected operation, got: '${comma_application_swiss_operation}', see --help with no other options for more information" >&2; return 1; } # quick and dirty + local common_options=$( cat ) + local found=0 + if [[ -z "$common_options" ]]; then echo "${operation}"; else echo "$( basename "$0" ) ${operation}"; fi # uber quick and dirty for now + [[ $( type -t ${operation}-description ) != "function" ]] || { ${operation}-description | sed 's#^# #'; found=1; } + if [[ $( type -t ${operation}-usage ) == "function" ]]; then echo " usage"; ${operation}-usage | sed 's#^# #'; found=1 + else echo " usage: $( basename $0 ) ${comma_application_swiss_operation} []"; fi + [[ -z "$common_options" ]] || { echo " common options"; echo "$common_options" | sed 's#^# #'; } + [[ $( type -t ${operation}-options ) != "function" ]] || { echo " options"; ${operation}-options | sed 's#^# #'; found=1; } + [[ $( type -t ${operation}-examples ) != "function" ]] || { echo " examples"; ${operation}-examples | sed 's#^# #'; found=1; } + (( ! found )) || return 0 + echo "$( basename $0 ): expected operation, got: '${operation}', see --help with no other options for more information" >&2 + return 1 +} + +export -f comma-application-swiss-usage-operation + +function comma-application-swiss-usage() +{ + local common_options=$( cat ) + echo + if [[ $( type -t usage ) == "function" ]]; then usage; else echo "usage: $( basename "$0" ) "; fi + echo; echo "common options" + { [[ -z "$common_options" ]] || echo "$common_options"; comma-application-common-options; } | sed 's#^# #' + echo; echo "available operations: ${operations[@]}" + echo; echo "operations" + for operation in ${operations[@]}; do + comma-application-swiss-usage-operation "${operation}" | sed 's#^# #' + echo + done +} + +export -f comma-application-swiss-usage + +function comma-application-swiss-run() { ${comma_application_swiss_operation}-run "$@"; } + +export -f comma-application-swiss-run diff --git a/bash/comma-name-value-util b/bash/comma-name-value-util index 450445d90..1774a7478 100644 --- a/bash/comma-name-value-util +++ b/bash/comma-name-value-util @@ -39,6 +39,9 @@ readonly comma_name_value_util_include_guard_=1 || source $( type -p comma-log-util ) \ || { echo "$name: cannot source 'comma-log-util' from '$BASH_SOURCE'" >&2; exit 1; } +# usage: echo | comma_path_value_mangle [] [] +# : default='_' +# # take path-value pairs, mangle path to turn them into bash # expressions, e.g: # echo hello/world=5 | comma_path_value_mangle @@ -53,15 +56,18 @@ readonly comma_name_value_util_include_guard_=1 # local hello_world="5" function comma_path_value_mangle() { - local delimiter="$1" + local delimiter="$1" default_value="$2" [[ -n "$delimiter" ]] || delimiter="_" local path name value while IFS='=' read -r path value || [[ -n "$path" ]]; do [[ -n "$path" && ! "$path" =~ ^[#\"] ]] || continue name=${path//\//$delimiter} name=${name//-/$delimiter} - value=${value#\"} - value=${value%\"} + name=${name//[/$delimiter} + name=${name//]/} # just remove trailing ] since it's always followed by / or the end of the line + value="${value#\"}" + value="${value%\"}" + [[ -n "$value" || -z "$default_value" ]] || value="$default_value" echo "$name='$value'" done } @@ -267,15 +273,15 @@ function comma_path_value_to_var() name=${name//-/$delimiter} # we are occasionally passed bad data in JSON etc # don't attempt to eval anything that is not a legal variable name - [[ "$name" =~ $regex ]] || { echo "comma-application-util: comma_path_value_to_var skipping invalid variable name '$name'" >&2 ; ret_code=1 ; continue ; } + [[ "$name" =~ $regex ]] || { echo "comma-name-value-util: comma_path_value_to_var skipping invalid variable name '$name'" >&2 ; ret_code=1 ; continue ; } # Strip quotes then add them back. This ensures consistent behaviour. value=${value#\"} value=\"${value%\"}\" eval "$export_variable $name=$value" \ - || { echo "comma-application-util: comma_path_value_to_var failed '$export_variable $name=$value'" >&2 ; ret_code=1 ; } + || { echo "comma-name-value-util: comma_path_value_to_var failed '$export_variable $name=$value'" >&2 ; ret_code=1 ; } done if (( $ret_code != 0 )) ; then - echo "comma-application-util: comma_path_value_to_var encountered an error, callstack:" >&2 + echo "comma-name-value-util: comma_path_value_to_var encountered an error, callstack:" >&2 comma_stacktrace >&2 fi return $ret_code diff --git a/bash/comma-resources-util b/bash/comma-resources-util index 165b72a0d..85ae2038d 100644 --- a/bash/comma-resources-util +++ b/bash/comma-resources-util @@ -168,13 +168,8 @@ export -f comma_queue_is_empty_infile # Extract cpu count, cores and sockets from lscpu function comma_cpu_resources() { - local lscpu=$( lscpu --parse ) - local fields=$( echo "$lscpu" | grep '^#' | tail -1 | sed 's/^[# ]*//' ) - { - # lscpu --parse gives us zero-based indexes. Add one to get the count - echo "$lscpu" | tail -1 | csv-shuffle --fields $fields --output CPU,Core,Socket - echo "1,1,1" - } | csv-calc sum --format 3i | name-value-from-csv --fields cpu,core,socket --prefix cpu + local fields='cpu,core,socket' + { lscpu --parse="$fields" | grep -v '^#' | tail -n1; echo 1,1,1; } | csv-calc sum --format 3i | name-value-from-csv --fields="$fields" --prefix=cpu } # Output the most commonly used system resources: number of CPUs, total RAM, diff --git a/bash/process/comma-process-util b/bash/process/comma-process-util index 011ca4e26..5cec38f29 100755 --- a/bash/process/comma-process-util +++ b/bash/process/comma-process-util @@ -454,9 +454,13 @@ declare -r terminate_comma_execute_and_wait # @description runs a given command as a background task and waits for it to terminate # # Usage: -# comma_execute_and_wait command[,command2[,command3]] [--signals=signal1,signal2,...] \ +# comma_execute_and_wait command[ command2[ command3]] [--signals=signal1,signal2,...] \ # [--max-wait=max_wait] [--process|--group] [--any|--all] \ # [--pid-fifo=] +# Quick example: +# function _hello() { echo "hello $1"; } +# export -f _hello +# comma_execute_and_wait --group "some-command-line-util --some-option" "bash -c _hello world" # # The function executes the given command(s) (non-option argument) in a background process(es) # and waits for its completion (or timeout, see below). diff --git a/bash/test/comma-application-util/comma-application-init/application b/bash/test/comma-application-util/comma-application-init/application new file mode 100755 index 000000000..001411dcf --- /dev/null +++ b/bash/test/comma-application-util/comma-application-init/application @@ -0,0 +1,24 @@ +#!/bin/bash + +source $( type -p comma-application-util ) + +function prologue() { echo "sample application: print parsed command line option values and exit"; } + +function epilogue() { echo "example: ./application some unnamed --mandatory blah stuff"; } + +function options-description() +{ + cat <; default=hello; default value +--flag; flag option +--mandatory,-m=; this is mandatory option +--optional,-o=[]; this is optional option +eof +} + +comma-application-init $@ < <( options-description ) || exit 1 +echo "options/defaulting=$options_defaulting" +echo "options/flag=$options_flag" +echo "options/mandatory=$options_mandatory" +echo "options/optional=$options_optional" +for (( i = 0; i < ${#options_unnamed[@]}; ++i )); do echo "options/unnamed[$i]=${options_unnamed[$i]}"; done diff --git a/bash/test/comma-application-util/comma-application-init/expected b/bash/test/comma-application-util/comma-application-init/expected new file mode 100644 index 000000000..c60f7fcb7 --- /dev/null +++ b/bash/test/comma-application-util/comma-application-init/expected @@ -0,0 +1,61 @@ +help/output/line[0]="" +help/output/line[1]="sample application: print parsed command line option values and exit" +help/output/line[2]="" +help/output/line[3]="usage: application []" +help/output/line[4]="" +help/output/line[5]="options" +help/output/line[6]="--defaulting,-d=; default=hello; default value" +help/output/line[7]="--flag; flag option" +help/output/line[8]="--mandatory,-m=; this is mandatory option" +help/output/line[9]="--optional,-o=[]; this is optional option" +help/output/line[10]="--verbose,-v; output more messages to stderr" +help/output/line[11]="" +help/output/line[12]="example: ./application some unnamed --mandatory blah stuff" +help/status=0 + +options/mandatory[0]/output/line[0]="options/defaulting=hello" +options/mandatory[0]/output/line[1]="options/flag=" +options/mandatory[0]/output/line[2]="options/mandatory=blah" +options/mandatory[0]/output/line[3]="options/optional=" +options/mandatory[0]/status=0 +options/mandatory[1]/output="" +options/mandatory[1]/status=1 + +options/optional[0]/output/line[0]="options/defaulting=hello" +options/optional[0]/output/line[1]="options/flag=" +options/optional[0]/output/line[2]="options/mandatory=blah" +options/optional[0]/output/line[3]="options/optional=" +options/optional[0]/status=0 +options/optional[1]/output/line[0]="options/defaulting=hello" +options/optional[1]/output/line[1]="options/flag=" +options/optional[1]/output/line[2]="options/mandatory=blah" +options/optional[1]/output/line[3]="options/optional=bye" +options/optional[1]/status=0 + +options/flag[0]/output/line[0]="options/defaulting=hello" +options/flag[0]/output/line[1]="options/flag=" +options/flag[0]/output/line[2]="options/mandatory=blah" +options/flag[0]/output/line[3]="options/optional=" +options/flag[0]/status=0 +options/flag[1]/output/line[0]="options/defaulting=hello" +options/flag[1]/output/line[1]="options/flag=1" +options/flag[1]/output/line[2]="options/mandatory=blah" +options/flag[1]/output/line[3]="options/optional=" +options/flag[1]/status=0 + +options/unnamed[0]/output/line[0]="options/defaulting=hello" +options/unnamed[0]/output/line[1]="options/flag=" +options/unnamed[0]/output/line[2]="options/mandatory=blah" +options/unnamed[0]/output/line[3]="options/optional=" +options/unnamed[0]/output/line[4]="options/unnamed[0]=a" +options/unnamed[0]/output/line[5]="options/unnamed[1]=b" +options/unnamed[0]/output/line[6]="options/unnamed[2]=c" +options/unnamed[0]/status=0 +options/unnamed[1]/output/line[0]="options/defaulting=hello" +options/unnamed[1]/output/line[1]="options/flag=1" +options/unnamed[1]/output/line[2]="options/mandatory=blah" +options/unnamed[1]/output/line[3]="options/optional=" +options/unnamed[1]/output/line[4]="options/unnamed[0]=a" +options/unnamed[1]/output/line[5]="options/unnamed[1]=b" +options/unnamed[1]/output/line[6]="options/unnamed[2]=c" +options/unnamed[1]/status=0 diff --git a/bash/test/comma-application-util/comma-application-init/input b/bash/test/comma-application-util/comma-application-init/input new file mode 100644 index 000000000..9ab3eccd0 --- /dev/null +++ b/bash/test/comma-application-util/comma-application-init/input @@ -0,0 +1,9 @@ +help="./application -h 2>&1" +options/mandatory[0]="./application --mandatory blah" +options/mandatory[1]="./application" +options/optional[0]="./application --mandatory blah" +options/optional[1]="./application --mandatory blah --optional bye" +options/flag[0]="./application --mandatory blah" +options/flag[1]="./application --mandatory blah --flag" +options/unnamed[0]="./application a b --mandatory blah c" +options/unnamed[1]="./application a b --mandatory blah --flag c" diff --git a/bash/test/comma-application-util/comma-application-swiss-init/application b/bash/test/comma-application-util/comma-application-swiss-init/application new file mode 100755 index 000000000..bd9db9fb4 --- /dev/null +++ b/bash/test/comma-application-util/comma-application-swiss-init/application @@ -0,0 +1,24 @@ +#!/bin/bash + +source $( type -p comma-application-util ) + +operations=( lol roll ) + +function prologue() { echo "sample application: for a given operation, print parsed command line option values and exit"; } + +function epilogue() { echo "example: ./application lol"; } + +function options-description() { echo "--some-common-option=[]; default=hello; a sample common option"; } + +function lol-description() { echo "laugh out loud"; } +function lol-usage() { echo "specialised usage: ./application lol []"; } +function lol-options() { echo "--how-loud=; default=0; how loud to laugh in dB"; } +function lol-run() { echo "lol/options/how_loud=$options_how_loud"; } + +function roll-description() { echo "roll on the floor"; } +function roll-options() { echo "--screaming; roll on the floor screaming"; } +function roll-run() { echo "roll/options/screaming=$options_screaming"; } + +comma-application-swiss-init $@ < <( options-description ) || exit 1 +echo "options/some_common_option=$options_some_common_option" +comma-application-swiss-run $@ diff --git a/bash/test/comma-application-util/comma-application-swiss-init/expected b/bash/test/comma-application-util/comma-application-swiss-init/expected new file mode 100644 index 000000000..38c190c8b --- /dev/null +++ b/bash/test/comma-application-util/comma-application-swiss-init/expected @@ -0,0 +1,69 @@ +help[0]/output/line[0]="" +help[0]/output/line[1]="sample application: for a given operation, print parsed command line option values and exit" +help[0]/output/line[2]="" +help[0]/output/line[3]="usage: application " +help[0]/output/line[4]="" +help[0]/output/line[5]="common options" +help[0]/output/line[6]="--some-common-option=[]; default=hello; a sample common option" +help[0]/output/line[7]="--verbose,-v; output more messages to stderr" +help[0]/output/line[8]="" +help[0]/output/line[9]="available operations: lol roll" +help[0]/output/line[10]="" +help[0]/output/line[11]="operations" +help[0]/output/line[12]="lol" +help[0]/output/line[13]="laugh out loud" +help[0]/output/line[14]="usage" +help[0]/output/line[15]="specialised usage: ./application lol []" +help[0]/output/line[16]="options" +help[0]/output/line[17]="--how-loud=; default=0; how loud to laugh in dB" +help[0]/output/line[18]="" +help[0]/output/line[19]="roll" +help[0]/output/line[20]="roll on the floor" +help[0]/output/line[21]="usage: application --help []" +help[0]/output/line[22]="options" +help[0]/output/line[23]="--screaming; roll on the floor screaming" +help[0]/output/line[24]="" +help[0]/output/line[25]="example: ./application lol" +help[0]/status=0 + +help[1]/output/line[0]="application lol" +help[1]/output/line[1]="laugh out loud" +help[1]/output/line[2]="usage" +help[1]/output/line[3]="specialised usage: ./application lol []" +help[1]/output/line[4]="common options" +help[1]/output/line[5]="--some-common-option=[]; default=hello; a sample common option" +help[1]/output/line[6]="--verbose,-v; output more messages to stderr" +help[1]/output/line[7]="options" +help[1]/output/line[8]="--how-loud=; default=0; how loud to laugh in dB" +help[1]/status=0 + +help[2]/output/line[0]="application roll" +help[2]/output/line[1]="roll on the floor" +help[2]/output/line[2]="usage: application roll []" +help[2]/output/line[3]="common options" +help[2]/output/line[4]="--some-common-option=[]; default=hello; a sample common option" +help[2]/output/line[5]="--verbose,-v; output more messages to stderr" +help[2]/output/line[6]="options" +help[2]/output/line[7]="--screaming; roll on the floor screaming" +help[2]/status=0 + +help[3]/status=1 + +run/lol[0]/output/line[0]="options/some_common_option=hello" +run/lol[0]/output/line[1]="lol/options/how_loud=0" +run/lol[0]/status=0 + +run/lol[1]/output/line[0]="options/some_common_option=blah" +run/lol[1]/output/line[1]="lol/options/how_loud=120" +run/lol[1]/status=0 + +run/roll[0]/output/line[0]="options/some_common_option=hello" +run/roll[0]/output/line[1]="roll/options/screaming=" +run/roll[0]/status=0 + +run/roll[1]/output/line[0]="options/some_common_option=hello" +run/roll[1]/output/line[1]="roll/options/screaming=1" +run/roll[1]/status=0 + +run/fall[0]/output="" +run/fall[0]/status=1 diff --git a/bash/test/comma-application-util/comma-application-swiss-init/input b/bash/test/comma-application-util/comma-application-swiss-init/input new file mode 100644 index 000000000..2aa5d0237 --- /dev/null +++ b/bash/test/comma-application-util/comma-application-swiss-init/input @@ -0,0 +1,10 @@ +help[0]="./application --help 2>&1" +help[1]="./application lol --help 2>&1" +help[2]="./application roll --help 2>&1" +help[3]="./application fall --help 2>&1" + +run/lol[0]="./application lol" +run/lol[1]="./application lol --some-common-option blah --how-loud=120" +run/roll[0]="./application roll" +run/roll[1]="./application roll --screaming" +run/fall[0]="./application fall" diff --git a/bash/test/comma-application-util/comma-source-relative/a/b/x b/bash/test/comma-application-util/comma-source-relative/a/b/x new file mode 100644 index 000000000..b2b26ad4d --- /dev/null +++ b/bash/test/comma-application-util/comma-source-relative/a/b/x @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "a/b/x/sourced=1" diff --git a/bash/test/comma-application-util/comma-source-relative/a/b/y b/bash/test/comma-application-util/comma-source-relative/a/b/y new file mode 100644 index 000000000..57886fadc --- /dev/null +++ b/bash/test/comma-application-util/comma-source-relative/a/b/y @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "a/b/y/sourced=1" diff --git a/bash/test/comma-application-util/comma-source-relative/a/z b/bash/test/comma-application-util/comma-source-relative/a/z new file mode 100644 index 000000000..c117c82e6 --- /dev/null +++ b/bash/test/comma-application-util/comma-source-relative/a/z @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "a/z/sourced=1" diff --git a/bash/test/comma-application-util/comma-source-relative/expected b/bash/test/comma-application-util/comma-source-relative/expected new file mode 100644 index 000000000..930585d1c --- /dev/null +++ b/bash/test/comma-application-util/comma-source-relative/expected @@ -0,0 +1,13 @@ +once/a/b/x/sourced=1 +multiple/a/b/x/sourced=1 +multiple/a/b/y/sourced=1 +multiple/a/z/sourced=1 +wildcard/a/b/x/sourced=1 +wildcard/a/b/y/sourced=1 +directory/a/b/x/sourced=1 +directory/a/b/y/sourced=1 +directories/a/b/x/sourced=1 +directories/a/b/y/sourced=1 +directories/a/b/x/sourced=1 +directories/a/b/y/sourced=1 +directories/a/z/sourced=1 diff --git a/bash/test/comma-application-util/comma-source-relative/test b/bash/test/comma-application-util/comma-source-relative/test new file mode 100755 index 000000000..5b8a9e95c --- /dev/null +++ b/bash/test/comma-application-util/comma-source-relative/test @@ -0,0 +1,9 @@ +#!/bin/bash + +source ../../../comma-application-util || exit 1 + +comma-source-relative a/b/x | sed 's#^#once/#' +comma-source-relative a/b/x a/b/y a/z | sed 's#^#multiple/#' +comma-source-relative a/b/* | sed 's#^#wildcard/#' +comma-source-relative a/b | sed 's#^#directory/#' +comma-source-relative a/b a | sed 's#^#directories/#' diff --git a/bash/test/comma_status_ok/expected b/bash/test/comma-application-util/comma_status_ok/expected similarity index 100% rename from bash/test/comma_status_ok/expected rename to bash/test/comma-application-util/comma_status_ok/expected diff --git a/bash/test/comma_status_ok/test b/bash/test/comma-application-util/comma_status_ok/test similarity index 100% rename from bash/test/comma_status_ok/test rename to bash/test/comma-application-util/comma_status_ok/test diff --git a/bash/test/comma_background/signature/args/expected b/bash/test/comma_background/signature/args/expected index 4e62e0068..5acb607de 100644 --- a/bash/test/comma_background/signature/args/expected +++ b/bash/test/comma_background/signature/args/expected @@ -1,5 +1,7 @@ #python +from functools import reduce + # iterate through a path of attributes: "obj/data/member/value" def deepgetattr(obj, attr): """Recurses through an attribute chain to get the ultimate value.""" diff --git a/bash/test/comma_background/signature/basic/test b/bash/test/comma_background/signature/basic/test index ad66984bf..f7bad1464 100755 --- a/bash/test/comma_background/signature/basic/test +++ b/bash/test/comma_background/signature/basic/test @@ -19,7 +19,7 @@ echo "clock_ticks_per_second=$ticks" comma_process_exec_and_validate "$fifo" sleep 100 || { echo "$scriptname: fatal system error, wrong background PID" >&2; exit 1; } background_pid=$! -now=$( python -c "import sys; from numpy import int64; a = sys.stdin.readline().split()[0]; print int64(float(a) * $ticks)" < /proc/uptime ) +now=$( python3 -c "import sys; from numpy import int64; a = sys.stdin.readline().split()[0]; print( int64(float(a) * $ticks) )" < /proc/uptime ) echo "time/now=$now" signature=$( comma_process_signature "$background_pid" ) diff --git a/bash/test/comma_name_value_to_options/expected b/bash/test/comma_name_value_to_options/expected index 28464c9e2..b45350421 100644 --- a/bash/test/comma_name_value_to_options/expected +++ b/bash/test/comma_name_value_to_options/expected @@ -6,10 +6,16 @@ basic[4]/text="" basic[5]/text="" basic[6]/text="----hello_world=1 " multi[0]/text="--hello_world=1 --hello=world --hello=world ----hello_world=1 " -co_to_nv[0]/text="debug=|1|@directory=|/tmp|@verbose=|1|@" -co_to_nv[1]/text="debug=|1|@directory=|/tmp|@verbose=|1|@" -co_to_nv[2]/text="debug=|1|@directory=|/tmp|@verbose=|1|@" -co_to_nv[3]/text="debug=|1|@directory=|\|/tmp\||@verbose=|1|@" + +#co_to_nv[0]/text="debug=|1|@directory=|/tmp|@verbose=|1|@" +#co_to_nv[1]/text="debug=|1|@directory=|/tmp|@verbose=|1|@" +#co_to_nv[2]/text="debug=|1|@directory=|/tmp|@verbose=|1|@" +#co_to_nv[3]/text="debug=|1|@directory=|\|/tmp\||@verbose=|1|@" +co_to_nv[0]/text="debug=__quote__1__quote__&directory=__quote__/tmp__quote__&verbose=__quote__1__quote__&" +co_to_nv[1]/text="debug=__quote__1__quote__&directory=__quote__/tmp__quote__&verbose=__quote__1__quote__&" +co_to_nv[2]/text="debug=__quote__1__quote__&directory=__quote__/tmp__quote__&verbose=__quote__1__quote__&" +co_to_nv[3]/text="debug=__quote__1__quote__&directory=__quote____backslash__quote__/tmp__backslash__quote____quote__&verbose=__quote__1__quote__&" + cnv_to_o[0]/text="--debug=1 --directory=/tmp --verbose=1 " cnv_to_o[1]/text="--debug=1 --directory=/tmp --verbose=1 " cnv_to_o[2]/text="--debug=1 --directory=/tmp --verbose=1 " diff --git a/bash/test/comma_name_value_to_options/test b/bash/test/comma_name_value_to_options/test index e8c1f5369..7e549281c 100755 --- a/bash/test/comma_name_value_to_options/test +++ b/bash/test/comma_name_value_to_options/test @@ -58,6 +58,8 @@ echo '--debug; much more debug output --verbose,-v; more output' } +function escape() { sed -e 's#"#__quote__#g' -e 's#\\#__backslash#g' | tr '\n' '&'; } + echo "basic[0]/text=\"$( comma_name_value_to_options <<< 'hello world' )\"" echo "basic[1]/text=\"$( comma_name_value_to_options <<< 'hello_world=1' )\"" echo "basic[2]/text=\"$( comma_name_value_to_options <<< 'hello=world' )\"" @@ -73,14 +75,14 @@ hello="world" hello_world --hello_world=1' | tr '\n' '@' )\"" -echo "co_to_nv[0]/text=\"$( command_description | comma-options-to-name-value --directory "/tmp" --debug --verbose | tr '\n' '@' | tr '"' '|' )\"" -echo "co_to_nv[1]/text=\"$( command_description | comma-options-to-name-value --directory="/tmp" --debug --verbose | tr '\n' '@' | tr '"' '|' )\"" -echo "co_to_nv[2]/text=\"$( command_description | comma-options-to-name-value "--directory=/tmp" --debug --verbose | tr '\n' '@' | tr '"' '|' )\"" -echo "co_to_nv[3]/text=\"$( command_description | comma-options-to-name-value '--directory="/tmp"' --debug --verbose | tr '\n' '@' | tr '"' '|' )\"" +echo "co_to_nv[0]/text=\"$( command_description | comma-options-to-name-value --directory "/tmp" --debug --verbose | escape )\"" +echo "co_to_nv[1]/text=\"$( command_description | comma-options-to-name-value --directory="/tmp" --debug --verbose | escape )\"" +echo "co_to_nv[2]/text=\"$( command_description | comma-options-to-name-value "--directory=/tmp" --debug --verbose | escape )\"" +echo "co_to_nv[3]/text=\"$( command_description | comma-options-to-name-value '--directory="/tmp"' --debug --verbose | escape )\"" -echo "cnv_to_o[0]/text=\"$( command_description | comma-options-to-name-value --directory "/tmp" --debug --verbose | comma_name_value_to_options | tr '"' '|' )\"" -echo "cnv_to_o[1]/text=\"$( command_description | comma-options-to-name-value --directory="/tmp" --debug --verbose | comma_name_value_to_options | tr '"' '|' )\"" -echo "cnv_to_o[2]/text=\"$( command_description | comma-options-to-name-value "--directory=/tmp" --debug --verbose | comma_name_value_to_options | tr '"' '|' )\"" +echo "cnv_to_o[0]/text=\"$( command_description | comma-options-to-name-value --directory "/tmp" --debug --verbose | comma_name_value_to_options | tr '"' '|' )\"" +echo "cnv_to_o[1]/text=\"$( command_description | comma-options-to-name-value --directory="/tmp" --debug --verbose | comma_name_value_to_options | tr '"' '|' )\"" +echo "cnv_to_o[2]/text=\"$( command_description | comma-options-to-name-value "--directory=/tmp" --debug --verbose | comma_name_value_to_options | tr '"' '|' )\"" comma_name_value_to_options <<< 'directory=/tmp verbose="1" diff --git a/bash/test/comma_name_value_util/comma_path_value_mangle/expected b/bash/test/comma_name_value_util/comma_path_value_mangle/expected index 7c500cc85..59c377d89 100644 --- a/bash/test/comma_name_value_util/comma_path_value_mangle/expected +++ b/bash/test/comma_name_value_util/comma_path_value_mangle/expected @@ -4,3 +4,14 @@ basic/var_b='22' no_newline_at_end/var_a=basic/var_a no_newline_at_end/var_b=basic/var_b +path/a_b_c='1' +dashes/a_b_c='1' +path_with_dashes/a_b_c='1' +subscript/a_b_0='1' +subscript_followed_by_path/a_b_0_c='1' +a="123" +b="123" +c="123" + +skip_empty_lines_n='a' +skip_empty_lines_v='b' diff --git a/bash/test/comma_name_value_util/comma_path_value_mangle/test b/bash/test/comma_name_value_util/comma_path_value_mangle/test index e619aa266..be4561030 100755 --- a/bash/test/comma_name_value_util/comma_path_value_mangle/test +++ b/bash/test/comma_name_value_util/comma_path_value_mangle/test @@ -7,6 +7,18 @@ input="var_a=21 var_b=22" echo "$input" | comma_path_value_mangle | sed 's|^|basic/|' - echo -n "$input" | comma_path_value_mangle | sed 's|^|no_newline_at_end/|' +echo "a/b/c=1" | comma_path_value_mangle | sed 's|^|path/|' +echo "a-b-c=1" | comma_path_value_mangle | sed 's|^|dashes/|' +echo "a/b-c=1" | comma_path_value_mangle | sed 's|^|path_with_dashes/|' +echo "a/b[0]=1" | comma_path_value_mangle | sed 's|^|subscript/|' +echo "a/b[0]/c=1" | comma_path_value_mangle | sed 's|^|subscript_followed_by_path/|' +( echo a; echo b; echo c ) | comma_path_value_mangle '_' 123 + +cat < +#include +#include +#include "../../application/command_line_options.h" +#include "../../base/exception.h" +#include "../../csv/stream.h" +#include "../../csv/traits.h" +#include "../../name_value/parser.h" +#include "../multidimensional/array.h" +#include "../multidimensional/traits.h" + +void usage( bool verbose ) +{ + std::cerr << "operations on a multidimensional lookup table" << std::endl; + std::cerr << "lookup tables with up to 4-dimension" << std::endl; + std::cerr << "values are currently supported; if you need more, just ask" << std::endl; + std::cerr << std::endl; + std::cerr << "usage: cat input.csv | math-lookup [[;]] " << std::endl; + std::cerr << std::endl; + std::cerr << "operations" << std::endl; + std::cerr << " index: todo: output index for a given input" << std::endl; + std::cerr << " interpolate: output interpolated value for the given input" << std::endl; + std::cerr << " nearest: todo: output table element index and value nearest to the given input" << std::endl; + std::cerr << " query: output table element index and value for the given input" << std::endl; + std::cerr << std::endl; + std::cerr << "options" << std::endl; + std::cerr << " --origin,-o=; e.g: --origin=0,1,2,3" << std::endl; + std::cerr << " --resolution,-r=; e.g: --resolution=0.5,3,2,3" << std::endl; + std::cerr << " --shape=; e.g: --shape=3,2,5,3, same as in numpy" << std::endl; + std::cerr << " i.e. shape[0] is the slowest-changing" << std::endl; + std::cerr << " i.e. expected lookup table memory layout" << std::endl; + std::cerr << " is rows first" << std::endl; + std::cerr << std::endl; + std::cerr << "input/output options" << std::endl; + std::cerr << " --input-fields; todo: print input fields for an operation to stdout and exit" << std::endl; + std::cerr << " --output-fields; todo: print output fields for an operation to stdout and exit" << std::endl; + std::cerr << " --output-format; todo: print output format for an operation to stdout and exit" << std::endl; + std::cerr << " --permissive; discard inputs outside lookup table" << std::endl; + std::cerr << std::endl; + std::cerr << "csv options" << std::endl; + std::cerr << comma::csv::options::usage( verbose ) << std::endl; + std::cerr << std::endl; + exit( 0 ); +} + +// todo +// - 1-dimensional: fix +// - nearest: fix +// - regression test: basics + +static bool permissive{false}; +static bool verbose{false}; + +// template< typename T, std::size_t D, typename S > +// std::array< T, D >& operator*=( std::array< T, D >& lhs, const S& rhs ) { for( unsigned int i = 0; i < D; ++i ) { lhs[i] *= rhs; } return lhs; } // quick and dirty; let compiler optimize + +// template< typename T, std::size_t D, typename S > +// std::array< T, D > operator*( const std::array< T, D >& lhs, const S& rhs ) { auto r = lhs; r *= rhs; return r; } + +// template< typename T, std::size_t D > +// std::array< T, D >& operator+=( std::array< T, D >& lhs, const std::array< T, D >& rhs ) { for( unsigned int i = 0; i < D; ++i ) { lhs[i] += rhs[i]; } return lhs; } // quick and dirty; let compiler optimize + +// template< typename T, std::size_t D > +// std::array< T, D > operator+( const std::array< T, D >& lhs, const std::array< T, D >& rhs ) { auto r = lhs; r += rhs; return r; } + +template< typename T, std::size_t D > +std::ostream& operator<<( std::ostream& os, const std::array< T, D >& rhs ) { std::string d; for( unsigned int i = 0; i < D; ++i ) { os << d << rhs[i]; d = ","; } return os; } + +namespace comma { namespace applications { namespace lookup { namespace operations { + +template< typename T, std::size_t D > +struct _array { std::array< T, D > point; }; + +template < typename T, std::size_t D, std::size_t E > +struct lut +{ + typedef std::array< double, D > point_t; + typedef comma::containers::multidimensional::index< D > index_t; // typedef std::array< std::size_t, D > index_t; + typedef std::array< T, E > value_t; + typedef comma::containers::multidimensional::grid< value_t, D, point_t > grid_t; + typedef _array< double, D > input_t; + typedef _array< T, D > output_t; + + static grid_t& load( grid_t& g, const comma::csv::options& csv ) + { + std::ifstream ifs( csv.filename, std::ios::binary ); + COMMA_ASSERT_BRIEF( ifs.is_open(), "lookup table: failed to open '" << csv.filename << "'" ); + std::size_t size = g.data().size() * sizeof( T ) * E; + ifs.read( reinterpret_cast< char* >( &g.data()[0] ), size ); + COMMA_ASSERT_BRIEF( ifs.gcount() > 0, "lookup table: failed to read from '" << csv.filename << "'" ); + COMMA_ASSERT_BRIEF( std::size_t( ifs.gcount() ) == size, "lookup table: on file '" << csv.filename << "': expected " << size << " bytes; got: " << ifs.gcount() ); + return g; + } + + static std::pair< index_t, value_t > interpolate( const grid_t& g, const point_t& p ) + { + std::pair< index_t, value_t > r; + r.first = g.index_of( p ); + r.second = g.interpolated( p ); + return r; + } + + static std::pair< index_t, value_t > nearest( const grid_t& g, const point_t& p ) + { + std::pair< index_t, value_t > r; + r.first = g.nearest_to( p ); + r.second = g[r.first]; + return r; + } + + static std::pair< index_t, value_t > query( const grid_t& g, const point_t& p ) + { + std::pair< index_t, value_t > r; + r.first = g.index_of( p ); + r.second = g[r.first]; + return r; + } + + static int run( const std::string& operation + , const comma::csv::options& csv + , const comma::csv::options& lut_csv + , const std::vector< double >& origin + , const std::vector< double >& resolution + , const std::vector< std::size_t >& shape ) + { + std::pair< index_t, value_t > ( *f )( const grid_t&, const point_t& ); + if( operation == "interpolate" ) { f = lut< T, D, E >::interpolate; } + else if( operation == "nearest" ) { COMMA_THROW_BRIEF( comma::exception, "nearest: todo" ); } //else if( operation == "nearest" ) { f = lut< T, D, E >::nearest; } + else if( operation == "query" ) { f = lut< T, D, E >::query; } + else { COMMA_THROW_BRIEF( comma::exception, "expected operation; got: '" << operation << "'" ); } + point_t o, r; + index_t s; + std::memcpy( &o[0], &origin[0], D * sizeof( double ) ); // quick and dirty + std::memcpy( &r[0], &resolution[0], D * sizeof( double ) ); // quick and dirty + std::memcpy( &s[0], &shape[0], D * sizeof( std::size_t ) ); // quick and dirty + grid_t grid( o, r, s ); + load( grid, lut_csv ); + input_t zero; + std::memset( &zero.point[0], 0, zero.point.size() * sizeof( T ) ); + comma::csv::input_stream< input_t > istream( std::cin, csv, zero ); + comma::csv::output_stream< std::pair< index_t, value_t > > ostream( std::cout, csv.binary() ); + auto tied = comma::csv::make_tied( istream, ostream ); + while( istream.ready() || std::cin.good() ) + { + const auto& p = istream.read(); + if( !p ) { break; } + if( !grid.has( p->point ) ) + { + if( permissive ) { comma::saymore() << "discarded input outside grid: " << p->point << std::endl; continue; } + comma::say() << "input outside grid: " << p->point << "; use --permissive to discard" << std::endl; + return 1; + } + tied.append( f( grid, p->point ) ); + if( csv.flush ) { std::cout.flush(); } + } + return 0; + } +}; + +template < typename T, std::size_t D > static int run_with_dim( const std::string& operation + , const comma::csv::options& csv + , const comma::csv::options& lut_csv + , const std::vector< double >& origin + , const std::vector< double >& resolution + , const std::vector< std::size_t >& shape ) +{ + switch( lut_csv.format().count() ) + { + case 1: return lut< T, D, 1 >::run( operation, csv, lut_csv, origin, resolution, shape ); + case 2: return lut< T, D, 2 >::run( operation, csv, lut_csv, origin, resolution, shape ); + case 3: return lut< T, D, 3 >::run( operation, csv, lut_csv, origin, resolution, shape ); + case 4: return lut< T, D, 4 >::run( operation, csv, lut_csv, origin, resolution, shape ); + default: COMMA_THROW( comma::exception, "up to 4-dimensional lookup table values currently supported; got: " << lut_csv.format().count() << " dimensions in " << lut_csv.format().string() ); + } + return 1; +} + +template < typename T > static int run_as( const std::string& operation + , const comma::csv::options& csv + , const comma::csv::options& lut_csv + , const std::vector< double >& origin + , const std::vector< double >& resolution + , const std::vector< std::size_t >& shape ) +{ + switch( origin.size() ) + { + // todo! case 1: return run_with_dim< T, 1 >( operation, csv, lut_csv, origin, resolution, shape ); + case 2: return run_with_dim< T, 2 >( operation, csv, lut_csv, origin, resolution, shape ); + case 3: return run_with_dim< T, 3 >( operation, csv, lut_csv, origin, resolution, shape ); + case 4: return run_with_dim< T, 4 >( operation, csv, lut_csv, origin, resolution, shape ); + default: COMMA_THROW( comma::exception, "up to 4-dimensional lookup tables currently supported; got: " << origin.size() << " dimensions" ); + } + return 1; +} + +} } } } // namespace comma { namespace applications { namespace lookup { namespace operations { + +namespace comma { namespace visiting { + +template < typename T, std::size_t D > struct traits< comma::applications::lookup::operations::_array< T, D > > +{ + template < typename Key, class Visitor > static void visit( const Key&, comma::applications::lookup::operations::_array< T, D >& p, Visitor& v ) { v.apply( "point", p.point ); } + template < typename Key, class Visitor > static void visit( const Key&, const comma::applications::lookup::operations::_array< T, D >& p, Visitor& v ) { v.apply( "point", p.point ); } +}; + +} } // namespace comma { namespace visiting { + +namespace comma { namespace applications { namespace lookup { namespace operations { + +static int run( const comma::command_line_options& options, const csv::options& csv, const std::vector< std::string >& unnamed ) +{ + COMMA_ASSERT_BRIEF( unnamed.size() > 1, "please specify lookup table file as: math-lookup " ); + auto lut_csv = comma::name_value::parser( "filename" ).get< comma::csv::options >( unnamed[1] ); + COMMA_ASSERT_BRIEF( lut_csv.binary(), "lookup table: on file '" << lut_csv.filename << "': only binary files are currently supported, e.g: 'lut.bin;binary=3f'" ); + const auto& origin = comma::split_as< double >( options.value< std::string >( "--origin,-o" ), ',' ); + const auto& resolution = comma::split_as< double >( options.value< std::string >( "--resolution,-r" ), ',' ); + const auto& shape = comma::split_as< std::size_t >( options.value< std::string >( "--shape" ), ',' ); + COMMA_ASSERT_BRIEF( origin.size() == resolution.size(), "expected --origin and --resolution of the same dimensions; got: " << origin.size() << " and " << resolution.size() ); + COMMA_ASSERT_BRIEF( origin.size() == shape.size(), "expected --origin and --shape of the same dimensions; got: " << origin.size() << " and " << shape.size() ); + switch( lut_csv.format().elements()[0].type ) // todo! quick and dirty + { + case comma::csv::format::float_t: return comma::applications::lookup::operations::run_as< float >( unnamed[0], csv, lut_csv, origin, resolution, shape ); + case comma::csv::format::double_t: return comma::applications::lookup::operations::run_as< double >( unnamed[0], csv, lut_csv, origin, resolution, shape ); + default: COMMA_THROW( comma::exception, "only float and double as lookup table values are supported; got: '" << unnamed[1] << "'" ); + } + return 1; +} + +} } } } // namespace comma { namespace applications { namespace lookup { namespace operations { + +int main( int ac, char** av ) +{ + try + { + comma::command_line_options options( ac, av, usage ); + comma::csv::options csv( options ); + const auto& unnamed = options.unnamed( "--flush,--permissive,--verbose,-v", "-.*" ); + if( unnamed.empty() ) { comma::say() << "please specify operation" << std::endl; return 1; } + permissive = options.exists( "--permissive" ); + verbose = options.exists( "--verbose,-v" ); + return comma::applications::lookup::operations::run( options, csv, unnamed ); + } + catch( std::exception& ex ) { comma::say() << "caught exception: " << ex.what() << std::endl; } + catch( ... ) { comma::say() << "caught unknown exception" << std::endl; } + return 1; +} diff --git a/containers/cached.h b/containers/cached.h new file mode 100644 index 000000000..0ec819807 --- /dev/null +++ b/containers/cached.h @@ -0,0 +1,70 @@ +// Copyright (c) 2024 Vsevolod Vlaskine +// All Rights Reserved + +#include +#include +#include +namespace comma { + +template < typename T, typename K, typename Hash = std::hash< K > > +class cached +{ + public: + cached( unsigned int max_size = 0 ): _size( max_size ) {} + + template < typename... Args > T& get( Args... args ); + + template < typename... Args > const T& get( Args... args ) const; + + template < typename... Args > auto operator()( Args... args ) { return get( args... )( args... ); } + + template < typename... Args > auto operator()( Args... args ) const { return get( args... )( args... ); } + + void clear() { _values.clear(); } + + void pop( unsigned int size = 1 ); + + const std::unordered_map< K, std::unique_ptr< T >, Hash >& values() const { return _values; } + + protected: + mutable std::unordered_map< K, std::unique_ptr< T >, Hash > _values; // todo! use proper move semantics instead of unique_ptr + mutable std::deque< K > _keys; + unsigned int _size{0}; +}; + +template < typename T, typename K, typename Hash > +template < typename... Args > +T& cached< T, K, Hash >::get( Args... args ) +{ + K k{ args... }; + auto i = _values.find( k ); + if( i != _values.end() ) { return *( i->second ); } + if( _size > 0 && _values.size() == _size ) { pop(); } + _keys.emplace_back( k ); + return *( _values.emplace( std::make_pair( k, std::make_unique< T >( k ) ) ).first->second ); +} + +template < typename T, typename K, typename Hash > +template < typename... Args > +const T& cached< T, K, Hash >::get( Args... args ) const +{ + K k{ args... }; + auto i = _values.find( k ); + if( i != _values.end() ) { return *( i->second ); } + if( _size > 0 && _values.size() == _size ) { pop(); } + _keys.emplace_back( k ); + return *( _values.emplace( std::make_pair( k, std::make_unique< T >( k ) ) ).first->second ); +} + +template < typename T, typename K, typename Hash > +inline void cached< T, K, Hash >::pop( unsigned int size ) +{ + for( unsigned int i = 0; i < size; ++i ) + { + if( _keys.empty() ) { return; } + _values.erase( _keys.front() ); + _keys.pop_front(); + } +} + +} // namespace comma { diff --git a/containers/cyclic_buffer.h b/containers/cyclic_buffer.h index fdbe97691..bfdbc45a9 100644 --- a/containers/cyclic_buffer.h +++ b/containers/cyclic_buffer.h @@ -1,37 +1,8 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine -#ifndef COMMA_CONTAINERS_CYCLIC_BUFFER_H_ -#define COMMA_CONTAINERS_CYCLIC_BUFFER_H_ +#pragma once #include #include @@ -50,49 +21,42 @@ template < typename T > class cyclic_buffer { public: - /// constructor cyclic_buffer( std::size_t size, const T& t = T() ); - /// copy constructor cyclic_buffer( const cyclic_buffer& rhs ) { operator=( rhs ); } - /// assignment const cyclic_buffer& operator=( const cyclic_buffer& rhs ); - /// return front T& front(); - /// return front const T& front() const; - /// return back T& back(); - /// return back - const T& back() const; + + std::size_t front_index() const { return begin_(); } + + std::size_t end_index() const { return end_(); } - /// push a new element at the end of the list - void push( const T& t ); + void push( const T& t, bool force = false ); - /// push a new element at the end of the list template < typename Iterator > - void push( Iterator begin, Iterator end ); + void push( Iterator begin, Iterator end, bool force = false ); - /// pop the new element at the front of the list void pop( std::size_t n = 1 ); - /// return current size std::size_t size() const; - /// return capacity std::size_t capacity() const; - /// return true, if empty bool empty() const; - /// clear void clear(); + + const std::vector< T >& data() const { return vector_; } + + std::vector< T >& data() { return vector_; } protected: std::vector< T > vector_; @@ -173,19 +137,27 @@ template < typename T > inline std::size_t cyclic_buffer< T >::capacity() const { return vector_.size(); } template < typename T > -inline void cyclic_buffer< T >::push( const T& t ) +inline void cyclic_buffer< T >::push( const T& t, bool force ) { - if( size() == vector_.size() ) { COMMA_THROW( comma::exception, "full" ); } - vector_[ end_() ] = t; + if( size() == vector_.size() ) + { + if( !force ) { COMMA_THROW( comma::exception, "full" ); } + vector_[ begin_() ] = t; + ++begin_; + } + else + { + vector_[ end_() ] = t; + } ++end_; empty_ = false; } template < typename T > template < typename Iterator > -inline void cyclic_buffer< T >::push( Iterator begin, Iterator end ) +inline void cyclic_buffer< T >::push( Iterator begin, Iterator end, bool force ) { - for( Iterator it = begin; it != end; ++it ) { push( *it ); } + for( Iterator it = begin; it != end; ++it ) { push( *it, force ); } } template < typename T > @@ -244,5 +216,3 @@ inline const T& fixed_cyclic_buffer< T, S >::operator[]( std::size_t i ) const } } // namespace comma { - -#endif // COMMA_CONTAINERS_CYCLIC_BUFFER_H_ diff --git a/containers/enums.h b/containers/enums.h new file mode 100644 index 000000000..b1b1fb572 --- /dev/null +++ b/containers/enums.h @@ -0,0 +1,53 @@ +// Copyright (c) 2024 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +#include +#include +#include +#include "find.h" + +namespace comma { namespace enums { + +/// trivial convenience wrapper; works only for enums with sequential values +template < typename Enum, typename K = std::string > +std::map< K, Enum > as_map( const std::vector< K >& keys, unsigned int begin = 0 ); + +/// trivial convenience wrapper; works only for enums with sequential values +template < typename Enum, typename K = std::string > +std::map< Enum, K > as_key_map( const std::vector< K >& keys, unsigned int begin = 0 ); + +/// trivial convenience wrapper +template < typename Enum, typename K = std::string > +Enum find( const K& k, const std::vector< K >& keys, unsigned int begin = 0 ); + +// todo: visiting traits +template < typename Enum, typename Names > +struct named: public Enum, Names +{ + const std::string& name() { return this->names()[static_cast< unsigned int >( *this )]; } +}; + + +template < typename Enum, typename K > +inline std::map< K, Enum > as_map( const std::vector< K >& keys, unsigned int begin ) +{ + std::map< K, Enum > m; + for( unsigned int i{0}, j{begin}; i < keys.size(); ++i, ++j ) { m[keys[i]] = static_cast< Enum >( j ); } + return m; +} + +template < typename Enum, typename K > +inline std::map< Enum, K > as_key_map( const std::vector< K >& keys, unsigned int begin ) +{ + std::map< Enum, K > m; + for( unsigned int i{0}, j{begin}; i < keys.size(); ++i, ++j ) { m[static_cast< Enum >( j )] = keys[i]; } + return m; +} + +template < typename Enum, typename K > +Enum find( const K& k, const std::vector< K >& keys, unsigned int begin ) { return containers::find_or_throw< K, Enum >( as_map< Enum, K >( keys, begin ), k ); } + +} } // namespace comma { namespace enums { diff --git a/containers/find.h b/containers/find.h new file mode 100644 index 000000000..76e7b58a6 --- /dev/null +++ b/containers/find.h @@ -0,0 +1,52 @@ +// Copyright (c) 2024 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +#include +#include +#include "../base/exception.h" + +namespace comma { namespace containers { + +/// trivial convenience wrapper: find element, if not found, throw exception +template < typename K, typename V > const V& find_or_throw( const std::map< K, V >& m, const K& k, const std::string& message = std::string() ); + +/// trivial convenience wrapper: find element, if not found, throw exception +template < typename K, typename V > V& find_or_throw( std::map< K, V >& m, const K& k, const std::string& message = std::string() ); + +/// trivial convenience wrapper: find element, if not found, throw exception +template < typename K, typename V > const V& find_or_throw( const std::unordered_map< K, V >& m, const K& k, const std::string& message = std::string() ); + +/// trivial convenience wrapper: find element, if not found, throw exception +template < typename K, typename V > V& find_or_throw( std::unordered_map< K, V >& m, const K& k, const std::string& message = std::string() ); + + +namespace impl { + +template < typename M, typename K > inline const auto& find_or_throw( const M& m, const K& k, const std::string& message ) +{ + auto it = m.find( k ); + COMMA_ASSERT( it != m.end(), ( message.empty() ? std::string() : ( message + ": " ) ) << "k '" << k << "' not found" ); + return it->second; +} + +template < typename M, typename K > inline auto& find_or_throw( M& m, const K& k, const std::string& message ) +{ + auto it = m.find( k ); + COMMA_ASSERT( it != m.end(), ( message.empty() ? std::string() : ( message + ": " ) ) << "k '" << k << "' not found" ); + return it->second; +} + +} // namespace impl { + +template < typename K, typename V > inline const V& find_or_throw( const std::map< K, V >& m, const K& k, const std::string& message ) { return impl::find_or_throw( m, k, message ); } + +template < typename K, typename V > inline V& find_or_throw( std::map< K, V >& m, const K& k, const std::string& message ) { return impl::find_or_throw( m, k, message ); } + +template < typename K, typename V > inline const V& find_or_throw( const std::unordered_map< K, V >& m, const K& k, const std::string& message ) { return impl::find_or_throw( m, k, message ); } + +template < typename K, typename V > inline V& find_or_throw( std::unordered_map< K, V >& m, const K& k, const std::string& message ) { return impl::find_or_throw( m, k, message ); } + +} } // namespace comma { namespace containers { diff --git a/containers/multidimensional/array.h b/containers/multidimensional/array.h new file mode 100644 index 000000000..5d08c17d4 --- /dev/null +++ b/containers/multidimensional/array.h @@ -0,0 +1,319 @@ +// Copyright (c) 2023 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +#include +#include +#include "../../base/types.h" +#include "array_traits.h" +#include "index.h" + +namespace comma { namespace containers { namespace multidimensional { + +template < typename V, unsigned int D > +class slice +{ + public: + typedef multidimensional::index< D > index_type; // typedef std::array< std::size_t, D > index_type; // todo: typedef multidimensional::index< D > index_type; + + typedef V value_type; + + static const unsigned int dimensions{D}; + + slice( const index_type& shape, V* data ): _shape( shape ), _size( _product( _shape ) ), _data( data ) {} + + slice& operator=( const slice& rhs ) = default; + + V& operator[]( const index_type& i ) { return _data[ _index( i ) ]; } + + const V& operator[]( const index_type& i ) const { return _data[ _index( i ) ]; } + + template < unsigned int I > + slice< V, D - I > at( const multidimensional::index< I >& i ); + + template < unsigned int I > + const slice< V, D - I > at( const multidimensional::index< I >& i ) const; + + slice< V, D - 1 > at( std::size_t i ) { return at< 1 >( multidimensional::index< 1 >{i} ); } + + const slice< V, D - 1 > at( std::size_t i ) const { return at< 1 >( multidimensional::index< 1 >{i} ); } + + V* data() { return _data; } + + const V* data() const { return _data; } + + const index_type& shape() const { return _shape; } + + class const_iterator; + + class iterator + { + public: + iterator() = default; + V& operator*() { return *_it; } + const V& operator*() const { return *_it; } + iterator& operator++() { ++_it; ++_i; return *this; } + index_type index() const; + bool operator==( const iterator& rhs ) const { return _it == rhs._it; } + bool operator==( const const_iterator& rhs ) const { return _it == rhs._it; } + bool operator!=( const iterator& rhs ) const { return !operator==( rhs ); } + bool operator!=( const const_iterator& rhs ) const { return !operator==( rhs ); } + + private: + friend class slice< V, D >; + std::size_t _i{0}; + V* _it{nullptr}; + index_type _shape; + iterator( std::size_t i, V* it, const index_type& shape ): _i( i ), _it( it ), _shape( shape ) {} + }; + + class const_iterator + { + public: + const_iterator() = default; + const V& operator*() const { return *_it; } + const_iterator& operator++() { ++_it; ++_i; return *this; } + index_type index() const; + bool operator==( const iterator& rhs ) const { return _it == rhs._it; } + bool operator==( const const_iterator& rhs ) const { return _it == rhs._it; } + bool operator!=( const iterator& rhs ) const { return !operator==( rhs ); } + bool operator!=( const const_iterator& rhs ) const { return !operator==( rhs ); } + + private: + friend class slice< V, D >; + std::size_t _i{0}; + const V* _it{nullptr}; + index_type _shape; + const_iterator( std::size_t i, V* it, const index_type& shape ): _i( i ), _it( it ), _shape( shape ) {} + }; + + iterator begin() { return iterator( 0, _data, _shape ); } + + const_iterator begin() const { return const_iterator( 0, _data, _shape ); } + + iterator end() { return iterator( _size, _data + _size, _shape ); } + + const_iterator end() const { return const_iterator( _size, _data + _size, _shape ); } + + std::size_t absolute_index( const index_type& i ) const { return _index( i ); } + + protected: + index_type _shape; + std::size_t _size; + V* _data; + std::size_t _index( const index_type& i ) const; + static std::size_t _product( const index_type& i ); +}; + + +template < typename V, unsigned int D, typename S = std::vector< V > > +class array +{ + public: + typedef slice< V, D > slice_type; + + typedef typename slice_type::index_type index_type; + + typedef V value_type; + + typedef S storage_type; + + const unsigned int dimensions{D}; + + array( const index_type& shape, const V& default_value = V() ); + + V& operator[]( const index_type& i ) { return _slice[i]; } // V& operator[]( const index_type& i ) { return _data[ absolute_index( i ) ]; } //{ return _slice[i]; } + + const V& operator[]( const index_type& i ) const { return _slice[i]; } // const V& operator[]( const index_type& i ) const { return _data[ absolute_index( i ) ]; } //{ return _slice[i]; } + + template < unsigned int I > + multidimensional::slice< V, D - I > at( const multidimensional::index< I >& i ) { return _slice.template at< I >( i ); } // todo! + + template < unsigned int I > + const multidimensional::slice< V, D - I > at( const multidimensional::index< I >& i ) const { return _slice.template at< I >( i ); } // todo! + + multidimensional::slice< V, D - 1 > at( std::size_t i ) { return _slice.at( i ); } + + const multidimensional::slice< V, D - 1 > at( std::size_t i ) const { return _slice.at( i ); } + + storage_type& data() { return _data; } + + const storage_type& data() const { return _data; } + + const index_type& shape() const { return _slice.shape(); } + + std::size_t absolute_index( const index_type& i ) const { return _slice.absolute_index( i ); } + + typedef typename slice_type::iterator iterator; + + typedef typename slice_type::const_iterator const_iterator; + + iterator begin() { return _slice.begin(); } + + const_iterator begin() const { return _slice.begin(); } + + iterator end() { return _slice.end(); } + + const_iterator end() const { return _slice.end(); } + + protected: + storage_type _data; + slice_type _slice; +}; + +template < typename V, unsigned int D, typename P = std::array< double, D >, typename Traits = impl::operations< D >, typename S = std::vector< V > > +class grid: public array< V, D, S > +{ + public: + typedef P point_type; + + typedef array< V, D, S > base_type; + + typedef typename base_type::index_type index_type; + + typedef typename base_type::value_type value_type; + + grid( const P& origin, const P& resolution, const index_type& shape, const V& default_value = V() ): base_type( shape, default_value ), _origin( origin ), _resolution( resolution ) {} + + index_type index_of( const point_type& point ) const { return Traits::template index_of< P, index_type >( point, _origin, _resolution ); } + + point_type lower_bound( const point_type& point ) const { return Traits::add( _origin + Traits::multiply( _resolution, index_of( point ) ) ); } + + V& operator()( const point_type& p ) { return this->operator[]( index_of( p ) ); } + + const V& operator()( const point_type& p ) const { return this->operator[]( index_of( p ) ); } + + const point_type& origin() const { return _origin; } + + const point_type& resolution() const { return _resolution; } + + V interpolated( const point_type& point ) const; // todo: flag/enum or alike for different interpolation types; currently linear only + + index_type nearest_to( const point_type& point ) const; + + bool has( const point_type& point ) const; + + private: + point_type _origin; + point_type _resolution; +}; + +namespace impl { + +template < unsigned int D, unsigned int I = D > +struct index_traits +{ + typedef comma::containers::multidimensional::index< D > index_type; // typedef std::array< std::size_t, D > index_type; + static unsigned int value( const index_type& i, const index_type& shape ) { return i[ I - 1 ] + index_traits< D, I - 1 >::value( i, shape ) * shape[ I - 1 ]; } + static void value( std::size_t j, index_type& i, const index_type& shape ) { i[ I - 1 ] = j % shape[ I - 1 ]; index_traits< D, I - 1 >::value( j / shape[ I - 1 ], i, shape ); } + static index_type value( std::size_t j, const index_type& shape ) { index_type i; value( j, i, shape ); return i; } + static std::size_t product( const index_type& i ) { return i[ I - 1 ] * index_traits< D, I - 1 >::product( i ); } + template < unsigned int J > + static std::pair< comma::containers::multidimensional::index< J >, comma::containers::multidimensional::index< D - J > > split( const index_type& i ) // todo: use metaprogramming, kinda same as product + { + std::pair< comma::containers::multidimensional::index< J >, comma::containers::multidimensional::index< D - J > > p; + unsigned int k = 0; + for( unsigned int n = 0; n < J; ++n, ++k ) { p.first[n] = i[k]; } + for( unsigned int n = 0; n < D - J; ++n, ++k ) { p.second[n] = i[k]; } + return p; + } +}; + +template < unsigned int D > +struct index_traits< D, 1 > +{ + typedef comma::containers::multidimensional::index< D > index_type; // typedef std::array< std::size_t, D > index_type; + static unsigned int value( const index_type& i, const index_type& ) { return i[0]; } + static void value( std::size_t j, index_type& i, const index_type& size ) { i[0] = j; } + static std::size_t product( const index_type& i ) { return i[0]; } +}; + +} // namespace impl { + +template < typename V, unsigned int D > +inline std::size_t slice< V, D >::_index( const typename slice< V, D >::index_type& i ) const { return impl::index_traits< D >::value( i, _shape ); } + +template < typename V, unsigned int D > +inline std::size_t slice< V, D >::_product( const typename slice< V, D >::index_type& i ) { return impl::index_traits< D >::product( i ); } + +template < typename V, unsigned int D > +inline typename slice< V, D >::index_type slice< V, D >::iterator::index() const { return impl::index_traits< D >::value( _i, _shape ); } + +template < typename V, unsigned int D > +inline typename slice< V, D >::index_type slice< V, D >::const_iterator::index() const { return impl::index_traits< D >::value( _i, _shape ); } + +template < typename V, unsigned int D > +template < unsigned int I > +inline slice< V, D - I > slice< V, D >::at( const multidimensional::index< I >& i ) +{ + auto s = impl::index_traits< D >::template split< I >( _shape ); + return slice< V, D - I >( s.second, _data + impl::index_traits< I >::value( i, s.first ) * impl::index_traits< D - I >::product( s.second ) ); +} + +template < typename V, unsigned int D > +template < unsigned int I > +inline const slice< V, D - I > slice< V, D >::at( const multidimensional::index< I >& i ) const +{ + auto s = impl::index_traits< D >::template split< I >( _shape ); + return slice< V, D - I >( s.second, _data + impl::index_traits< I >::value( i, s.first ) * impl::index_traits< D - I >::product( s.second ) ); +} + +template < typename V, unsigned int D, typename S > +inline array< V, D, S >::array( const typename array< V, D, S >::index_type& shape, const V& default_value ): _data( impl::index_traits< D >::product( shape ), default_value ), _slice( shape, &_data[0] ) {} + +template < typename V, unsigned int D, typename P, typename Traits, typename S > +V grid< V, D, P, Traits, S >::interpolated( const P& point ) const +{ + const index_type i = index_of( point ); + P element_origin = Traits::add( Traits::vmultiply( _resolution, i ), _origin ); + const auto& weights = Traits::interpolation::linear::weights( point, element_origin, _resolution ); + const auto& neighbours = impl::neighbours< index_type, D >; + V v = this->operator[]( i ) * weights[0]; // todo?! value traits?! + for( unsigned int j = 1; j < weights.size(); ++j ) { v += this->operator[]( Traits::add( i, neighbours[j] ) ) * weights[j]; } // todo?! value traits?! + return v; +} + +// template < typename V, unsigned int D, typename P, typename Traits, typename S > +// V grid< V, D, P, Traits, S >::interpolated( const P& point ) const +// { +// const index_type i = index_of( point ); +// const P p = Traits::subtract( point, Traits::add( Traits::vmultiply( _resolution, i ), _origin ) ); +// //std::cerr << "==> a: point: " << point[0] << "," << point[1] << " p: " << p[0] << "," << p[1] << " _resolution: " << _resolution[0] << "," << _resolution[1] << std::endl; +// const auto& neighbours = impl::neighbours< index_type, D >; +// double s = 0; +// V v = this->operator[]( i ); // todo?! value traits?! +// for( unsigned int j = 0; j < neighbours.size(); ++j ) +// { +// P d = Traits::subtract( p, Traits::vmultiply( _resolution, neighbours[j] ) ); +// double n = Traits::dot( d, d ); //double n = std::sqrt( Traits::dot( d, d ) ); +// index_type k = Traits::add( i, neighbours[j] ); +// if( math::equal( n, 0 ) ) { return this->operator[]( k ); } +// double w = 1 / n; +// s += w; +// //std::cerr << "==> b: j: " << j << " k: " << k[0] << "," << k[1] << " n: " << n << " w: " << w << std::endl; +// if( j == 0 ) { v *= w; } else { v += this->operator[]( k ) * w; } // quick and dirty for now +// } +// //std::cerr << "==> c: s: " << s << std::endl; +// return v * ( 1 / s ); +// } + +template < typename V, unsigned int D, typename P, typename Traits, typename S > +typename grid< V, D, P, Traits, S >::index_type grid< V, D, P, Traits, S >::nearest_to( const P& point ) const +{ + P element_origin = _resolution; + index_type i = index_of( point ); + Traits::add( Traits::vmultiply( element_origin, i ), _origin ); + return Traits::add( Traits::template nearest< index_type >( point, const_cast< const P& >( element_origin ), _resolution ), i ); +} + +template < typename V, unsigned int D, typename P, typename Traits, typename S > +inline bool grid< V, D, P, Traits, S >::has( const P& point ) const // quick and dirty for now +{ + const auto& i = index_of( point ); + for( unsigned int k = 0; k < D; ++k ) { if( i[k] < 0 || i[k] >= this->shape()[k] ) { return false; } } + return true; +} + +} } } // namespace comma { namespace containers { namespace multidimensional { diff --git a/containers/multidimensional/array_traits.h b/containers/multidimensional/array_traits.h new file mode 100644 index 000000000..59d9eec0e --- /dev/null +++ b/containers/multidimensional/array_traits.h @@ -0,0 +1,153 @@ +// Copyright (c) 2023 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +#include +#include +#include +#include "../../math/compare.h" +#include "index.h" + +template< typename T, std::size_t D, typename S > +inline std::array< T, D >& operator*=( std::array< T, D >& lhs, const S& rhs ) { for( unsigned int i = 0; i < D; ++i ) { lhs[i] *= rhs; } return lhs; } // quick and dirty; let compiler optimize + +template< typename T, std::size_t D, typename S > +inline std::array< T, D > operator*( const std::array< T, D >& lhs, const S& rhs ) { auto r = lhs; r *= rhs; return r; } + +template< typename T, std::size_t D > +inline std::array< T, D >& operator+=( std::array< T, D >& lhs, const std::array< T, D >& rhs ) { for( unsigned int i = 0; i < D; ++i ) { lhs[i] += rhs[i]; } return lhs; } // quick and dirty; let compiler optimize + +template< typename T, std::size_t D > +inline std::array< T, D > operator+( const std::array< T, D >& lhs, const std::array< T, D >& rhs ) { auto r = lhs; r += rhs; return r; } + +namespace comma { namespace containers { namespace multidimensional { namespace impl { + +template < typename T > inline int index( T p, T origin, T resolution ) +{ + static constexpr int negative_flooring = static_cast< int >( -1.5 ) == -1 ? -1 : static_cast< int >( -1.5 ) == -2 ? 0 : 0; + static constexpr int positive_flooring = static_cast< int >( 1.5 ) == 1 ? 0 : static_cast< int >( 1.5 ) == 2 ? -1 : -1; + double diff = ( p - origin ) / resolution; + int i = diff; + if( i == 0 || diff != i ) { i += diff < 0 ? negative_flooring : positive_flooring; } + return i; +} + +template < int Base, unsigned int Pow > static constexpr int pow = Base * pow< Base, Pow - 1 >; +template < int Base > static constexpr int pow< Base, 0 >{1}; +template < typename I, std::size_t Size > static constexpr std::array< I, Size > neighbours; // quick and dirty, for now leaving it to the enthusiasts to implement it using metaprogramming +template < typename I > static constexpr std::array< I, 2 > neighbours< I, 1 > = {{ I{ 0 }, I{ 1 } }}; +template < typename I > static std::array< I, 4 > neighbours< I, 2 > = {{ I{ 0, 0 }, I{ 0, 1 }, I{ 1, 0 }, I{ 1, 1 } }}; +template < typename I > static std::array< I, 8 > neighbours< I, 3 > = {{ I{ 0, 0, 0 }, I{ 0, 0, 1 }, I{ 0, 1, 0 }, I{ 0, 1, 1 }, I{ 1, 0, 0 }, I{ 1, 0, 1 }, I{ 1, 1, 0 }, I{ 1, 1, 1 } }}; +template < typename I > static std::array< I, 16 > neighbours< I, 4 > = {{ I{ 0, 0, 0, 0 }, I{ 0, 0, 0, 1 }, I{ 0, 0, 1, 0 }, I{ 0, 0, 1, 1 }, I{ 0, 1, 0, 0 }, I{ 0, 1, 0, 1 }, I{ 0, 1, 1, 0 }, I{ 0, 1, 1, 1 }, I{ 1, 0, 0, 0 }, I{ 1, 0, 0, 1 }, I{ 1, 0, 1, 0 }, I{ 1, 0, 1, 1 }, I{ 1, 1, 0, 0 }, I{ 1, 1, 0, 1 }, I{ 1, 1, 1, 0 }, I{ 1, 1, 1, 1 } }}; +// template < typename I > static constexpr std::array< I, 2 > neighbours< I, 1 > = {{ {{ 0 }}, {{ 1 }} }}; +// template < typename I > static constexpr std::array< I, 4 > neighbours< I, 2 > = {{ {{ 0, 0 }}, {{ 0, 1 }}, {{ 1, 0 }}, {{ 1, 1 }} }}; +// template < typename I > static constexpr std::array< I, 8 > neighbours< I, 3 > = {{ {{ 0, 0, 0 }}, {{ 0, 0, 1 }}, {{ 0, 1, 0 }}, {{ 0, 1, 1 }}, {{ 1, 0, 0 }}, {{ 1, 0, 1 }}, {{ 1, 1, 0 }}, {{ 1, 1, 1 }} }}; +// template < typename I > static constexpr std::array< I, 16 > neighbours< I, 4 > = {{ {{ 0, 0, 0, 0 }}, {{ 0, 0, 0, 1 }}, {{ 0, 0, 1, 0 }}, {{ 0, 0, 1, 1 }}, {{ 0, 1, 0, 0 }}, {{ 0, 1, 0, 1 }}, {{ 0, 1, 1, 0 }}, {{ 0, 1, 1, 1 }}, {{ 1, 0, 0, 0 }}, {{ 1, 0, 0, 1 }}, {{ 1, 0, 1, 0 }}, {{ 1, 0, 1, 1 }}, {{ 1, 1, 0, 0 }}, {{ 1, 1, 0, 1 }}, {{ 1, 1, 1, 0 }}, {{ 1, 1, 1, 1 }} }}; +// todo: add more dimensions as required or write that little metaprogramming piece + +template < std::size_t Size > struct operations +{ + template < typename S, typename T > static S& add( S& s, const T& t ) { s[ Size - 1 ] += t[ Size - 1 ]; operations< Size - 1 >::add( s, t ); return s; } + template < typename S, typename T > static S& subtract( S& s, const T& t ) { s[ Size - 1 ] -= t[ Size - 1 ]; operations< Size - 1 >::subtract( s, t ); return s; } + template < typename S, typename T > static S& vdivide( S& s, const T& t ) { s[ Size - 1 ] /= t[ Size - 1 ]; operations< Size - 1 >::vdivide( s, t ); return s; } + template < typename S, typename T > static S& multiply( S& s, const T& t ) { s[ Size - 1 ] *= t; operations< Size - 1 >::multiply( s, t ); return s; } + template < typename S, typename T > static S& vmultiply( S& s, const T& t ) { s[ Size - 1 ] *= t[ Size - 1 ]; operations< Size - 1 >::vmultiply( s, t ); return s; } + template < typename S, typename T > static double dot( S& s, const T& t ) { return s[ Size - 1 ] * t[ Size - 1 ] + operations< Size - 1 >::dot( s, t ); } + template < typename S, typename T > static S& mask( S& s, const T& t ) { s[ Size - 1 ] = t[ Size - 1 ] ? t[ Size - 1 ] : s[ Size - 1 ]; operations< Size - 1 >::mask( s, t ); return s; } + template < typename S, typename T > static S masked( const S& s, const T& t ) { S m = s; mask( m, t ); return m; } + template < typename S, typename T > static S& logical_not( S& s ) { s[ Size - 1 ] = !s[ Size - 1 ]; operations< Size - 1 >::logical_not( s ); return s; } + template < typename S > static S& fill( S&s, double value ) { s[ Size - 1 ] = value; operations< Size - 1 >::fill( s, value ); return s; } + template < typename S > static S filled( double value ) { S s; fill( s, value ); return s; } + template < typename S > static S zero() { S s; fill( s, 0 ); return s; } + template < typename S > static auto product( const S& s ) -> typename std::remove_reference< decltype( s[Size] ) >::type { return s[ Size - 1 ] * operations< Size - 1 >::product( s ); } + template < typename S, typename T, typename Diff > static bool near( const S& s, const T& t, const Diff& epsilon ) { return comma::math::equal( s[ Size - 1 ], t[ Size - 1 ], epsilon ) && operations< Size - 1 >::near( s, t, epsilon ); } + template < typename S, typename T > static S add( const S& s, const T& t ) { S r = s; add( r, t ); return r; } + template < typename S, typename T > static S subtract( const S& s, const T& t ) { S r = s; subtract( r, t ); return r; } + template < typename S, typename T > static S vdivide( const S& s, const T& t ) { S r = s; vdivide( r, t ); return r; } + template < typename S, typename T > static S multiply( const S& s, const T& t ) { S r = s; multiply( r, t ); return r; } + template < typename S, typename T > static S vmultiply( const S& s, const T& t ) { S r = s; vmultiply( r, t ); return r; } + + template < typename S, typename I > static I& index_of( I& i, const S& p, const S& origin, const S& resolution ) + { + i[ Size - 1 ] = index( p[ Size - 1 ], origin[ Size - 1 ], resolution[ Size - 1 ] ); + operations< Size - 1 >::index_of( i, p, origin, resolution ); + return i; + } + + template < typename S, typename I > static I index_of( const S& p, const S& origin, const S& resolution ) { I i; index_of( i, p, origin, resolution ); return i; } + + template < typename S, typename I > static I nearest( const S& p, const S& origin, const S& resolution ) // todo? metaprogram? + { + const S& s = subtract( p, origin ); + double m = dot( s, s ); + unsigned int j = 0; + const auto& n = neighbours< I, Size >; + for( unsigned int i = 1; m > 0 && i < n.size(); ++i ) + { + const S& r = subtract( vmultiply( resolution, n[i] ), s ); + double d = dot( r, r ); + if( d < m ) { m = d; j = i; } + } + return n[j]; + } + + template < typename S, typename I > static std::array< double, pow< 2, Size > > squared_norms( const S& p, const S& origin, const S& resolution ) // todo? metaprogram? + { + const S& s = subtract( p, origin ); + std::array< double, pow< 2, Size > > d; + const auto& n = neighbours< I, Size >; + for( unsigned int i = 0; i < n.size(); ++i ) + { + const S& r = subtract( vmultiply( resolution, n[i] ), s ); + d[i] = dot( r, r ); + } + return d; + } + + template < typename S, typename I > static std::array< double, pow< 2, Size > > norms( const S& p, const S& origin, const S& resolution ) + { + std::array< double, pow< 2, Size > > d = squared_norms( p, origin, resolution ); + for( auto& v: d ) { v = std::sqrt( v ); } + return d; + } + + struct interpolation + { + struct linear + { + template < typename S > static std::array< double, pow< 2, Size > > weights( const S& s ) + { + S t; + subtract( fill( t, 1. ), s ); + const auto& n = neighbours< std::array< unsigned int, Size >, Size >; + std::array< double, pow< 2, Size > > w; + for( unsigned int i = 0; i < pow< 2, Size >; ++i ) { w[i] = product( masked( s, n[ pow< 2, Size > - i - 1 ] ) ) * product( masked( t, n[i] ) ); } + return w; + } + + template < typename S > static std::array< double, pow< 2, Size > > weights( const S& p, const S& origin, const S& resolution ) { return weights( vdivide( subtract( p, origin ), resolution ) ); } + }; + }; +}; + +template <> struct operations< 1 > +{ + template < typename S, typename T > static S& add( S& s, const T& t ) { s[0] += t[0]; return s; } + template < typename S, typename T > static S& subtract( S& s, const T& t ) { s[0] -= t[0]; return s; } + template < typename S, typename T > static S& vdivide( S& s, const T& t ) { s[0] /= t[0]; return s; } + template < typename S, typename T > static S& multiply( S& s, const T& t ) { s[0] *= t; return s; } + template < typename S, typename T > static S& vmultiply( S& s, const T& t ) { s[0] *= t[0]; return s; } + template < typename S, typename T > static double dot( S& s, const T& t ) { return s[0] * t[0]; } + template < typename S, typename T > static S& logical_not( S& s ) { s[0] = !s[0]; return s; } + template < typename S, typename T > static S& mask( S& s, const T& t ) { s[0] = t[0] ? t[0] : s[0]; return s; } + template < typename S > static auto product( const S& s ) -> typename std::remove_reference< decltype( s[0] ) >::type { return s[0]; } + template < typename S, typename T, typename Diff > static bool near( const S& s, const T& t, const Diff& epsilon ) { return comma::math::equal( s[0], t[0], epsilon ); } + template < typename S > static S& fill( S&s, double value ) { s[0] = value; return s; } + template < typename S > static S zero() { S s; fill( s, 0 ); return s; } + template < typename S, typename I > static I& index_of( I& i, const S& p, const S& origin, const S& resolution ) { i[0] = index( p[0], origin[0], resolution[0] ); return i; } + template < typename S, typename I > static I nearest( const S& p, const S& origin, const S& resolution ) { return p[0] - origin[0] < resolution[0] / 2 ? I{ 0 } : I{ 1 }; } +}; + +} } } } // namespace comma { namespace containers { namespace multidimensional { namespace impl { diff --git a/containers/multidimensional/index.h b/containers/multidimensional/index.h new file mode 100644 index 000000000..6df4a3430 --- /dev/null +++ b/containers/multidimensional/index.h @@ -0,0 +1,108 @@ +// Copyright (c) 2024 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +#include +#include +#include "../../base/types.h" + +namespace comma { namespace containers { namespace multidimensional { + +template < unsigned int D, typename T = std::size_t > +struct index: public std::array< T, D > +{ + typedef T value_t; + + typedef std::array< T, D > base_t; + + index(): base_t{} {} + + index( const index& rhs ): base_t( static_cast< const base_t& >( rhs ) ) {} + + template < typename... Args > index( T t, Args... args ); // quick and dirty for now to avoid compile warning // template < typename... Args > index( Args... args ): base_t( { args... } ) {} + + bool operator<( const index& rhs ) const; + + bool operator==( const index& rhs ) const; + + bool operator!=( const index& rhs ) const { return !operator==( rhs ); } + + index& operator=( const index& rhs ) = default; + + index& operator=( const base_t& rhs ) { static_cast< base_t& >( *this ) = rhs; } + + operator base_t() { return static_cast< base_t& >( *this ); } + + operator base_t() const { return static_cast< const base_t& >( *this ); } + + index& increment( const index& sizes ); + + class iterator + { + public: + iterator( const index< D >& shape ): _shape( shape ) {} + + iterator& operator++() { _valid = _index.increment( _shape ) != index< D >{}; return *this; } + + operator bool() const { return _valid; } + + const index< D >& operator*() const { return _index; } + + private: + index< D > _index; + index< D > _shape; + bool _valid{true}; + }; +}; + +namespace impl { + +template < typename T, unsigned int D, unsigned int I > struct type_cast +{ + template < typename... Args > static void assign( index< D, T >& i, T v, Args... args ) + { + i[ D - I ] = v; + type_cast< T, D, I - 1 >::assign( i, args... ); + } +}; + +template < typename T, unsigned int D > struct type_cast< T, D, 1 > +{ + static void assign( index< D, T >& i, T v ) { i[ D - 1 ] = v; } +}; + +} // namespace impl { + +template < unsigned int D, typename T > +template < typename... Args > inline index< D, T >::index( T t, Args... args ) +{ + impl::type_cast< T, D, D >::assign( *this, t, args... ); +} + +template < unsigned int D, typename T > inline bool index< D, T >::operator<( const index& rhs ) const // todo: unravel in compile time (compiler probably will do it anyway) +{ + for( unsigned int i = 0; i < D; ++i ) + { + if( ( *this )[i] < rhs[i] ) { return true; } + } + return false; +} + +template < unsigned int D, typename T > inline bool index< D, T >::operator==( const index& rhs ) const // todo: unravel in compile time (compiler probably will do it anyway) +{ + return std::memcmp( reinterpret_cast< const char* >( this ), reinterpret_cast< const char* >( &rhs ), sizeof( std::size_t ) * D ) == 0; +} + +template < unsigned int D, typename T > inline index< D, T >& index< D, T >::increment( const index< D, T >& sizes ) // todo: unravel in compile time (compiler probably will do it anyway) +{ + for( unsigned int i{0}, j{D - 1}; i < D; ++i, --j ) + { + if( ++( *this )[j] < sizes[j] ) { return *this; } + ( *this )[j] = 0; + } + return *this; +} + +} } } // namespace comma { namespace containers { namespace multidimensional { diff --git a/containers/multidimensional/map.h b/containers/multidimensional/map.h new file mode 100644 index 000000000..b4347d7d8 --- /dev/null +++ b/containers/multidimensional/map.h @@ -0,0 +1,177 @@ +// Copyright (c) 2023 Vsevolod Vlaskine + +/// @author vsevolod vlaskine + +#pragma once + +#include +#include +#include +#include +#include "../../base/types.h" +#include "array_traits.h" + +namespace comma { namespace containers { namespace multidimensional { + +/// quick and dirty hash for array-like containers (its support is awkward in boost) +template < typename Array, std::size_t Size > +struct array_hash : public std::function< std::size_t( const Array& ) > // struct array_hash : public std::unary_function< Array, std::size_t > +{ + std::size_t operator()( Array const& array ) const + { + std::size_t seed = 0; + for( std::size_t i = 0; i < Size; ++i ) { boost::hash_combine( seed, array[i] ); } + return seed; + // return boost::hash_range( &array[0], &array[Size] ); // not so easy... + } +}; + +/// unordered map with array-like keys +template < typename K, typename V, std::size_t Size, typename P = std::array< K, Size >, typename Traits = impl::operations< Size > > +class map : public std::unordered_map< std::array< comma::int32, Size >, V, array_hash< std::array< comma::int32, Size >, Size > > +{ + public: + typedef std::unordered_map< std::array< comma::int32, Size >, V, array_hash< std::array< comma::int32, Size >, Size > > base_type; + + typedef base_type map_type; + + typedef base_type as_map; + + enum { dimensions = Size }; + + typedef P point_type; + + typedef typename base_type::key_type index_type; + + typedef typename base_type::key_type key_type; // for brevity + + typedef typename base_type::mapped_type mapped_type; // for brevity + + typedef typename base_type::iterator iterator; // otherwise it does not build on windows... + + typedef typename base_type::const_iterator const_iterator; // otherwise it does not build on windows... + + /// constructor + map( const point_type& origin, const point_type& resolution ); + + /// constructor, origin is all zeroes + map( const point_type& resolution ); + + /// insert element at the given point, if it does not exist + iterator touch_at( const point_type& point ); + + /// insert element at the given point, if it does not exist + std::pair< iterator, bool > insert( const point_type& point, const mapped_type& value ); + + /// return index of the point, always rounds it down (does floor for a given resolution) + key_type index_of( const point_type& point ) const; + + /// same as index_of( point ), but static + static key_type index_of( const point_type& point, const point_type& origin, const point_type& resolution ); + + /// same as index_of( point ), but static with origin assumed all zeroes + static key_type index_of( const point_type& point, const point_type& resolution ); + + /// find value by point + iterator at( const point_type& point ); + + /// find value by point + const_iterator at( const point_type& point ) const; + + /// find value by key + iterator find( const key_type& index ); + + /// find value by key + const_iterator find( const key_type& index ) const; + + /// return origin + const point_type& origin() const; + + /// return resolution + const point_type& resolution() const; + + private: + point_type _origin; + point_type _resolution; +}; + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline map< K, V, Size, P, Traits >::map( const typename map< K, V, Size, P, Traits >::point_type& origin, const typename map< K, V, Size, P, Traits >::point_type& resolution ) + : _origin( origin ) + , _resolution( resolution ) +{ +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline map< K, V, Size, P, Traits >::map( const typename map< K, V, Size, P, Traits >::point_type& resolution ) + : _origin( Traits::template zero< P >() ) + , _resolution( resolution ) +{ +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline typename map< K, V, Size, P, Traits >::iterator map< K, V, Size, P, Traits >::touch_at( const typename map< K, V, Size, P, Traits >::point_type& point ) +{ + key_type index = index_of( point ); + iterator it = this->base_type::find( index ); + if( it != this->end() ) { return it; } + return this->base_type::insert( std::make_pair( index, mapped_type() ) ).first; +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline std::pair< typename map< K, V, Size, P, Traits >::iterator, bool > map< K, V, Size, P, Traits >::insert( const typename map< K, V, Size, P, Traits >::point_type& point, const typename map< K, V, Size, P, Traits >::mapped_type& value ) +{ + return this->base_type::insert( std::make_pair( index_of( point ), value ) ); +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline typename map< K, V, Size, P, Traits >::key_type map< K, V, Size, P, Traits >::index_of( const typename map< K, V, Size, P, Traits >::point_type& point, const typename map< K, V, Size, P, Traits >::point_type& origin, const typename map< K, V, Size, P, Traits >::point_type& resolution ) +{ + return Traits::template index_of< P, key_type >( point, origin, resolution ); +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline typename map< K, V, Size, P, Traits >::key_type map< K, V, Size, P, Traits >::index_of( const typename map< K, V, Size, P, Traits >::point_type& point, const typename map< K, V, Size, P, Traits >::point_type& resolution ) +{ + return index_of( point, Traits::template zero< P >(), resolution ); +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline typename map< K, V, Size, P, Traits >::key_type map< K, V, Size, P, Traits >::index_of( const typename map< K, V, Size, P, Traits >::point_type& point ) const +{ + return index_of( point, _origin, _resolution ); +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline typename map< K, V, Size, P, Traits >::iterator map< K, V, Size, P, Traits >::at( const typename map< K, V, Size, P, Traits >::point_type& point ) +{ + index_type i = index_of( point ); + return this->base_type::find( i ); +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline typename map< K, V, Size, P, Traits >::const_iterator map< K, V, Size, P, Traits >::at( const typename map< K, V, Size, P, Traits >::point_type& point ) const +{ + index_type i = index_of( point ); + return this->base_type::find( i ); +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline typename map< K, V, Size, P, Traits >::iterator map< K, V, Size, P, Traits >::find( const typename map< K, V, Size, P, Traits >::key_type& index ) +{ + return this->base_type::find( index ); // otherwise strange things happen... debug, when we have time +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline typename map< K, V, Size, P, Traits >::const_iterator map< K, V, Size, P, Traits >::find( const typename map< K, V, Size, P, Traits >::key_type& index ) const +{ + return this->base_type::find( index ); // otherwise strange things happen... debug, when we have time +} + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline const typename map< K, V, Size, P, Traits >::point_type& map< K, V, Size, P, Traits >::origin() const { return _origin; } + +template < typename K, typename V, std::size_t Size, typename P, typename Traits > +inline const typename map< K, V, Size, P, Traits >::point_type& map< K, V, Size, P, Traits >::resolution() const { return _resolution; } + +} } } // namespace comma { namespace containers { namespace multidimensional { diff --git a/containers/multidimensional/traits.h b/containers/multidimensional/traits.h new file mode 100644 index 000000000..1e32fbff1 --- /dev/null +++ b/containers/multidimensional/traits.h @@ -0,0 +1,25 @@ +// Copyright (c) 2024 Vsevolod Vlaskine + +#pragma once + +#include "../../visiting/traits.h" +#include "index.h" + +namespace comma { namespace visiting { + +template < unsigned int D, typename T > struct traits< comma::containers::multidimensional::index< D, T > > +{ + typedef comma::containers::multidimensional::index< D, T > value_t; + + template < typename Key, class Visitor > static void visit( const Key& k, value_t& p, Visitor& v ) + { + comma::visiting::traits< std::array< T, D > >::visit( k, static_cast< std::array< T, D >& >( p ), v ); + } + + template < typename Key, class Visitor > static void visit( const Key& k, const value_t& p, Visitor& v ) + { + comma::visiting::traits< std::array< T, D > >::visit( k, static_cast< const std::array< T, D >& >( p ), v ); + } +}; + +} } // namespace comma { namespace visiting { diff --git a/containers/ordered/queues.h b/containers/ordered/queues.h new file mode 100644 index 000000000..27580392a --- /dev/null +++ b/containers/ordered/queues.h @@ -0,0 +1,77 @@ +// Copyright (c) 2024 Mission Systems + +/// @authors aspen eyers, vsevolod vlaskine + +#pragma once + +#include +#include + +namespace comma { namespace containers { namespace ordered { + +/// @todo variadic types +/// @todo don't use std::pair, use traits instead? +/// @todo max_diff, grater or grater_equal? & document; unit test on ints; unit test on max diff 0] +/// @todo pop_all will remove the first elements from both queues, but we may want to pop just one element and still keep the other one +/// Use case: we want to get every (valid) element from both queues with its corresponding element from the other queue and process them +/// independently. If we pop both elements, we may lose the correspondence between the elements from the two queues. +template < typename K, typename T, typename S > +class queues: public std::tuple< std::deque< std::pair< K, T > >, std::deque< std::pair< K, S > > > +{ + public: + typedef std::tuple< std::deque< std::pair< K, T > >, std::deque< std::pair< K, S > > > queues_type; + typedef std::tuple< std::pair< K, T >, std::pair< K, S > > values_type; + typedef std::tuple< const std::pair< K, T >&, const std::pair< K, S >& > ref_type; + typedef decltype( K() - K() ) diff_type; + queues( diff_type max_diff ): _max_diff( max_diff ) {} + bool ready() const; + void purge(); + void pop_all(); + ref_type front() const; + + private: + diff_type _max_diff; + static diff_type _abs_diff(K lhs, K rhs) { return lhs < rhs ? (rhs - lhs) : (lhs - rhs); } + template < unsigned int I, unsigned int J > bool _purge(); +}; + +template < typename K, typename T, typename S > +inline bool queues::ready() const +{ + if( std::get<0>(*this).empty() || std::get<1>(*this).empty() ) { return false; } + return _abs_diff( std::get<1>(*this).front().first, std::get<0>(*this).front().first ) <= _max_diff; +} + +template < typename K, typename T, typename S > +inline void queues::purge() +{ + if( std::get<1>(*this).empty() || std::get<0>(*this).empty() ) { return; } + // If not purge 0,1, then purge 1,0 + while( std::get<0>(*this).front().first - std::get<1>(*this).front().first > _max_diff ) + { + if( std::get<1>(*this).empty() ) { return; } + std::get<1>(*this).pop_front(); + } + while( std::get<1>(*this).front().first - std::get<0>(*this).front().first > _max_diff ) + { + if( std::get<0>(*this).empty() ) { return; } + std::get<0>(*this).pop_front(); + } +} + +template < typename K, typename T, typename S > +void queues::pop_all() +{ + std::get<0>(*this).pop_front(); + std::get<1>(*this).pop_front(); + return; +} + + +template < typename K, typename T, typename S > +inline typename queues::ref_type queues::front() const +{ + return { std::get<0>(*this).front(), std::get<1>(*this).front() }; +} + +} } } // namespace comma { namespace containers { namespace ordered { diff --git a/containers/test/CMakeLists.txt b/containers/test/CMakeLists.txt index bdfd90360..4442cc2c5 100644 --- a/containers/test/CMakeLists.txt +++ b/containers/test/CMakeLists.txt @@ -1,16 +1,13 @@ -SET( KIT containers ) - -FILE( GLOB source ${SOURCE_CODE_BASE_DIR}/${KIT}/test/*.cpp ) - -ADD_EXECUTABLE( ${CMAKE_PROJECT_NAME}_test_${KIT} ${source} ) - -TARGET_LINK_LIBRARIES( ${CMAKE_PROJECT_NAME}_test_${KIT} comma_base ${GTEST_BOTH_LIBRARIES} ) - -IF( INSTALL_TESTS ) -INSTALL ( - FILES ${PROJECT_BINARY_DIR}/bin/${CMAKE_PROJECT_NAME}_test_${KIT} - PERMISSIONS WORLD_READ GROUP_READ OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE - DESTINATION ${comma_CPP_TESTS_INSTALL_DIR} ) -ENDIF( INSTALL_TESTS ) - -add_test( NAME ${CMAKE_PROJECT_NAME}_test_${KIT} COMMAND ${CMAKE_PROJECT_NAME}_test_${KIT} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/bin ) +set( KIT containers ) +file( GLOB source ${SOURCE_CODE_BASE_DIR}/${KIT}/test/*test.cpp ) +set( test_name ${CMAKE_PROJECT_NAME}_test_${KIT} ) +add_executable( ${test_name} ${source} ) +target_link_libraries( ${test_name} ${GTEST_BOTH_LIBRARIES} comma_base pthread ) # target_link_libraries( ${test_name} comma_${KIT} ${GTEST_BOTH_LIBRARIES} pthread ) +add_test( NAME ${test_name} COMMAND ${CMAKE_PROJECT_NAME}_test_${KIT} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/bin ) +if( INSTALL_TESTS ) + install( TARGETS ${test_name} RUNTIME DESTINATION ${comma_CPP_TESTS_INSTALL_DIR} COMPONENT Runtime ) + #INSTALL ( + # FILES ${PROJECT_BINARY_DIR}/bin/${CMAKE_PROJECT_NAME}_test_${KIT} + # PERMISSIONS WORLD_READ GROUP_READ OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE + # DESTINATION ${comma_CPP_TESTS_INSTALL_DIR} ) +endif( INSTALL_TESTS ) diff --git a/containers/test/array_traits_test.cpp b/containers/test/array_traits_test.cpp new file mode 100644 index 000000000..6be69a2f3 --- /dev/null +++ b/containers/test/array_traits_test.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2023 Vsevolod Vlaskine + +#include +#include +#include "../../math/compare.h" +#include "../multidimensional/array_traits.h" + +namespace ccmi = comma::containers::multidimensional::impl; + +TEST( array_traits, operations_nearest ) +{ + { + typedef std::array< double, 1 > point_t; + typedef std::array< int, 1 > index_t; + { index_t i = ccmi::operations< 1 >::nearest< point_t, index_t >( point_t{0}, point_t{0}, point_t{1} ); EXPECT_EQ( index_t{0}, i ); } + { index_t i = ccmi::operations< 1 >::nearest< point_t, index_t >( point_t{0.3}, point_t{0}, point_t{1} ); EXPECT_EQ( index_t{0}, i ); } + { index_t i = ccmi::operations< 1 >::nearest< point_t, index_t >( point_t{0.5}, point_t{0}, point_t{1} ); EXPECT_EQ( index_t{1}, i ); } + { index_t i = ccmi::operations< 1 >::nearest< point_t, index_t >( point_t{0.999}, point_t{0}, point_t{1} ); EXPECT_EQ( index_t{1}, i ); } + { index_t i = ccmi::operations< 1 >::nearest< point_t, index_t >( point_t{0}, point_t{0}, point_t{1} ); EXPECT_EQ( index_t{0}, i ); } + { index_t i = ccmi::operations< 1 >::nearest< point_t, index_t >( point_t{0.3}, point_t{0}, point_t{1} ); EXPECT_EQ( index_t{0}, i ); } + { index_t i = ccmi::operations< 1 >::nearest< point_t, index_t >( point_t{0.5}, point_t{0}, point_t{1} ); EXPECT_EQ( index_t{1}, i ); } + { index_t i = ccmi::operations< 1 >::nearest< point_t, index_t >( point_t{0.999}, point_t{0}, point_t{1} ); EXPECT_EQ( index_t{1}, i ); } + } + { + typedef std::array< double, 2 > point_t; + typedef std::array< int, 2 > index_t; + { index_t expected{0, 0}; index_t actual = ccmi::operations< 2 >::nearest< point_t, index_t >( point_t{0, 0}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 0}; index_t actual = ccmi::operations< 2 >::nearest< point_t, index_t >( point_t{0.1, 0}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 0}; index_t actual = ccmi::operations< 2 >::nearest< point_t, index_t >( point_t{0, 0.1}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 0}; index_t actual = ccmi::operations< 2 >::nearest< point_t, index_t >( point_t{0.1, 0.2}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 1}; index_t actual = ccmi::operations< 2 >::nearest< point_t, index_t >( point_t{0, 0.7}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 1}; index_t actual = ccmi::operations< 2 >::nearest< point_t, index_t >( point_t{0.1, 0.7}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{1, 1}; index_t actual = ccmi::operations< 2 >::nearest< point_t, index_t >( point_t{1, 1}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + } + { + typedef std::array< double, 3 > point_t; + typedef std::array< int, 3 > index_t; + { index_t expected{0, 0, 0}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0, 0, 0}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{1, 0, 0}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0.7, 0, 0}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 1, 0}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0, 0.7, 0}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 0, 1}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0, 0, 0.7}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{1, 1, 1}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0.7, 0.7, 0.7}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + } + { + typedef std::array< float, 3 > point_t; + typedef std::array< float, 3 > index_t; + { index_t expected{0, 0, 0}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0, 0, 0}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{1, 0, 0}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0.7, 0, 0}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 1, 0}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0, 0.7, 0}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{0, 0, 1}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0, 0, 0.7}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + { index_t expected{1, 1, 1}; index_t actual = ccmi::operations< 3 >::nearest< point_t, index_t >( point_t{0.7, 0.7, 0.7}, point_t{0, 0, 0}, point_t{1, 1, 1} ); EXPECT_EQ( expected, actual ); } + } +} + +TEST( array_traits, interpolation_linear_weights ) +{ + { + typedef std::array< double, 2 > point_t; + typedef std::array< double, 4 > weights_t; + { weights_t expected{1, 0, 0, 0}; auto actual = ccmi::operations< 2 >::interpolation::linear::weights( point_t{0, 0}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { weights_t expected{0, 1, 0, 0}; auto actual = ccmi::operations< 2 >::interpolation::linear::weights( point_t{0, 1}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { weights_t expected{0, 0, 1, 0}; auto actual = ccmi::operations< 2 >::interpolation::linear::weights( point_t{1, 0}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { weights_t expected{0, 0, 0, 1}; auto actual = ccmi::operations< 2 >::interpolation::linear::weights( point_t{1, 1}, point_t{0, 0}, point_t{1, 1} ); EXPECT_EQ( expected, actual ); } + { weights_t expected{0.3, 0.7, 0, 0}; auto actual = ccmi::operations< 2 >::interpolation::linear::weights( point_t{0, 0.7}, point_t{0, 0}, point_t{1, 1} ); EXPECT_TRUE( ccmi::operations< 3 >::near( expected, actual, 1e-6 ) ); } + { weights_t expected{0.3, 0, 0.7, 0}; auto actual = ccmi::operations< 2 >::interpolation::linear::weights( point_t{0.7, 0}, point_t{0, 0}, point_t{1, 1} ); EXPECT_TRUE( ccmi::operations< 3 >::near( expected, actual, 1e-6 ) ); } + // todo: way more testing + } +} diff --git a/containers/test/cached_test.cpp b/containers/test/cached_test.cpp new file mode 100644 index 000000000..68f4238b5 --- /dev/null +++ b/containers/test/cached_test.cpp @@ -0,0 +1,148 @@ +// Copyright (c) 2024 Vsevolod Vlaskine +// All Rights Reserved + +#include +#include +#include +#include "../cached.h" + +struct square +{ + typedef int key; + + square( int x ): a( x ) {} + + int calculate( int x ) const { return a * x; } + + int a{0}; +}; + +TEST( cached, basics ) +{ + comma::cached< square, int > c; + EXPECT_EQ( c.get( 5 ).calculate( 5 ), 25 ); + EXPECT_EQ( c.values().size(), 1 ); + EXPECT_EQ( c.get( 5 ).calculate( 5 ), 25 ); + EXPECT_EQ( c.values().size(), 1 ); + EXPECT_EQ( c.get( 10 ).calculate( 10 ), 100 ); + EXPECT_EQ( c.values().size(), 2 ); + EXPECT_EQ( c.get( 10 ).calculate( 10 ), 100 ); + EXPECT_EQ( c.values().size(), 2 ); + EXPECT_EQ( c.get( 20 ).calculate( 20 ), 400 ); + EXPECT_EQ( c.values().size(), 3 ); + EXPECT_EQ( c.get( 20 ).calculate( 20 ), 400 ); + EXPECT_EQ( c.values().size(), 3 ); + c.pop(); + EXPECT_EQ( c.values().size(), 2 ); + EXPECT_EQ( c.values().size(), 2 ); + EXPECT_EQ( c.get( 20 ).calculate( 20 ), 400 ); + EXPECT_EQ( c.values().size(), 2 ); +} + +struct someclass +{ + typedef std::pair< int, int > key; + + struct hash + { + std::size_t operator()( key const& p ) const + { + std::size_t seed = 0; + boost::hash_combine( seed, p.first ); + boost::hash_combine( seed, p.second ); + return seed; + } + }; + + someclass( const key& k ) {} + + void dummy( int x, int y ) const {} +}; + +TEST( cached, key ) +{ + comma::cached< someclass, someclass::key, someclass::hash > c; + c.get( 1, 2 ).dummy( 1, 2 ); + EXPECT_EQ( c.values().size(), 1 ); + c.get( 1, 2 ).dummy( 1, 2 ); + EXPECT_EQ( c.values().size(), 1 ); + c.get( 3, 1 ).dummy( 3, 1 ); + EXPECT_EQ( c.values().size(), 2 ); + c.get( 3, 1 ).dummy( 3, 1 ); + EXPECT_EQ( c.values().size(), 2 ); +} + +struct plan +{ + struct params + { + int size{0}; + bool real{false}; + bool inverse{false}; + + bool operator==( const params& rhs ) const { return size == rhs.size && real == rhs.real && inverse == rhs.inverse; } + + params( const std::vector< int >& v, bool real, bool inverse ): size( v.size() ), real( real ), inverse( inverse ) {} + + params( const std::set< int >& v, bool x ): size( v.size() ), real( x ), inverse( x ) {} + }; + + plan( const params& ) {} + + void operator()( const std::vector< int >&, bool, bool ) {} + + void operator()( const std::set< int >&, bool ) {} + + void size() {} +}; + +namespace std { + +template <> struct hash< plan::params > +{ + std::size_t operator()( plan::params const& k ) const + { + std::size_t seed = 0; + boost::hash_combine( seed, k.size ); + boost::hash_combine( seed, k.real ); + boost::hash_combine( seed, k.inverse ); + return seed; + } +}; + +} // namespace std { + +TEST( cached, hashing_non_intrusive ) +{ + comma::cached< plan, plan::params > c; + c.get( std::vector< int >{ 1, 2, 3 }, true, false )( std::vector< int >{ 1, 2, 3 }, true, false ); + EXPECT_EQ( c.values().size(), 1 ); + c.get( std::vector< int >{ 1, 2, 3 }, true, false )( std::vector< int >{ 1, 2, 3 }, true, false ); + EXPECT_EQ( c.values().size(), 1 ); + c.get( std::vector< int >{ 1, 2 }, true, false )( std::vector< int >{ 1, 2 }, true, false ); + EXPECT_EQ( c.values().size(), 2 ); + c.get( std::vector< int >{ 1, 2 }, true, false )( std::vector< int >{ 1, 2 }, true, false ); + c.get( std::vector< int >{ 1, 2 }, true, false ).size(); + EXPECT_EQ( c.values().size(), 2 ); +} + +TEST( cached, operators ) +{ + comma::cached< plan, plan::params > plans; + plans( std::vector< int >{ 1, 2, 3 }, true, false ); + EXPECT_EQ( plans.values().size(), 1 ); + plans( std::vector< int >{ 1, 2, 3 }, true, false ); + EXPECT_EQ( plans.values().size(), 1 ); + plans( std::vector< int >{ 1, 2 }, true, false ); + EXPECT_EQ( plans.values().size(), 2 ); + plans( std::vector< int >{ 1, 2 }, true, false ); + EXPECT_EQ( plans.values().size(), 2 ); + + plans( std::set< int >{ 1, 2 }, true ); + EXPECT_EQ( plans.values().size(), 3 ); + plans( std::set< int >{ 1, 2 }, true ); + EXPECT_EQ( plans.values().size(), 3 ); + plans( std::vector< int >{ 1, 2 }, true, true ); + EXPECT_EQ( plans.values().size(), 3 ); +} + diff --git a/containers/test/cyclic_buffer_test.cpp b/containers/test/cyclic_buffer_test.cpp index 29eb69e8e..cc12f841d 100644 --- a/containers/test/cyclic_buffer_test.cpp +++ b/containers/test/cyclic_buffer_test.cpp @@ -29,6 +29,7 @@ #include +#include "../../base/exception.h" #include "../cyclic_buffer.h" namespace comma { @@ -82,8 +83,7 @@ TEST( cyclic_buffer, push_pop ) EXPECT_EQ( b.size(), 4 - i ); } EXPECT_TRUE( b.empty() ); - EXPECT_EQ( b.size(), 0u ); - + EXPECT_EQ( b.size(), 0u ); for( unsigned int i = 0; i < 5u; ++i ) { b.push( i ); @@ -92,6 +92,28 @@ TEST( cyclic_buffer, push_pop ) EXPECT_EQ( b.size(), 1u ); } +TEST( cyclic_buffer, push_force ) +{ + cyclic_buffer< unsigned int > b( 3 ); + b.push( 0 ); + b.push( 1 ); + b.push( 2 ); + EXPECT_EQ( b.front(), 0 ); + EXPECT_EQ( b.back(), 2 ); + EXPECT_THROW( b.push( 3 ), comma::exception ); + b.push( 3, true ); + EXPECT_EQ( b.front(), 1 ); + EXPECT_EQ( b.back(), 3 ); + EXPECT_THROW( b.push( 4 ), comma::exception ); + b.push( 4, true ); + EXPECT_EQ( b.front(), 2 ); + EXPECT_EQ( b.back(), 4 ); + EXPECT_THROW( b.push( 5 ), comma::exception ); + b.push( 5, true ); + EXPECT_EQ( b.front(), 3 ); + EXPECT_EQ( b.back(), 5 ); +} + TEST( cyclic_buffer, fixed_cyclic_buffer ) { fixed_cyclic_buffer< unsigned int, 3 > b; diff --git a/containers/test/multidimensional_array_test.cpp b/containers/test/multidimensional_array_test.cpp new file mode 100644 index 000000000..8a48b321e --- /dev/null +++ b/containers/test/multidimensional_array_test.cpp @@ -0,0 +1,285 @@ +// Copyright (c) 2023 Vsevolod Vlaskine + +#include +//#include +#include "../multidimensional/array.h" + +namespace cmd = comma::containers::multidimensional; + +TEST( multidimensional_array, impl_index_value ) +{ + EXPECT_EQ( cmd::impl::index_traits< 2 >::value( {0, 0}, {1, 5} ), 0 ); + EXPECT_EQ( cmd::impl::index_traits< 2 >::value( {0, 1}, {1, 5} ), 1 ); + EXPECT_EQ( cmd::impl::index_traits< 2 >::value( {0, 4}, {1, 5} ), 4 ); + EXPECT_EQ( cmd::impl::index_traits< 2 >::value( {1, 0}, {1, 5} ), 5 ); + EXPECT_EQ( cmd::impl::index_traits< 2 >::value( {1, 1}, {2, 5} ), 6 ); + EXPECT_EQ( cmd::impl::index_traits< 2 >::value( {1, 2}, {2, 5} ), 7 ); + EXPECT_EQ( cmd::impl::index_traits< 2 >::value( {1, 2}, {3, 5} ), 7 ); + EXPECT_EQ( cmd::impl::index_traits< 2 >::value( {2, 2}, {3, 5} ), 12 ); + EXPECT_EQ( cmd::impl::index_traits< 3 >::value( {0, 0, 0}, {2, 3, 4} ), 0 ); + EXPECT_EQ( cmd::impl::index_traits< 3 >::value( {0, 0, 3}, {2, 3, 4} ), 3 ); + EXPECT_EQ( cmd::impl::index_traits< 3 >::value( {0, 1, 3}, {2, 3, 4} ), 7 ); + EXPECT_EQ( cmd::impl::index_traits< 3 >::value( {1, 2, 3}, {2, 3, 4} ), 3 + 4 * ( 2 + 3 * 1 ) ); + EXPECT_EQ( cmd::impl::index_traits< 3 >::value( {1, 2, 4}, {2, 3, 4} ), 2 * 3 * 4 ); + EXPECT_EQ( cmd::impl::index_traits< 3 >::product( {2, 3, 4} ), 2 * 3 * 4 ); +} + +TEST( multidimensional_array, impl_index_product ) +{ + EXPECT_EQ( cmd::impl::index_traits< 1 >::product( {2} ), 2 ); + EXPECT_EQ( cmd::impl::index_traits< 2 >::product( {2, 3} ), 6 ); + EXPECT_EQ( cmd::impl::index_traits< 3 >::product( {2, 3, 4} ), 2 * 3 * 4 ); + EXPECT_EQ( cmd::impl::index_traits< 4 >::product( {2, 3, 4, 5} ), 2 * 3 * 4 * 5 ); +} + +TEST( multidimensional_array, impl_index_inverted_value ) +{ + typedef std::array< std::size_t, 3 > array_t; + { array_t a{0, 0, 0}; EXPECT_EQ( cmd::impl::index_traits< 3 >::value( 0, {2, 3, 4} ), a ); } + { array_t a{0, 0, 1}; EXPECT_EQ( cmd::impl::index_traits< 3 >::value( 1, {2, 3, 4} ), a ); } + { array_t a{2, 1, 3}; EXPECT_EQ( cmd::impl::index_traits< 3 >::value( 3 + 4 * ( 1 + 3 * 2 ), {2, 3, 4} ), a ); } + // todo: way more tests +} + +TEST( multidimensional_array, index ) +{ + { + cmd::array< int, 3 > a( {2, 3, 4}, 0 ); + unsigned int i = 0; + for( auto it = a.begin(); it != a.end(); ++it ) { *it = i++; } + typedef cmd::array< int, 3 >::index_type array_index_t; + typedef cmd::index< 3 > index_t; + EXPECT_EQ( ( array_index_t{1, 2, 3} ), ( index_t{1, 2, 3} ) ); + EXPECT_EQ( ( index_t{1, 2, 3} ), ( array_index_t{1, 2, 3} ) ); + EXPECT_EQ( ( a[{1, 2, 3}]), 23 ); + EXPECT_EQ( ( a[ index_t( {1, 2, 3} )] ), 23 ); + EXPECT_EQ( ( a[ index_t( {1, 2, 3} )] ), 23 ); + } +} + +TEST( multidimensional_array, iteration ) +{ + { + typedef std::array< std::size_t, 3 > array_t; + cmd::array< int, 3 > a( {2, 3, 4}, 0 ); + { array_t s{2, 3, 4}; EXPECT_EQ( a.shape(), s ); EXPECT_EQ( a.data().size(), 2 * 3 * 4 ); } + unsigned int i = 0; + for( auto it = a.begin(); it != a.end(); ++it ) { *it = i++; } + i = 0; + for( auto it = a.data().begin(); it != a.data().end(); ++it ) { EXPECT_EQ( *it, i++ ); } + auto it = a.begin(); + { array_t a{0, 0, 0}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 0 ); ++it; } + { array_t a{0, 0, 1}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 1 ); ++it; } + { array_t a{0, 0, 2}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 2 ); ++it; } + { array_t a{0, 0, 3}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 3 ); ++it; } + { array_t a{0, 1, 0}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 4 ); ++it; } + { array_t a{0, 1, 1}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 5 ); ++it; } + { array_t a{0, 1, 2}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 6 ); ++it; } + { array_t a{0, 1, 3}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 7 ); ++it; } + { array_t a{0, 2, 0}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 8 ); ++it; } + { array_t a{0, 2, 1}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 9 ); ++it; } + { array_t a{0, 2, 2}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 10 ); ++it; } + { array_t a{0, 2, 3}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 11 ); ++it; } + { array_t a{1, 0, 0}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 12 ); ++it; } + { array_t a{1, 0, 1}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 13 ); ++it; } + { array_t a{1, 0, 2}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 14 ); ++it; } + { array_t a{1, 0, 3}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 15 ); ++it; } + { array_t a{1, 1, 0}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 16 ); ++it; } + { array_t a{1, 1, 1}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 17 ); ++it; } + { array_t a{1, 1, 2}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 18 ); ++it; } + { array_t a{1, 1, 3}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 19 ); ++it; } + { array_t a{1, 2, 0}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 20 ); ++it; } + { array_t a{1, 2, 1}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 21 ); ++it; } + { array_t a{1, 2, 2}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 22 ); ++it; } + { array_t a{1, 2, 3}; EXPECT_EQ( it.index(), a ); EXPECT_EQ( *it, 23 ); ++it; } + EXPECT_TRUE( it == a.end() ); + } +} + +TEST( multidimensional_array, array ) +{ + { + cmd::array< int, 3 > a( {2, 3, 4}, 0 ); + unsigned int i = 0; + for( auto it = a.begin(); it != a.end(); ++it ) { *it = i++; } + EXPECT_EQ( ( a[{0, 0, 0}] ), 0 ); + EXPECT_EQ( ( a[{0, 1, 0}] ), 4 ); + EXPECT_EQ( ( a[{1, 2, 3}] ), 23 ); + a[{0, 0, 0}] = 111; EXPECT_EQ( ( a[{0, 0, 0}] ), 111 ); + a[{1, 1, 2}] = 222; EXPECT_EQ( ( a[{1, 1, 2}] ), 222 ); + a[{1, 2, 3}] = 333; EXPECT_EQ( ( a[{1, 2, 3}] ), 333 ); + } +} + +TEST( multidimensional_array, slice ) +{ + { + { + cmd::array< int, 3 > a( {2, 3, 4}, 0 ); + unsigned int i = 0; + for( auto it = a.begin(); it != a.end(); ++it ) { *it = i++; } + typedef cmd::array< int, 2 >::index_type index_t; + cmd::slice< int, 2 > s = a.at( 0 ); + { index_t i{0, 0}; EXPECT_EQ( s[i], 0 ); } + { index_t i{0, 1}; EXPECT_EQ( s[i], 1 ); } + { index_t i{0, 2}; EXPECT_EQ( s[i], 2 ); } + { index_t i{0, 3}; EXPECT_EQ( s[i], 3 ); } + { index_t i{1, 0}; EXPECT_EQ( s[i], 4 ); } + { index_t i{1, 1}; EXPECT_EQ( s[i], 5 ); } + { index_t i{1, 2}; EXPECT_EQ( s[i], 6 ); } + { index_t i{1, 3}; EXPECT_EQ( s[i], 7 ); } + { index_t i{2, 0}; EXPECT_EQ( s[i], 8 ); } + { index_t i{2, 1}; EXPECT_EQ( s[i], 9 ); } + { index_t i{2, 2}; EXPECT_EQ( s[i], 10 ); } + { index_t i{2, 3}; EXPECT_EQ( s[i], 11 ); } + { + auto t = s.at( 0 ); + typedef cmd::array< int, 1 >::index_type index_t; + { index_t i{0}; EXPECT_EQ( t[i], 0 ); } + { index_t i{1}; EXPECT_EQ( t[i], 1 ); } + { index_t i{2}; EXPECT_EQ( t[i], 2 ); } + { index_t i{3}; EXPECT_EQ( t[i], 3 ); } + t = s.at( 1 ); + { index_t i{0}; EXPECT_EQ( t[i], 4 ); } + { index_t i{1}; EXPECT_EQ( t[i], 5 ); } + { index_t i{2}; EXPECT_EQ( t[i], 6 ); } + { index_t i{3}; EXPECT_EQ( t[i], 7 ); } + } + s = a.at( 1 ); + { index_t i{0, 0}; EXPECT_EQ( s[i], 12 ); } + { index_t i{0, 1}; EXPECT_EQ( s[i], 13 ); } + { index_t i{0, 2}; EXPECT_EQ( s[i], 14 ); } + { index_t i{0, 3}; EXPECT_EQ( s[i], 15 ); } + { index_t i{1, 0}; EXPECT_EQ( s[i], 16 ); } + { index_t i{1, 1}; EXPECT_EQ( s[i], 17 ); } + { index_t i{1, 2}; EXPECT_EQ( s[i], 18 ); } + { index_t i{1, 3}; EXPECT_EQ( s[i], 19 ); } + { index_t i{2, 0}; EXPECT_EQ( s[i], 20 ); } + { index_t i{2, 1}; EXPECT_EQ( s[i], 21 ); } + { index_t i{2, 2}; EXPECT_EQ( s[i], 22 ); } + { index_t i{2, 3}; EXPECT_EQ( s[i], 23 ); } + s[{1, 3}] = 111; EXPECT_EQ( ( a[{1, 1, 3}] ), 111 ); + } + { + cmd::array< int, 3 > a( {2, 3, 4}, 0 ); + unsigned int i = 0; + for( auto it = a.begin(); it != a.end(); ++it ) { *it = i++; } + typedef cmd::array< int, 1 >::index_type index_t; + { + cmd::slice< int, 1 > s = a.at< 2 >( {0, 0} ); // todo! super-ugly! improve templating! + { index_t i{0}; EXPECT_EQ( s[i], 0 ); } // todo: improve usage on 1-dimensional slices + { index_t i{1}; EXPECT_EQ( s[i], 1 ); } + { index_t i{2}; EXPECT_EQ( s[i], 2 ); } + { index_t i{3}; EXPECT_EQ( s[i], 3 ); } + s = a.at< 2 >( {0, 1} ); + { index_t i{0}; EXPECT_EQ( s[i], 4 ); } + { index_t i{1}; EXPECT_EQ( s[i], 5 ); } + { index_t i{2}; EXPECT_EQ( s[i], 6 ); } + { index_t i{3}; EXPECT_EQ( s[i], 7 ); } + s = a.at< 2 >( {0, 2} ); + { index_t i{0}; EXPECT_EQ( s[i], 8 ); } + { index_t i{1}; EXPECT_EQ( s[i], 9 ); } + { index_t i{2}; EXPECT_EQ( s[i], 10 ); } + { index_t i{3}; EXPECT_EQ( s[i], 11 ); } + s = a.at< 2 >( {1, 0} ); + { index_t i{0}; EXPECT_EQ( s[i], 12 ); } + { index_t i{1}; EXPECT_EQ( s[i], 13 ); } + { index_t i{2}; EXPECT_EQ( s[i], 14 ); } + { index_t i{3}; EXPECT_EQ( s[i], 15 ); } + s = a.at< 2 >( {1, 1} ); + { index_t i{0}; EXPECT_EQ( s[i], 16 ); } + { index_t i{1}; EXPECT_EQ( s[i], 17 ); } + { index_t i{2}; EXPECT_EQ( s[i], 18 ); } + { index_t i{3}; EXPECT_EQ( s[i], 19 ); } + s = a.at< 2 >( {1, 2} ); + { index_t i{0}; EXPECT_EQ( s[i], 20 ); } + { index_t i{1}; EXPECT_EQ( s[i], 21 ); } + { index_t i{2}; EXPECT_EQ( s[i], 22 ); } + { index_t i{3}; EXPECT_EQ( s[i], 23 ); } + } + } + } +} + +TEST( multidimensional_array, grid_index ) +{ + { + cmd::grid< double, 2 > g( {0, 0}, {1, 1}, {2, 3}, 0 ); + typedef std::array< std::size_t, 2 > index_t; + int i = 0; + for( auto it = g.begin(); it != g.end(); ++it ) { *it = i++; } + { index_t i = {0, 0}; EXPECT_EQ( g.index_of( {0, 0} ), i ); } + { index_t i = {0, 1}; EXPECT_EQ( g.index_of( {0, 1} ), i ); } + { index_t i = {1, 0}; EXPECT_EQ( g.index_of( {1, 0} ), i ); } + { index_t i = {1, 1}; EXPECT_EQ( g.index_of( {1, 1} ), i ); } + { index_t i = {0, 1}; EXPECT_EQ( g.index_of( {0, 1.01} ), i ); } + // todo: more tests + } +} + +TEST( multidimensional_array, grid_interpolate ) +{ + { + cmd::grid< double, 2 > g( {0, 0}, {1, 1}, {2, 2}, 0 ); + g[{0, 0}] = 0; g[{0, 1}] = 1; g[{1, 0}] = 0; g[{1, 1}] = 1; + EXPECT_EQ( g.interpolated( {0, 0} ), 0 ); + EXPECT_EQ( g.interpolated( {0, 0.5} ), 0.5 ); + EXPECT_EQ( g.interpolated( {0.5, 0.5} ), 0.5 ); + //EXPECT_EQ( g.interpolated( {0.5, 0} ), 0.5 ); + //EXPECT_EQ( g.interpolated( {1, 0} ), 2 ); + //EXPECT_EQ( g.interpolated( {1, 1} ), 3 ); + } +} + +// TEST( vector_of_vectors, performance ) +// { +// //std::pair< unsigned int, unsigned int > size{ 10000000, 4 }; +// std::pair< unsigned int, unsigned int > size{ 4096, 8 }; +// std::vector< std::vector< float > > a( size.first, std::vector< float >( size.second, 0 ) ); +// std::vector< float > b( size.first * size.second, 0 ); +// { +// auto t0 = boost::posix_time::microsec_clock::universal_time(); +// for( auto& c: a ) +// { +// for( auto& d: c ) +// { +// d += 10; +// } +// } +// auto t1 = boost::posix_time::microsec_clock::universal_time(); +// for( auto& d: b ) +// { +// d += 10; +// } +// auto t2 = boost::posix_time::microsec_clock::universal_time(); +// auto e0 = double( ( t1 - t0 ).total_microseconds() ) / 1e6; +// auto e1 = double( ( t2 - t1 ).total_microseconds() ) / 1e6; +// //std::cerr << "==> cashe hits:\tspeedup: " << ( e0 / e1 ) << "\tvector of vectors: elapsed: " << e0 << "\tvector: elapsed: " << e1 << std::endl; +// std::cerr << "==> cashe hits:\tspeedup: " << ( e0 / e1 ) << std::endl; +// } +// { +// std::vector< float > z( size.first * size.second, 0 ); +// auto t0 = boost::posix_time::microsec_clock::universal_time(); +// for( unsigned int i = 0; i < b.size(); ++i ) { z[i] = b[i]; } +// auto t1 = boost::posix_time::microsec_clock::universal_time(); +// std::memcpy( reinterpret_cast< char* >( &z[0] ), reinterpret_cast< const char* >( &b[0] ), b.size() * sizeof( b[0] ) ); +// auto t2 = boost::posix_time::microsec_clock::universal_time(); +// auto e0 = double( ( t1 - t0 ).total_microseconds() ) / 1e6; +// auto e1 = double( ( t2 - t1 ).total_microseconds() ) / 1e6; +// //std::cerr << "==> memcpy:\tspeedup: " << ( e0 / e1 ) << "\tvector of vectors: elapsed: " << e0 << "\tvector: elapsed: " << e1 << std::endl; +// std::cerr << "==> memcpy vs element-wise assignment:\tspeedup: " << ( e0 / e1 ) << std::endl; +// } +// { +// std::vector< float > z( size.first * size.second, 0 ); +// auto t0 = boost::posix_time::microsec_clock::universal_time(); +// std::copy( b.begin(), b.end(), z.begin() ); +// auto t1 = boost::posix_time::microsec_clock::universal_time(); +// std::memcpy( reinterpret_cast< char* >( &z[0] ), reinterpret_cast< const char* >( &b[0] ), b.size() * sizeof( b[0] ) ); +// auto t2 = boost::posix_time::microsec_clock::universal_time(); +// auto e0 = double( ( t1 - t0 ).total_microseconds() ) / 1e6; +// auto e1 = double( ( t2 - t1 ).total_microseconds() ) / 1e6; +// //std::cerr << "==> memcpy:\tspeedup: " << ( e0 / e1 ) << "\tvector of vectors: elapsed: " << e0 << "\tvector: elapsed: " << e1 << std::endl; +// std::cerr << "==> memcpy vs std::copy:\tspeedup: " << ( e0 / e1 ) << std::endl; +// } +// // todo! multidimensional::array performance +// } diff --git a/containers/test/multidimensional_index_test.cpp b/containers/test/multidimensional_index_test.cpp new file mode 100644 index 000000000..186ac816b --- /dev/null +++ b/containers/test/multidimensional_index_test.cpp @@ -0,0 +1,80 @@ +// Copyright (c) 2023 Vsevolod Vlaskine + +#include +//#include +#include "../multidimensional/index.h" + +namespace cmd = comma::containers::multidimensional; + +TEST( multidimentional_index, basics ) +{ + { + cmd::index< 4 > i; + EXPECT_EQ( i[0], 0 ); + EXPECT_EQ( i[1], 0 ); + EXPECT_EQ( i[2], 0 ); + EXPECT_EQ( i[3], 0 ); + } + { + cmd::index< 4 > i{}; + EXPECT_EQ( i[0], 0 ); + EXPECT_EQ( i[1], 0 ); + EXPECT_EQ( i[2], 0 ); + EXPECT_EQ( i[3], 0 ); + } + { + cmd::index< 4 > i{0, 1, 2, 3}; + EXPECT_EQ( i[0], 0 ); + EXPECT_EQ( i[1], 1 ); + EXPECT_EQ( i[2], 2 ); + EXPECT_EQ( i[3], 3 ); + } + { + cmd::index< 1 > i; + cmd::index< 1 > j{5}; + EXPECT_TRUE( i < j ); + EXPECT_TRUE( i != j ); + EXPECT_EQ( i.increment( j ), cmd::index< 1 >{1} ); + EXPECT_EQ( i.increment( j ), cmd::index< 1 >{2} ); + } + { + cmd::index< 2 > i; + cmd::index< 2 > j{3, 2}; + EXPECT_EQ( i , ( cmd::index< 2 >{0, 0} ) ); + EXPECT_TRUE( i < j ); + EXPECT_EQ( i.increment( j ), ( cmd::index< 2 >{0, 1} ) ); + EXPECT_TRUE( i < j ); + EXPECT_EQ( i.increment( j ), ( cmd::index< 2 >{1, 0} ) ); + EXPECT_TRUE( i < j ); + EXPECT_EQ( i.increment( j ), ( cmd::index< 2 >{1, 1} ) ); + EXPECT_TRUE( i < j ); + EXPECT_EQ( i.increment( j ), ( cmd::index< 2 >{2, 0} ) ); + EXPECT_TRUE( i < j ); + EXPECT_EQ( i.increment( j ), ( cmd::index< 2 >{2, 1} ) ); + EXPECT_TRUE( i < j ); + EXPECT_EQ( i.increment( j ), ( cmd::index< 2 >{0, 0} ) ); + EXPECT_TRUE( i == cmd::index< 2 >{} ); + } + { + cmd::index< 2 >::iterator i{{3u, 2u}}; + EXPECT_TRUE( bool( i ) ); + EXPECT_EQ( *i, ( cmd::index< 2 >{0u, 0u} ) ); + EXPECT_TRUE( bool( ++i ) ); + EXPECT_EQ( *i, ( cmd::index< 2 >{0u, 1u} ) ); + EXPECT_TRUE( bool( ++i ) ); + EXPECT_EQ( *i, ( cmd::index< 2 >{1u, 0u} ) ); + EXPECT_TRUE( bool( ++i ) ); + EXPECT_EQ( *i, ( cmd::index< 2 >{1u, 1u} ) ); + EXPECT_TRUE( bool( ++i ) ); + EXPECT_EQ( *i, ( cmd::index< 2 >{2u, 0u} ) ); + EXPECT_TRUE( bool( ++i ) ); + EXPECT_EQ( *i, ( cmd::index< 2 >{2u, 1u} ) ); + EXPECT_FALSE( bool( ++i ) ); + EXPECT_EQ( *i, ( cmd::index< 2 >{0u, 0u} ) ); + } + { + unsigned int count{0}; + for( cmd::index< 2 >::iterator i{{3u, 2u}}; i; ++i, ++count ); + EXPECT_EQ( count, 6 ); + } +} diff --git a/containers/test/multidimensional_map_test.cpp b/containers/test/multidimensional_map_test.cpp new file mode 100644 index 000000000..e0516f19d --- /dev/null +++ b/containers/test/multidimensional_map_test.cpp @@ -0,0 +1,141 @@ +// Copyright (c) 2023 Vsevolod Vlaskine + +#include +#include "../multidimensional/map.h" + +TEST( multikey_map, usage ) +{ + comma::containers::multidimensional::map< double, int, 3 > m( { 0, 0, 0 }, { 1, 2, 3 } ); + m.touch_at( { 1, 2, 3 } ); + // todo +} + +TEST( multikey_map, index ) +{ + typedef comma::containers::multidimensional::map< double, int, 3 > map_type; + { + map_type m( {1, 1, 1} ); + { + map_type::index_type i = {{ 0, 0, 0 }}; + EXPECT_EQ( i, m.index_of( {0., 0., 0.} ) ); + EXPECT_EQ( i, m.index_of( {0.001, 0.001, 0.001} ) ); + EXPECT_EQ( i, m.index_of( {0.999, 0.999, 0.999} ) ); + } + { + map_type::index_type i = {{ 1, 1, 1 }}; + EXPECT_EQ( i, m.index_of( {1.0, 1.0, 1.0} ) ); + EXPECT_EQ( i, m.index_of( {1.001, 1.001, 1.001} ) ); + EXPECT_EQ( i, m.index_of( {1.999, 1.999, 1.999} ) ); + } + { + map_type::index_type i = {{ -1, -1, -1 }}; + EXPECT_EQ( i, m.index_of( {-1.0, -1.0, -1.0} ) ); + EXPECT_EQ( i, m.index_of( {-0.999, -0.999, -0.999} ) ); + EXPECT_EQ( i, m.index_of( {-0.001, -0.001, -0.001} ) ); + } + } + { + map_type m( {0.3, 0.3, 0.3} ); + { + map_type::index_type i = {{ 0, 0, 0 }}; + EXPECT_EQ( i, m.index_of( {0, 0, 0} ) ); + EXPECT_EQ( i, m.index_of( {0.001, 0.001, 0.001} ) ); + EXPECT_EQ( i, m.index_of( {0.299, 0.299, 0.299} ) ); + } + { + map_type::index_type i = {{ 1, 1, 1 }}; + EXPECT_EQ( i, m.index_of( {0.3, 0.3, 0.3} ) ); + EXPECT_EQ( i, m.index_of( {0.3001, 0.3001, 0.3001} ) ); + EXPECT_EQ( i, m.index_of( {0.3999, 0.3999, 0.3999} ) ); + } + { + map_type::index_type i = {{ -1, -1, -1 }}; + EXPECT_EQ( i, m.index_of( {-0.3, -0.3, -0.3} ) ); + EXPECT_EQ( i, m.index_of( {-0.299, -0.299, -0.299} ) ); + EXPECT_EQ( i, m.index_of( {-0.001, -0.001, -0.001} ) ); + } + } +} + +TEST( multikey_map, operations ) +{ + typedef comma::containers::multidimensional::map< double, int, 3 > map_type; + map_type m( {1, 1, 1} ); + { + EXPECT_TRUE( ( m.at( map_type::point_type{1., 1., 1.} ) == m.end() ) ); + EXPECT_TRUE( ( m.touch_at( map_type::point_type{1., 1., 1.} ) != m.end() ) ); + EXPECT_EQ( 1, m.size() ); + EXPECT_TRUE( ( m.at( map_type::point_type{1., 1., 1.} ) != m.end() ) ); + EXPECT_TRUE( ( m.at( map_type::point_type{1., 1., 1.} ) == m.at( map_type::point_type{1.1, 1.1, 1.1} ) ) ); + EXPECT_TRUE( ( m.touch_at( {1, 1, 1} ) != m.end() ) ); + EXPECT_EQ( 1, m.size() ); + EXPECT_TRUE( ( m.touch_at( {1.1, 1.1, 1.1} ) != m.end() ) ); + EXPECT_EQ( 1, m.size() ); + } + { + EXPECT_TRUE( ( m.at( map_type::point_type{-1., -1., -1.} ) == m.end() ) ); + EXPECT_TRUE( ( m.touch_at( {-1., -1., -1.} ) != m.end() ) ); + EXPECT_EQ( 2, m.size() ); + EXPECT_TRUE( ( m.at( map_type::point_type{-1., -1., -1.} ) != m.end() ) ); + EXPECT_TRUE( ( m.at( map_type::point_type{-1., -1., -1.} ) == m.at( map_type::point_type{-0.1, -0.1, -0.1} ) ) ); + EXPECT_TRUE( ( m.touch_at( {-1., -1., -1.} ) != m.end() ) ); + EXPECT_EQ( 2, m.size() ); + EXPECT_TRUE( ( m.touch_at( {-0.1, -0.1, -0.1} ) != m.end() ) ); + EXPECT_EQ( 2, m.size() ); + } + { + EXPECT_TRUE( ( m.at( map_type::point_type{0., 0., 0.} ) == m.end() ) ); + EXPECT_TRUE( ( m.touch_at( {0., 0., 0.} ) != m.end() ) ); + EXPECT_EQ( 3, m.size() ); + EXPECT_TRUE( ( m.at( map_type::point_type{0., 0, 0} ) != m.end() ) ); + EXPECT_TRUE( ( m.at( map_type::point_type{0., 0, 0} ) == m.at( map_type::point_type{0.1, 0.1, 0.1} ) ) ); + EXPECT_TRUE( ( m.touch_at( {0., 0, 0} ) != m.end() ) ); + EXPECT_EQ( 3, m.size() ); + EXPECT_TRUE( ( m.touch_at( {0.1, 0.1, 0.1} ) != m.end() ) ); + EXPECT_EQ( 3, m.size() ); + } +} + +TEST( multikey_map, test ) +{ + typedef comma::containers::multidimensional::map< double, int, 3 > map_type; + map_type m( {1, 1, 1} ); + EXPECT_TRUE( m.empty() ); +} + +TEST( multikey_map, neighbourhood ) +{ + typedef comma::containers::multidimensional::map< double, int, 3 > map_type; + map_type m( {1, 1, 1} ); + { + EXPECT_TRUE( ( m.at( map_type::point_type{1, 1, 1} ) == m.end() ) ); + { + EXPECT_TRUE( ( m.touch_at( {1, 1, 1} ) != m.end() ) ); + EXPECT_EQ( 1, m.size() ); + m.touch_at( {1, 1, 1} )->second = 111; + EXPECT_EQ( 111, m.at( map_type::point_type{1, 1, 1} )->second ); + map_type::index_type index = {{ 1, 1, 1 }}; + EXPECT_EQ( 111, m.base_type::find( index )->second ); + } + { + EXPECT_TRUE( ( m.touch_at( {2, 2, 2} ) != m.end() ) ); + EXPECT_EQ( 2, m.size() ); + m.touch_at( {2, 2, 2} )->second = 222; + EXPECT_EQ( 222, m.at( map_type::point_type{2, 2, 2} )->second ); + map_type::index_type index = {{ 2, 2, 2 }}; + EXPECT_EQ( 222, m.base_type::find( index )->second ); + } + { + map_type::index_type index = {{ -1, 0, 0 }}; + EXPECT_TRUE( m.base_type::find( index ) == m.end() ); + } + { + map_type::index_type index = {{ 0, 0, 0 }}; + EXPECT_TRUE( m.base_type::find( index ) == m.end() ); + } + { + map_type::index_type index = {{ 2, 2, 3 }}; + EXPECT_TRUE( m.base_type::find( index ) == m.end() ); + } + } +} diff --git a/containers/test/ordered_queues_test.cpp b/containers/test/ordered_queues_test.cpp new file mode 100644 index 000000000..923197ea7 --- /dev/null +++ b/containers/test/ordered_queues_test.cpp @@ -0,0 +1,202 @@ +// Copyright (c) 2023 Mission Systems Pty Ltd + +#include +#include +#include "../ordered/queues.h" + +TEST( queues, usage ) +{ + typedef comma::containers::ordered::queues< int, int, int > queues_t; + queues_t q{ 2 /*timeout*/ }; + + EXPECT_EQ( std::get<0>(q).size(), 0 ); + EXPECT_EQ( std::get<1>(q).size(), 0 ); + EXPECT_EQ( q.ready(), false ); + + std::get<0>(q).push_back( std::make_pair( 0, 1 ) ); + EXPECT_EQ( std::get<0>(q).front().second, 1 ); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 0 ); + EXPECT_EQ( q.ready(), false ); + + std::get<1>(q).push_back( std::make_pair( 0, 1 ) ); + EXPECT_EQ( std::get<1>(q).front().second, 1 ); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( q.ready(), true ); + + // Purge should only remove items if they are unsynced + q.purge(); + EXPECT_EQ( std::get<0>(q).front().second, 1 ); + EXPECT_EQ( std::get<1>(q).front().second, 1 ); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( q.ready(), true ); +} + +TEST( queues, sync_first_to_second ){ + typedef comma::containers::ordered::queues< float, int, int > queues_t; + queues_t q{ 2 /*timeout*/ }; + + std::get<0>(q) = std::deque< std::pair< float, int > >({ {0, 0}, {2, 0}, {4, 5} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {5, 5} }); + EXPECT_EQ( q.ready(), false ); + q.purge(); + EXPECT_EQ( std::get<0>(q).front().second, 5 ); + EXPECT_EQ( std::get<1>(q).front().second, 5 ); + EXPECT_EQ( std::get<0>(q).front().first, 4 ); + EXPECT_EQ( std::get<1>(q).front().first, 5 ); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( q.ready(), true ); + +} + +TEST( queues, sync_second_to_first ){ + typedef comma::containers::ordered::queues< float, int, int > queues_t; + queues_t q{ 2 /*timeout*/ }; + + std::get<0>(q) = std::deque< std::pair< float, int > >({ {5, 5} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {1, 0}, {2, 0}, {4, 5} }); + EXPECT_EQ( q.ready(), false ); + q.purge(); + EXPECT_EQ( std::get<0>(q).front().second, 5 ); + EXPECT_EQ( std::get<1>(q).front().second, 5 ); + EXPECT_EQ( std::get<0>(q).front().first, 5 ); + EXPECT_EQ( std::get<1>(q).front().first, 4 ); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( q.ready(), true ); + +} + +TEST( queues, empty_list_before_sync ){ + typedef comma::containers::ordered::queues< float, int, int > queues_t; + queues_t q{ 2 /*timeout*/ }; + std::get<0>(q) = std::deque< std::pair< float, int > >({ {0, 0}, {1, 0}, {2, 5} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {5, 5} }); + q.purge(); + EXPECT_EQ( std::get<0>(q).size(), 0 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( q.ready(), false ); +} + +TEST( queues, sync_and_pop ){ + typedef comma::containers::ordered::queues< float, int, int > queues_t; + queues_t q{ 0.9 /*timeout*/ }; + std::get<0>(q) = std::deque< std::pair< float, int > >({ {0, 1}, {1, 1}, {2, 1}, {3, 1}, {4, 1}, {5, 1} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {3, 2}, {5, 2}}); + q.purge(); + EXPECT_EQ( std::get<0>(q).size(), 3 ); // {3, 1}, {4, 1}, {5, 1} }); + EXPECT_EQ( std::get<1>(q).size(), 2 ); // {3, 2}, {5, 2}}); + EXPECT_EQ( std::get<0>(q).front().first, 3 ); + EXPECT_EQ( std::get<1>(q).front().first, 3 ); + EXPECT_EQ( q.ready(), true ); + + auto data = q.front(); + q.pop_all(); + EXPECT_EQ( std::get<0>(data).first, 3 ); + EXPECT_EQ( std::get<1>(data).first, 3 ); + EXPECT_EQ( std::get<0>(q).size(), 2 ); // {4, 1}, {5, 1} }); + EXPECT_EQ( std::get<1>(q).size(), 1 ); // {5, 2}}); + EXPECT_EQ( std::get<0>(q).front().first, 4 ); + EXPECT_EQ( std::get<1>(q).front().first, 5 ); + + EXPECT_EQ( q.ready(), false ); + q.purge(); + EXPECT_EQ( q.ready(), true ); + + EXPECT_EQ( std::get<0>(q).size(), 1 ); // {5, 1} }); + EXPECT_EQ( std::get<1>(q).size(), 1 ); // {5, 2}}); + EXPECT_EQ( std::get<0>(q).front().first, 5 ); + EXPECT_EQ( std::get<1>(q).front().first, 5 ); + EXPECT_EQ( q.ready(), true ); +} + +TEST( queues, max_time_offset ){ + typedef comma::containers::ordered::queues< float, int, int > queues_t; + queues_t q{ 2 /*timeout*/ }; + std::get<0>(q) = std::deque< std::pair< float, int > >({ {3, 5} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {5, 5} }); + q.purge(); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( std::get<0>(q).front().second, 5 ); + EXPECT_EQ( std::get<1>(q).front().second, 5 ); + EXPECT_EQ( std::get<0>(q).front().first, 3 ); + EXPECT_EQ( std::get<1>(q).front().first, 5 ); + EXPECT_EQ( q.ready(), true ); +} + +TEST( queues, floating_point_error ){ + typedef comma::containers::ordered::queues< float, int, int > queues_t; + { + queues_t q{ 2 /*timeout*/ }; + std::get<0>(q) = std::deque< std::pair< float, int > >({ {3.000001, 5} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {5, 5} }); + q.purge(); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( std::get<0>(q).front().second, 5 ); + EXPECT_EQ( std::get<1>(q).front().second, 5 ); + EXPECT_NEAR( std::get<0>(q).front().first, 3.000001, 1e-6 ); + EXPECT_EQ( std::get<1>(q).front().first, 5 ); + EXPECT_EQ( q.ready(), true ); + } + { + queues_t q{ 2 /*timeout*/ }; + std::get<0>(q) = std::deque< std::pair< float, int > >({ {2.999999, 5} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {5, 5} }); + q.purge(); + EXPECT_EQ( std::get<0>(q).size(), 0 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).front().second, 5 ); + EXPECT_EQ( std::get<1>(q).front().first, 5 ); + EXPECT_EQ( q.ready(), false ); + } + { + queues_t q{ 2 /*timeout*/ }; + std::get<0>(q) = std::deque< std::pair< float, int > >({ {5, 5} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {3.000001, 5} }); + q.purge(); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( std::get<0>(q).front().second, 5 ); + EXPECT_EQ( std::get<1>(q).front().second, 5 ); + EXPECT_EQ( std::get<0>(q).front().first, 5 ); + EXPECT_NEAR( std::get<1>(q).front().first, 3.000001, 1e-6 ); + EXPECT_EQ( q.ready(), true ); + } + { + queues_t q{ 2 /*timeout*/ }; + std::get<0>(q) = std::deque< std::pair< float, int > >({ {5, 5} }); + std::get<1>(q) = std::deque< std::pair< float, int > >({ {2.999999, 5} }); + q.purge(); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 0 ); + EXPECT_EQ( std::get<0>(q).front().second, 5 ); + EXPECT_EQ( std::get<0>(q).front().first, 5 ); + EXPECT_EQ( q.ready(), false ); + } +} + +TEST( queues, type_difference ) +{ + typedef comma::containers::ordered::queues< float, int, double > queues_t; + queues_t q{ 2 /*timeout*/ }; + std::get<0>(q) = std::deque< std::pair< float, int > >({ {0, 1} }); + std::get<1>(q) = std::deque< std::pair< float, double > >({ {0, 1.0} }); +} + +TEST( queues, boost_time ) +{ + typedef comma::containers::ordered::queues< boost::posix_time::ptime, double, double > queues_t; + queues_t q{boost::posix_time::seconds( 2 /*timeout*/ ) }; + + boost::posix_time::ptime t( boost::gregorian::date( 2023, 1, 1 ) ); + std::get<0>(q) = std::deque< std::pair< boost::posix_time::ptime, double > >({ {t, 1.0} }); + std::get<1>(q) = std::deque< std::pair< boost::posix_time::ptime, double > >({ {t, 1.0} }); + EXPECT_EQ( std::get<0>(q).size(), 1 ); + EXPECT_EQ( std::get<1>(q).size(), 1 ); + EXPECT_EQ( q.ready(), true ); +} diff --git a/containers/vector.h b/containers/vector.h index 342187fbb..091c550f0 100644 --- a/containers/vector.h +++ b/containers/vector.h @@ -1,37 +1,8 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine -#ifndef COMMA_CONTAINERS_VECTOR_H_ -#define COMMA_CONTAINERS_VECTOR_H_ +#pragma once #include #include @@ -69,5 +40,3 @@ struct regular_vector : public std::vector< T > }; } // namespace comma { - -#endif // COMMA_CONTAINERS_VECTOR_H_ diff --git a/csv/CMakeLists.txt b/csv/CMakeLists.txt index 3de46ff97..ec9d76ec4 100644 --- a/csv/CMakeLists.txt +++ b/csv/CMakeLists.txt @@ -11,7 +11,7 @@ SOURCE_GROUP( ${TARGET_NAME} FILES ${source} ${includes} ${impl_includes} ) ADD_LIBRARY( ${TARGET_NAME} ${source} ${includes} ${impl_source} ${impl_includes} ) SET_TARGET_PROPERTIES( ${TARGET_NAME} PROPERTIES ${comma_LIBRARY_PROPERTIES} ) -TARGET_LINK_LIBRARIES( ${TARGET_NAME} comma_application comma_xpath ${comma_ALL_EXTERNAL_LIBRARIES} ) +target_link_libraries( ${TARGET_NAME} comma_application comma_timing comma_xpath ${comma_ALL_EXTERNAL_LIBRARIES} ) INSTALL( FILES ${includes} DESTINATION ${comma_INSTALL_INCLUDE_DIR}/${PROJECT}/ ) INSTALL( FILES ${impl_includes} DESTINATION ${comma_INSTALL_INCLUDE_DIR}/${PROJECT}/impl ) diff --git a/csv/applications/CMakeLists.txt b/csv/applications/CMakeLists.txt index 7cd1fb6c4..9a3b6fa06 100644 --- a/csv/applications/CMakeLists.txt +++ b/csv/applications/CMakeLists.txt @@ -7,11 +7,17 @@ SOURCE_GROUP( ${TARGET_NAME} FILES ${play_source} ${play_includes} ${source} ${i add_executable( csv-quote csv-quote.cpp ) target_link_libraries( csv-quote ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv ) +set_target_properties( csv-quote PROPERTIES LINK_FLAGS_RELEASE -s ) install( TARGETS csv-quote RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) add_executable( csv-fields ${dir}/csv-fields.cpp ) +target_link_libraries ( csv-fields ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv ) +set_target_properties( csv-fields PROPERTIES LINK_FLAGS_RELEASE -s ) +install( TARGETS csv-fields RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) + add_executable( csv-format ${dir}/csv-format.cpp ) add_executable( csv-size ${dir}/csv-size.cpp ) +add_executable( csv-seek ${dir}/csv-seek.cpp ) add_executable( csv-select ${dir}/csv-select.cpp ) add_executable( csv-bin-cut ${dir}/csv-bin-cut.cpp ) add_executable( csv-from-columns ${dir}/csv-from-columns.cpp ) @@ -28,39 +34,64 @@ add_executable( csv-from-bin ${dir}/csv-from-bin.cpp ) add_executable( csv-calc ${dir}/csv-calc.cpp ) add_executable( csv-calc-new ${dir}/csv-calc.new.cpp ) add_executable( csv-crc ${dir}/csv-crc.cpp ) -add_executable( csv-play ${dir}/csv-play.cpp ${dir}/play/multiplay.cpp ${dir}/play/play.cpp ) +add_executable( csv-play ${dir}/csv-play.cpp ${dir}/play/multiplay.h ${dir}/play/multiplay.cpp ${dir}/play/play.h ${dir}/play/play.cpp ) add_executable( csv-shape ${dir}/csv-shape.cpp ) add_executable( csv-shuffle ${dir}/csv-shuffle.cpp ) add_executable( csv-thin ${dir}/csv-thin.cpp ) add_executable( csv-analyse ${dir}/csv-analyse.cpp ) add_executable( csv-to-sql ${dir}/csv-to-sql.cpp ) -target_link_libraries ( csv-fields ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv ) target_link_libraries ( csv-format ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv ) target_link_libraries ( csv-size ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv ) target_link_libraries ( csv-bin-cut ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv comma_xpath ) -target_link_libraries ( csv-split comma_csv comma_application comma_io comma_string comma_xpath ${comma_ALL_EXTERNAL_LIBRARIES} ) +target_link_libraries ( csv-split comma_csv comma_application comma_io comma_string comma_xpath comma_name_value ${comma_ALL_EXTERNAL_LIBRARIES} ) target_link_libraries ( csv-from-columns ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_io comma_string ) -target_link_libraries ( csv-join ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_xpath comma_string ) +target_link_libraries ( csv-join ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_xpath comma_string comma_name_value ) target_link_libraries ( csv-sort ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_xpath comma_string ) -target_link_libraries ( csv-select ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_xpath comma_string ) -target_link_libraries ( csv-paste ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv comma_io ) -target_link_libraries ( csv-time ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_xpath comma_string ) +target_link_libraries ( csv-seek ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_xpath comma_string comma_name_value ) +target_link_libraries ( csv-select ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_xpath comma_string comma_name_value ) +target_link_libraries ( csv-paste ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv comma_io comma_name_value ) +target_link_libraries ( csv-time ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_xpath comma_string comma_timing ) target_link_libraries ( csv-time-delay ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_string comma_xpath ) -target_link_libraries ( csv-time-join ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_string comma_xpath ) +target_link_libraries ( csv-time-join ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_string comma_xpath comma_name_value ) target_link_libraries ( csv-time-stamp ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_string ) target_link_libraries ( csv-to-bin ${comma_ALL_EXTERNAL_LIBRARIES} comma_csv comma_xpath comma_application ) # profiler ) target_link_libraries ( csv-from-bin ${comma_ALL_EXTERNAL_LIBRARIES} comma_csv comma_xpath comma_application ) target_link_libraries ( csv-calc ${comma_ALL_EXTERNAL_LIBRARIES} comma_csv comma_xpath comma_application comma_string ) target_link_libraries ( csv-calc-new ${comma_ALL_EXTERNAL_LIBRARIES} comma_csv comma_xpath comma_application comma_string ) target_link_libraries ( csv-crc ${comma_ALL_EXTERNAL_LIBRARIES} comma_csv comma_xpath comma_application comma_string ) -target_link_libraries ( csv-play ${comma_ALL_EXTERNAL_LIBRARIES} comma_csv comma_xpath comma_application comma_io ) +target_link_libraries ( csv-play ${comma_ALL_EXTERNAL_LIBRARIES} comma_csv comma_xpath comma_application comma_io comma_name_value ) target_link_libraries ( csv-shape ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv ) target_link_libraries ( csv-shuffle ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv ) target_link_libraries ( csv-thin ${comma_ALL_EXTERNAL_LIBRARIES} comma_csv comma_xpath comma_application comma_io ) target_link_libraries ( csv-analyse ${comma_ALL_EXTERNAL_LIBRARIES} comma_application ) target_link_libraries ( csv-to-sql ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_csv ) +set_target_properties( csv-bin-cut PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-format PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-join PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-sort PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-from-columns PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-paste PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-split PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-time PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-time-delay PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-time-join PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-time-stamp PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-to-bin PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-from-bin PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-size PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-calc PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-play PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-shape PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-shuffle PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-crc PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-seek PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-select PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-thin PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-analyse PROPERTIES LINK_FLAGS_RELEASE -s ) +set_target_properties( csv-to-sql PROPERTIES LINK_FLAGS_RELEASE -s ) + install( TARGETS csv-bin-cut csv-fields csv-format @@ -81,6 +112,7 @@ install( TARGETS csv-bin-cut csv-shape csv-shuffle csv-crc + csv-seek csv-select csv-thin csv-analyse @@ -92,34 +124,57 @@ install ( PROGRAMS csv-gate DESTINATION ${comma_INSTALL_BIN_DIR} ) add_executable( csv-blocks ${dir}/csv-blocks.cpp ) target_link_libraries ( csv-blocks ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_io comma_string comma_xpath comma_csv ) +set_target_properties( csv-blocks PROPERTIES LINK_FLAGS_RELEASE -s ) install( TARGETS csv-blocks RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) add_executable( csv-cast ${dir}/csv-cast.cpp ) target_link_libraries ( csv-cast ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv ) +set_target_properties( csv-cast PROPERTIES LINK_FLAGS_RELEASE -s ) install( TARGETS csv-cast RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) add_executable( csv-enumerate ${dir}/csv-enumerate.cpp ) target_link_libraries ( csv-enumerate ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_io comma_string comma_xpath comma_csv ) +set_target_properties( csv-enumerate PROPERTIES LINK_FLAGS_RELEASE -s ) install( TARGETS csv-enumerate RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) -add_executable( csv-interval ${dir}/csv-interval.cpp ) -target_link_libraries ( csv-interval ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_xpath ) -install( TARGETS csv-interval RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) +add_executable( csv-intervals ${dir}/csv-intervals.cpp ) +target_link_libraries ( csv-intervals ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_xpath comma_name_value ) +set_target_properties( csv-intervals PROPERTIES LINK_FLAGS_RELEASE -s ) +install( TARGETS csv-intervals RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) add_executable( csv-units ${dir}/csv-units.cpp ) target_link_libraries ( csv-units ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_io comma_xpath comma_string ) +set_target_properties( csv-units PROPERTIES LINK_FLAGS_RELEASE -s ) install( TARGETS csv-units RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) +add_executable( csv-random ${dir}/csv-random.cpp ) +target_link_libraries ( csv-random ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_io comma_string comma_xpath comma_csv ) +set_target_properties( csv-random PROPERTIES LINK_FLAGS_RELEASE -s ) +install( TARGETS csv-random RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) + +add_executable( csv-strings ${dir}/csv-strings.cpp ) +target_link_libraries ( csv-strings ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_io comma_string comma_xpath comma_csv ) +set_target_properties( csv-strings PROPERTIES LINK_FLAGS_RELEASE -s ) +install( TARGETS csv-strings RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) + add_executable( csv-update ${dir}/csv-update.cpp ) target_link_libraries ( csv-update ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_io comma_string comma_xpath comma_csv ) +set_target_properties( csv-update PROPERTIES LINK_FLAGS_RELEASE -s ) install( TARGETS csv-update RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) if( NOT WIN32 ) add_executable( csv-repeat ${dir}/csv-repeat.cpp ) target_link_libraries ( csv-repeat comma_application comma_csv comma_io ) + set_target_properties( csv-repeat PROPERTIES LINK_FLAGS_RELEASE -s ) install( TARGETS csv-repeat RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) endif() add_executable( csv-bin-reverse ${dir}/csv-bin-reverse.cpp ) target_link_libraries ( csv-bin-reverse ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_string comma_xpath comma_csv ) +set_target_properties( csv-bin-reverse PROPERTIES LINK_FLAGS_RELEASE -s ) install( TARGETS csv-bin-reverse RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) + +add_executable( csv-bits ${dir}/csv-bits.cpp ) +target_link_libraries ( csv-bits ${comma_ALL_EXTERNAL_LIBRARIES} comma_application comma_csv comma_string ) +set_target_properties( csv-bits PROPERTIES LINK_FLAGS_RELEASE -s ) +install( TARGETS csv-bits RUNTIME DESTINATION ${comma_INSTALL_BIN_DIR} COMPONENT Runtime ) diff --git a/csv/applications/csv-analyse.cpp b/csv/applications/csv-analyse.cpp index d07a38c4b..d86a9c760 100644 --- a/csv/applications/csv-analyse.cpp +++ b/csv/applications/csv-analyse.cpp @@ -37,7 +37,6 @@ #include #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" using namespace comma; @@ -63,18 +62,15 @@ class histogram //sort, ugly std::multimap< std::size_t, std::size_t > sorted; std::size_t sum=0; - for(std::map< std::size_t, std::size_t >::const_iterator it=histogram_.begin(), end=histogram_.end(); it!=end; ++it ) { sorted.insert( std::make_pair(it->second,it->first) ); sum += it->second; - } - + } for(std::multimap< std::size_t, std::size_t >::const_reverse_iterator it=sorted.rbegin(), end=sorted.rend(); it!=end; ++it ) { os << it->second << "," << it->first << "," << (double)((double)(it->first)/(double)sum) << std::endl; } - return os; } @@ -84,12 +80,9 @@ class histogram std::map< std::size_t, std::size_t > histogram_; //length, count }; -std::ostream& operator<<(std::ostream& os, const histogram & h) -{ - return h.print_sorted(os); -} +std::ostream& operator<<(std::ostream& os, const histogram & h) { return h.print_sorted(os); } -static void usage() +static void usage( bool ) { std::cerr << std::endl; std::cerr << "Analyse binary data to guess message lengths in unknown binary stream: output candidate lengths, repeat counts and normalised probabilities" << std::endl; @@ -121,7 +114,6 @@ static void usage() std::cerr << std::endl; std::cerr << "See also: \"csv-size\", \"csv-bin-cut\"" << std::endl; std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; std::cerr << std::endl; exit( -1 ); } @@ -133,33 +125,22 @@ int main( int ac, char** av ) #ifdef WIN32 _setmode( _fileno( stdin ), _O_BINARY ); #endif - - command_line_options options( ac, av ); - if( ac > 1 || options.exists( "--help" ) || options.exists( "-h" ) ) { usage(); } //could just say ac > 1... but leave for future args - - histogram h; - - const std::size_t read_size=65535; //todo: better way? + command_line_options options( ac, av, usage ); + const std::size_t read_size = 65535; // todo? better way? std::vector< unsigned char > data( read_size ); - std::size_t offset=0; - - //read as many bytes as available on stdin - while( std::cin.good() && !std::cin.eof() ) + std::size_t offset = 0; + histogram h; + while( std::cin.good() && !std::cin.eof() ) //read as many bytes as available on stdin { int bytes_read = ::read( 0, &data[0], read_size ); if( bytes_read <= 0 ) { break; } - - for( int i=0; i #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../csv/format.h" #include "../../csv/options.h" #include "../../string/string.h" @@ -150,7 +122,6 @@ namespace { std::cerr << "Format specifications:" << std::endl; std::cerr << csv::format::usage() << std::endl; } - std::cerr << comma::contact_info << std::endl; std::cerr << std::endl; exit( 0 ); } @@ -291,7 +262,8 @@ namespace { // - check for read errors / end of input // - write the buffer to stdout std::streampos record_start = 0; - if ( skip_ ) { + if ( skip_ ) + { ifs.seekg( 0, std::ios_base::end ); std::streampos fsize = ifs.tellg(); unsigned int nrecords = fsize / irecord_size_; @@ -309,7 +281,8 @@ namespace { std::streamoff off = fields_[i].input_offset - ( i == 0 ? 0 : fields_[i - 1].input_offset + fields_[i - 1].size ); ifs.seekg( off, std::ios_base::cur ); ifs.read( &obuf_[ fields_[i].offset ], fields_[i].size ); - if ( ifs.eof() ) { + if ( ifs.eof() ) + { if ( i == 0 ) { break; } std::cerr << "csv-bin-cut: encountered eof mid-record in '" << fname << "'" << std::endl; exit( 1 ); } @@ -334,12 +307,16 @@ namespace { int seeker::process( const std::vector< std::string > & files ) { if ( files.empty() ) { return read_all( std::cin ); } - for ( std::vector< std::string >::const_iterator ifile = files.begin(); ifile < files.end(); ++ifile ) { - if ( count_max_ >= 0 && count_ >= count_max_ ) { return 0; } - if ( *ifile == "-" ) { + for ( std::vector< std::string >::const_iterator ifile = files.begin(); ifile < files.end(); ++ifile ) + { + if( count_max_ >= 0 && count_ >= count_max_ ) { return 0; } + if( *ifile == "-" ) + { int rv = read_all( std::cin ); if ( rv != 0 ) { return rv; } - } else { + } + else + { std::ifstream ifs( &( *ifile )[0], std::ifstream::binary ); if ( !ifs.is_open() ) { std::cerr << "csv-bin-cut: cannot open '" << *ifile << "' for reading" << std::endl; exit( 1 ); } int rv = ( force_read_ ? read_all( ifs ) : read_fields( ifs, *ifile ) ); @@ -361,31 +338,22 @@ int main( int ac, char** av ) { command_line_options options( ac, av, usage ); comma::csv::options csv( options ); + csv.full_xpath = false; std::vector< std::string > files = options.unnamed( "--help,-h,--verbose,-v,--flush,--read-all,--force-read", "--fields,-f,--output-fields,--output,-o,--binary,-b,--skip,--count" ); if( !csv.binary() ) { if( files.size() == 1 && files[0] != "-" ) // deprecated, left for backward compatibility { - try - { - csv.format( comma::csv::format( files[0] ) ); - files.clear(); - } - catch ( comma::exception & ) - { - // it's not a format string - } + try { csv.format( comma::csv::format( files[0] ) ); files.clear(); } + catch ( comma::exception & ) {} // it's not a format string } } if( !csv.binary() ) { std::cerr << "csv-bin-cut: please specify --binary" << std::endl; exit( 1 ); } - const std::vector< field >& fields = setup_fields( options, csv ); - unsigned int skip = options.value< unsigned int >( "--skip", 0 ); long int count_max = options.value< long int >( "--count", -1 ); bool flush = options.exists( "--flush" ); bool force_read = options.exists( "--read-all,--force-read" ); - seeker seek( fields, csv, skip, count_max, flush, force_read ); return seek.process( files ); } diff --git a/csv/applications/csv-bits.cpp b/csv/applications/csv-bits.cpp new file mode 100644 index 000000000..b4c7edebc --- /dev/null +++ b/csv/applications/csv-bits.cpp @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include +#include "../../application/command_line_options.h" +#include "../../base/exception.h" +#include "../../base/types.h" +#include "../../string/string.h" +#include "../stream.h" +#include "../traits.h" + +void usage( bool verbose ) +{ + std::cerr << "converting between bits and csv and other bit operations" << std::endl; + std::cerr << std::endl; + std::cerr << "usage: cat input.bin | csv-bits > output.bin" << std::endl; + std::cerr << std::endl; + std::cerr << "operations: from-csv (unpack), to-csv (pack)" << std::endl; + std::cerr << std::endl; + std::cerr << "operations" << std::endl; + std::cerr << " from-csv (pack): todo; convert input csv as to packed bits in big endian order" << std::endl; + std::cerr << " options" << std::endl; + //std::cerr << " --endian=; default=big; todo: endianness of input: big or little" << std::endl; + std::cerr << " --binary,-b=[]; input is binary; see details below" << std::endl; + std::cerr << " --flush; see below" << std::endl; + std::cerr << " --sizes=; comma-separated bit field sizes (todo: support multiplier)" << std::endl; + std::cerr << std::endl; + std::cerr << " to-csv (unpack): convert packed bits to integers as csv; input bits are expected in big endian order" << std::endl; + std::cerr << " options" << std::endl; + //std::cerr << " --endian=; default=big; todo: endianness of output: big or little" << std::endl; + std::cerr << " --binary,-b=[]; output is binary; see details below" << std::endl; + std::cerr << " --flush; see below" << std::endl; + std::cerr << " --sizes=; comma-separated bit field sizes (todo: support multiplier)" << std::endl; + std::cerr << std::endl; + std::cerr << "csv options" << std::endl; + std::cerr << comma::csv::options::usage( verbose ) << std::endl; + std::cerr << std::endl; + exit( 0 ); +} + +// todo +// - to-csv +// - sort out endianness: big endian vs transparent semantic use in the mainstream use case +// - unit test +// - support uint64 +// - from-csv +// - use constexpr + +namespace comma { namespace csv_bits { + +struct unpacked +{ + std::vector< comma::uint32 > values; + unpacked( unsigned int size = 0 ): values( size, 0 ) {} +}; + +struct field +{ + unsigned int begin; + unsigned int begin_byte; + unsigned char begin_mask; + unsigned int size; + unsigned int bytes; + unsigned int shift; + bool little_endian; + + static std::array< unsigned char, 8 > begin_masks; + + field() = default; + field( unsigned int begin, unsigned int size, bool little_endian ) + : begin( begin ) + , begin_byte( begin / 8 ) + , begin_mask( begin_masks[ begin % 8 ] ) + , size( size ) + , bytes( size / 8 + int( size % 8 > 0 ) ) + , shift( 64 - begin % 8 - size ) + , little_endian( little_endian ) + { + if( size > sizeof( comma::uint32 ) * 8 ) { COMMA_THROW( comma::exception, "expected size up to " << ( sizeof( comma::uint32 ) * 8 ) << " bits; got: " << size ); } + } + + comma::uint32 get( const std::vector< char >& buf ) const // todo: quick and dirty, watch performance + { + comma::uint64 r = 0; + char* p = reinterpret_cast< char* >( &r ); + std::memcpy( p, &buf[ begin_byte ], bytes ); + p[0] &= begin_mask; + // todo: something like: r = little_endian ? le64toh( r ) : be64toh( r ); + comma::uint64 s = htobe64( r ) >> shift; + return s; + } +}; + +std::array< unsigned char, 8 > field::begin_masks = { 255, 127, 63, 31, 15, 7, 3, 1 }; // todo: use constexpr + +} } // namespace comma { namespace csv_bits { + +namespace comma { namespace visiting { + +template <> struct traits< comma::csv_bits::unpacked > +{ + template < typename K, typename V > static void visit( const K&, const comma::csv_bits::unpacked& p, V& v ) { v.apply( "values", p.values ); } + template < typename K, typename V > static void visit( const K&, comma::csv_bits::unpacked& p, V& v ) { v.apply( "values", p.values ); } +}; + +} } // namespace comma { namespace visiting { + +namespace comma { namespace csv_bits { namespace from_csv { + +int run( const comma::command_line_options& options ) +{ + #ifdef WIN32 + _setmode( _fileno( stdout ), _O_BINARY ); + #endif + comma::csv::options csv( options ); + std::cerr << "csv-bits: from-csv: todo" << std::endl; + return 1; +} + +} } } // namespace comma { namespace csv_bits { namespace from_csv { + +namespace comma { namespace csv_bits { namespace to_csv { + +int run( const comma::command_line_options& options ) +{ + #ifdef WIN32 + _setmode( _fileno( stdin ), _O_BINARY ); + #endif + comma::csv::options csv( options ); + if( !csv.flush ) { std::cin.tie( NULL ); } + const auto& sizes = comma::split_as< unsigned int >( options.value< std::string >( "--sizes" ), ',' ); + unsigned int size = std::accumulate( sizes.begin(), sizes.end(), 0 ); + if( size % 8 > 0 ) { std::cerr << "csv-bits: to-csv: expected input record size in bits divisible by 8; got: " << size << " (oddly-sized record support: todo)" << std::endl; return 1; } + size /= 8; + bool little_endian = options.value< std::string >( "--endian", "big" ) == "little"; + std::vector< std::pair< unsigned int, unsigned int > > indices; + std::vector< comma::csv_bits::field > fields; + unsigned int begin = 0; + for( auto s: sizes ) { fields.push_back( comma::csv_bits::field( begin, s, little_endian ) ); begin += s; } + std::vector< char > buf( size ); + comma::csv_bits::unpacked output( sizes.size() ); + comma::csv::output_stream< comma::csv_bits::unpacked > os( std::cout, csv, output ); + while( std::cin.good() ) + { + std::cin.read( &buf[0], size ); + if( std::cin.gcount() <= 0 ) { break; } + if( std::cin.gcount() < size ) { std::cerr << "csv-bits: to-csv: expected " << size << " byte(s); got: " << std::cin.gcount() << std::endl; return 1; } + for( unsigned int i = 0; i < sizes.size(); ++i ) { output.values[i] = fields[i].get( buf ); } + os.write( output ); + } + return 0; +} + +} } } // namespace comma { namespace csv_bits { namespace to_csv { + +int main( int ac, char** av ) +{ + try + { + comma::command_line_options options( ac, av, usage ); + const auto& unnamed = options.unnamed( "--flush, --verbose, -v", "-.*" ); + if( unnamed.empty() ) { std::cerr << "csv-bits: please specify operation" << std::endl; return 1; } + if( unnamed.size() > 1 ) { std::cerr << "csv-bits: expected operation; got: " << comma::join( unnamed, ',' ) << std::endl; return 1; } + const std::string& operation = unnamed[0]; + if( operation == "to-csv" || operation == "unpack" ) { return comma::csv_bits::to_csv::run( options ); } + if( operation == "from-csv" || operation == "pack" ) { return comma::csv_bits::from_csv::run( options ); } + std::cerr << "csv-bits: expected operation; got: \"" << operation << "\"" << std::endl; + } + catch( std::exception& ex ) { std::cerr << "csv-bits: " << ex.what() << std::endl; } + catch( ... ) { std::cerr << "csv-bits: unknown exception" << std::endl; } + return 1; +} diff --git a/csv/applications/csv-blocks.cpp b/csv/applications/csv-blocks.cpp index 5e3c9b24a..ebcab1dd0 100644 --- a/csv/applications/csv-blocks.cpp +++ b/csv/applications/csv-blocks.cpp @@ -27,7 +27,7 @@ // OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -/// @author dewey nguyen +/// @authors dewey nguyen, vsevolod vlaskine #include #include @@ -41,7 +41,6 @@ #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../base/types.h" #include "../../csv/stream.h" #include "../../csv/impl/unstructured.h" @@ -139,8 +138,20 @@ static void usage( bool more ) std::cerr << " attention: output does not preserve input order, since there is no reasonable tradeof there" << std::endl; std::cerr << " use csv-sort for post-processing, if required" << std::endl; std::cerr << " group|make-blocks" << std::endl; - std::cerr << " cat something.csv | csv-blocks group --fields=,id, " << std::endl; + std::cerr << " usage: cat something.csv | csv-blocks group --fields=,id, " << std::endl; std::cerr << " appends group's block field based on specified id key or keys" << std::endl; + std::cerr << " options" << std::endl; + std::cerr << " --fields=" << std::endl; + std::cerr << " id: any number of id fields to group by" << std::endl; + std::cerr << " scalar: group by scalar, which can be integer, floating point number, or time" << std::endl; + std::cerr << " options" << std::endl; + std::cerr << " --block-gap,--gap=; minimum gap in values between block" << std::endl; + std::cerr << " double (for time: seconds as double), see examples" << std::endl; + std::cerr << " --block-span,--span=; maximum block span, double (for time: seconds as" << std::endl; + std::cerr << " double), see examples" << std::endl; + std::cerr << " --discard-out-of-range; discard input records with scalar out of range defined by --min and --max" << std::endl; + std::cerr << " --min=[]; convenience option: min value for the scalar range, see examples" << std::endl; + std::cerr << " --max=[]; convenience option: max value for the scalar range, see examples" << std::endl; std::cerr << " head" << std::endl; std::cerr << " reads records from first block to stdout, if --num-of-blocks= specified, read more than one blocks" << std::endl; std::cerr << " requires the index from 'index' mode in the inputs" << std::endl; @@ -181,10 +192,16 @@ static void usage( bool more ) std::cerr << " ( echo \"a,1,2,3\"; echo \"a,4,2,3\"; echo \"b,5,5,6\"; echo \"c,7,5,6\"; echo \"c,7,8,9\"; echo \"c,7,8,9\" ) >$block_csv" << std::endl; std::cerr << std::endl; std::cerr << " group|make-blocks" << std::endl; - std::cerr << " cat $block_csv | csv-blocks group --fields=id" << std::endl; - std::cerr << " unique ascending block number are assigned based on one id field" << std::endl; - std::cerr << " cat $block_csv | csv-blocks group --fields=id,,id" << std::endl; - std::cerr << " unique ascending block number are assigned based on two id fields" << std::endl; + std::cerr << " unique ascending block number are assigned based on one id field" << std::endl; + std::cerr << " cat $block_csv | csv-blocks group --fields=id" << std::endl; + std::cerr << " unique ascending block number are assigned based on two id fields" << std::endl; + std::cerr << " cat $block_csv | csv-blocks group --fields=id,,id" << std::endl; + std::cerr << " group by scalar span - try it" << std::endl; + std::cerr << " seq 20 | csv-blocks group --fields=scalar --span 5" << std::endl; + std::cerr << " seq 1 3 20 | csv-blocks group --fields scalar --span 4" << std::endl; + std::cerr << " group by scalar gap - try it" << std::endl; + std::cerr << " seq 20 | csv-blocks group --fields=scalar --gap 1" << std::endl; + std::cerr << " seq 20 | csv-blocks group --fields=scalar --gap 2" << std::endl; std::cerr << std::endl; std::cerr << " index" << std::endl; std::cerr << " cat $block_csv | csv-blocks group --fields=id | csv-blocks index --fields=,,,,block" << std::endl; @@ -207,7 +224,6 @@ static void usage( bool more ) std::cerr << " After indexing the input in reverse order, the stream is read block-by-block, with the first line given the maximal" << std::endl; std::cerr << " element id in the block, essentially, the block size" << std::endl; std::cerr << std::endl; - std::cerr << "contact info: " << comma::contact_info < block_records; -static comma::csv::impl::unstructured keys; -static comma::uint32 current_block = 1; static comma::int32 increment_step = 1; static void output_record_and_index( const std::string& input, comma::uint32 index, bool is_binary, char delimiter ) @@ -255,7 +269,27 @@ static bool empty_( const std::string& s ) // quick and dirty return true; } -template < typename T > static void set_fields( const comma::command_line_options& options, std::string& first_line, T& default_input ) +static double to_double( const input_t& lhs ) // quick and dirty +{ + if( lhs.key.longs.size() == 1 ) { return lhs.key.longs[0]; } + if( lhs.key.doubles.size() == 1 ) { return lhs.key.doubles[0]; } + if( lhs.key.time.size() == 1 ) { COMMA_THROW( comma::exception, "cannot convert time to double" ); } + if( lhs.key.strings.size() == 1 ) { COMMA_THROW( comma::exception, "cannot convert strings to double" ); } + COMMA_THROW( comma::exception, "never here" ); +} + +static double diff( const input_t& lhs, const input_t& rhs ) // quick and dirty +{ + if( lhs.key.longs.size() == 1 ) { return std::abs( double( lhs.key.longs[0] ) - rhs.key.longs[0] ); } + if( lhs.key.doubles.size() == 1 ) { return std::abs( lhs.key.doubles[0] - rhs.key.doubles[0] ); } + if( lhs.key.time.size() == 1 ) { return std::abs( double( ( lhs.key.time[0] - rhs.key.time[0] ).total_microseconds() ) / 1000000 ); } + if( lhs.key.strings.size() == 1 ) { COMMA_THROW( comma::exception, "difference for strings: not implemented" ); } + COMMA_THROW( comma::exception, "never here" ); +} + +struct how_t { enum values { none, by_id, by_scalar }; }; + +template < typename T > static how_t::values set_fields( const comma::command_line_options& options, std::string& first_line, T& default_input ) { std::vector< std::string > v = comma::split( csv.fields, ',' ); comma::csv::format f; @@ -270,8 +304,29 @@ template < typename T > static void set_fields( const comma::command_line_option } // This is to load the keys into input_t structure unsigned int size = f.count(); - for( std::size_t i = 0; i < size; ++i ) { if( i < v.size() ) { if( v[i] == "id" ) { v[i] = "key/" + default_input.key.append( f.offset( i ).type ); continue; } } } + how_t::values how = how_t::none; + for( std::size_t i = 0; i < size; ++i ) + { + if( i < v.size() ) + { + if( v[i] == "id" ) + { + if( how == how_t::by_scalar ) { COMMA_THROW( comma::exception, "expected either id or scalar in --fields; got both in: \"" << csv.fields << "\"" ); } + how = how_t::by_id; + v[i] = "key/" + default_input.key.append( f.offset( i ).type ); + } + else if( v[i] == "scalar" ) + { + if( how == how_t::by_id ) { COMMA_THROW( comma::exception, "expected either id or scalar in --fields; got both in: \"" << csv.fields << "\"" ); } + if( how == how_t::by_scalar ) { COMMA_THROW( comma::exception, "expected not more than one scalar in --fields; got: \"" << csv.fields << "\"" ); } + how = how_t::by_scalar; + v[i] = "key/" + default_input.key.append( f.offset( i ).type ); + } + } + } + if( how == how_t::none ) { COMMA_THROW( comma::exception, "please specify at least one id or scalar in --fields; got: \"" << csv.fields << "\"" ); } csv.fields = comma::join( v, ',' ); + return how; } #ifndef WIN32 @@ -413,16 +468,14 @@ int main( int ac, char** av ) verbose = options.exists( "--verbose,-v" ); strict = options.exists( "--strict" ); csv = comma::csv::options( options ); - csv.full_xpath = true; csv.quote.reset(); comma::csv::options csv_out; + csv_out.full_xpath = false; if( csv.binary() ) { csv_out.format( comma::csv::format("ui") ); } - std::vector< std::string > unnamed = options.unnamed( "--help,-h,--reverse,--verbose,-v", "-.*" ); + std::vector< std::string > unnamed = options.unnamed( "--help,-h,--reverse,--verbose,-v,--discard-out-of-range", "-.*" ); if( unnamed.empty() ) { std::cerr << name() << "please specify operation" << std::endl; return 1; } const std::string operation = unnamed.front(); - if( verbose ) { std::cerr << name() << "csv fields: " << csv.fields << std::endl; } - if( operation == "accumulate" ) { std::string first_line; @@ -471,38 +524,93 @@ int main( int ac, char** av ) } if( operation == "group" || operation == "make-blocks" ) { - current_block = options.value< comma::uint32 >( "--starting-block,--from", 0 ); // default is 0 - + comma::uint32 current_block = options.value< comma::uint32 >( "--starting-block,--from", 0 ); std::string first_line; input_t default_input; - set_fields( options, first_line, default_input ); - if( verbose ) { std::cerr << name() << "csv fields: " << csv.fields << std::endl; } - if ( default_input.key.empty() ) { std::cerr << name() << "please specify at least one id field" << std::endl; return 1; } - + auto how = set_fields( options, first_line, default_input ); + if( verbose ) { std::cerr << name() << "csv fields: " << csv.fields << "; making blocks by " << ( how == how_t::by_id ? "id" : "scalar" ) << std::endl; } + boost::optional< double > gap; + boost::optional< double > span; + boost::optional< double > min; + boost::optional< double > max; + if( how == how_t::by_scalar ) + { + options.assert_mutually_exclusive( "--gap,--span", "--min,--max" ); // for now + gap = options.optional< double >( "--block-gap,--gap" ); + span = options.optional< double >( "--block-span,--span" ); + min = options.optional< double >( "--min" ); + max = options.optional< double >( "--max" ); + } comma::csv::input_stream< input_t > istream( std::cin, csv, default_input ); comma::csv::output_stream< appended_column > ostream( std::cout, csv_out ); comma::csv::tied< input_t, appended_column > tied( istream, ostream ); - + auto update_block = [&]( const input_t& p )->bool + { + static input_t last = p; + switch( how ) + { + case how_t::by_id: + if( !( last.key == p.key ) ) { ++current_block; } + last = p; + return true; + case how_t::by_scalar: + { + static input_t first = p; + if( gap || span ) + { + if( ( gap && diff( last, p ) >= *gap ) || ( span && diff( first, p ) >= *span ) ) { ++current_block; first = p; } + last = p; + return true; + } + else + { + static bool last_in_range = false; + static bool discard_output_out_of_range = options.exists( "--discard-out-of-range" ); + double v = to_double( p ); + bool in_range = ( !min || !comma::math::less( v, *min ) ) && ( !max || !comma::math::less( *max, v ) ); + static bool first_record = true; + if( !first_record ) // quick and dirty + { + if( discard_output_out_of_range ) + { + if( last_in_range && !in_range ) { ++current_block; } + } + else + { + if( last_in_range != in_range ) { ++current_block; } // quick and dirty + } + } + last = p; + last_in_range = in_range; + first_record = false; + return in_range || !discard_output_out_of_range; + } + } + case how_t::none: // never here + return true; + } + return true; // never here + }; if( !first_line.empty() ) { input_t p = comma::csv::ascii< input_t >( csv, default_input ).get( first_line ); - if( !(keys == p.key) ) { ++current_block; } - keys = p.key; - // This is needed because the record wasnt read in by istream - // Write it out - if( istream.is_binary() ) { std::cout.write( (char*)&p, istream.binary().size() ); } - else { std::cout << first_line << istream.ascii().ascii().delimiter(); } - ostream.write( appended_column( current_block ) ); - if( csv.flush ) { std::cout.flush(); } + if( update_block( p ) ) + { + if( istream.is_binary() ) { std::cout.write( (char*)&p, istream.binary().size() ); } + else { std::cout << first_line << istream.ascii().ascii().delimiter(); } + ostream.write( appended_column( current_block ) ); + if( csv.flush ) { std::cout.flush(); } + } } while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ) { const input_t* p = istream.read(); if( !p ) { break; } - if( !(keys == p->key) ) { ++current_block; } - keys = p->key; - tied.append( appended_column( current_block ) ); - if( csv.flush ) { std::cout.flush(); } + if( update_block( *p ) ) + { + tied.append( appended_column( current_block ) ); + if( csv.flush ) { std::cout.flush(); } + } } return 0; } @@ -546,60 +654,51 @@ int main( int ac, char** av ) else if( operation == "index" ) { reverse_index = options.exists("--reverse"); - comma::csv::input_stream< input_with_block > istream( std::cin, csv ); - char delimiter = istream.is_binary() ? ',' : istream.ascii().ascii().delimiter(); comma::uint32 block = 0; comma::uint32 index = 0; std::string buffer; if( istream.is_binary() ) { buffer.resize( istream.binary().size() ); } - while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ) { const input_with_block* p = istream.read(); if( !p ) { break; } - if( block != p->block ) { if ( reverse_index ) { output_reverse_indexing( block_records, istream.is_binary(), delimiter ); } else { index = 0; } } block = p->block; - if ( reverse_index ) { - // Reverse index mode - accumulate whole block before indexing if( istream.is_binary() ) { ::memcpy( &buffer[0], istream.binary().last(), istream.binary().size() ); block_records.push_back( buffer ); } - else { block_records.push_back( comma::join( istream.ascii().last(), delimiter ) ); } + else + { + block_records.push_back( comma::join( istream.ascii().last(), delimiter ) ); + } } else { - // Forward index mode - append index to each record if( istream.is_binary() ) { ::memcpy( &buffer[0], istream.binary().last(), istream.binary().size() ); } else { buffer = comma::join( istream.ascii().last(), delimiter ); } output_record_and_index( buffer, index, istream.is_binary(), delimiter ); index++; } } - - // flushes the last block if ( reverse_index ) { output_reverse_indexing( block_records, istream.is_binary(), delimiter ); } - return 0; } else if( operation == "increment" ) // operation is head { increment_step = options.value< comma::int32 >( "--step", 1 ); - comma::csv::input_stream< input_with_block > istream( std::cin, csv ); comma::csv::output_stream< appended_column > ostream( std::cout, csv_out ); comma::csv::tied< input_with_block, appended_column > tied( istream, ostream ); - appended_column incremented; while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ) { @@ -609,7 +708,6 @@ int main( int ac, char** av ) tied.append( incremented ); if( csv.flush ) { std::cout.flush(); } } - return 0; } else if( operation == "read-until" ) diff --git a/csv/applications/csv-calc.cpp b/csv/applications/csv-calc.cpp index 1670f4ac2..2d1208ab6 100644 --- a/csv/applications/csv-calc.cpp +++ b/csv/applications/csv-calc.cpp @@ -1,32 +1,5 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +// Copyright (c) 2020 Vsevolod Vlaskine /// @author vsevolod vlaskine @@ -36,17 +9,21 @@ #include #endif +#include +#include #include -#include +#include +#include +#include +#include #include #include #include #include #include #include -#include "../../application/contact_info.h" -#include "../../application/verbose.h" #include "../../base/exception.h" +#include "../../base/none.h" #include "../../csv/format.h" #include "../../csv/options.h" #include "../../string/string.h" @@ -72,7 +49,7 @@ static void usage( bool verbose ) std::cerr << std::endl; std::cerr << "column-wise calculation, optionally by id and block" << std::endl; std::cerr << std::endl; - std::cerr << "usage: cat data.csv | " << comma::verbose.app_name() << " [] > calc.csv" << std::endl; + std::cerr << "usage: cat data.csv | csv-calc [] > calc.csv" << std::endl; std::cerr << std::endl; std::cerr << ": comma-separated list of operations" << std::endl; std::cerr << " results will be output in the same order" << std::endl; @@ -90,7 +67,7 @@ static void usage( bool verbose ) std::cerr << " is the desired percentile (e.g. 0.9)" << std::endl; std::cerr << " is one of 'nearest' or 'interpolate' (default: nearest)" << std::endl; std::cerr << " see --help --verbose for more details" << std::endl; - std::cerr << " radius: size / 2" << std::endl; + std::cerr << " radius: diameter / 2" << std::endl; std::cerr << " size: number of values" << std::endl; std::cerr << " skew[=sample]: skew" << std::endl; std::cerr << " sample: use sample skew (default: population stddev)" << std::endl; @@ -102,6 +79,7 @@ static void usage( bool verbose ) std::cerr << std::endl; std::cerr << "" << std::endl; std::cerr << " --append: append statistics to each input line" << std::endl; + std::cerr << " --append-once,--append-to-first: append statistics to first input line for each block and/or each id" << std::endl; std::cerr << " --delimiter,-d : default ','" << std::endl; std::cerr << " --fields,-f: field names for which the extents should be computed, default: all fields" << std::endl; std::cerr << " if 'block' field present, calculate block-wise" << std::endl; @@ -137,14 +115,14 @@ static void usage( bool verbose ) std::cerr << std::endl; } std::cerr << "examples" << std::endl; - std::cerr << " seq 1 1000 | " << comma::verbose.app_name() << " percentile=0.9" << std::endl; - std::cerr << " seq 1 1000 | " << comma::verbose.app_name() << " percentile=0.9:interpolate --verbose" << std::endl; + std::cerr << " seq 1 1000 | csv-calc percentile=0.9" << std::endl; + std::cerr << " seq 1 1000 | csv-calc percentile=0.1,percentile=0.9" << std::endl; + std::cerr << " seq 1 1000 | csv-calc percentile=0.9:interpolate --verbose" << std::endl; std::cerr << std::endl; - std::cerr << " {(seq 1 500 | csv-paste \"-\" \"value=0\") ; (seq 1 100 | csv-paste \"-\" \"value=1\") ; (seq 501 1000 | csv-paste \"-\" \"value=0\")} | " << comma::verbose.app_name() << " --fields=a,block percentile=0.9" << std::endl; + std::cerr << " {(seq 1 500 | csv-paste \"-\" \"value=0\") ; (seq 1 100 | csv-paste \"-\" \"value=1\") ; (seq 501 1000 | csv-paste \"-\" \"value=0\")} | csv-calc --fields=a,block percentile=0.9" << std::endl; std::cerr << std::endl; - std::cerr << " {(seq 1 500 | csv-paste \"-\" \"value=0\") ; (seq 1 100 | csv-paste \"-\" \"value=1\") ; (seq 501 1000 | csv-paste \"-\" \"value=0\")} | " << comma::verbose.app_name() << " --fields=a,id percentile=0.9" << std::endl; + std::cerr << " {(seq 1 500 | csv-paste \"-\" \"value=0\") ; (seq 1 100 | csv-paste \"-\" \"value=1\") ; (seq 501 1000 | csv-paste \"-\" \"value=0\")} | csv-calc --fields=a,id percentile=0.9" << std::endl; std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; std::cerr << std::endl; exit( -1 ); } @@ -213,14 +191,14 @@ class Values std::vector< comma::csv::format::element > input_elements_; std::vector< comma::csv::format::element > elements_; std::vector< char > buffer_; - boost::optional< unsigned int > block_index_; - boost::optional< unsigned int > id_index_; + boost::optional< unsigned int > block_index_{ comma::silent_none< unsigned int >() }; + boost::optional< unsigned int > id_index_{ comma::silent_none< unsigned int >() }; comma::csv::format::element block_element_; comma::csv::format::element id_element_; unsigned int block_; unsigned int id_; - boost::function< comma::uint32( const char* ) > block_from_bin_; - boost::function< comma::uint32( const char* ) > id_from_bin_; + std::function< comma::uint32( const char* ) > block_from_bin_; + std::function< comma::uint32( const char* ) > id_from_bin_; template < typename T > static comma::uint32 from_bin_( const char* buf ) { return comma::csv::format::traits< T >::from_bin( buf ); } void init_indices_() @@ -260,15 +238,15 @@ class Values block_element_ = input_format_.offset( *block_index_ ); switch( block_element_.type ) { - case comma::csv::format::char_t: block_from_bin_ = boost::bind( &Values::from_bin_< char >, _1 ); break; - case comma::csv::format::int8: block_from_bin_ = boost::bind( &Values::from_bin_< char >, _1 ); break; - case comma::csv::format::uint8: block_from_bin_ = boost::bind( &Values::from_bin_< unsigned char >, _1 ); break; - case comma::csv::format::int16: block_from_bin_ = boost::bind( &Values::from_bin_< comma::int16 >, _1 ); break; - case comma::csv::format::uint16: block_from_bin_ = boost::bind( &Values::from_bin_< comma::uint16 >, _1 ); break; - case comma::csv::format::int32: block_from_bin_ = boost::bind( &Values::from_bin_< comma::int32 >, _1 ); break; - case comma::csv::format::uint32: block_from_bin_ = boost::bind( &Values::from_bin_< comma::uint32 >, _1 ); break; - case comma::csv::format::int64: block_from_bin_ = boost::bind( &Values::from_bin_< comma::int64 >, _1 ); break; - case comma::csv::format::uint64: block_from_bin_ = boost::bind( &Values::from_bin_< comma::uint64 >, _1 ); break; + case comma::csv::format::char_t: block_from_bin_ = std::bind( &Values::from_bin_< char >, std::placeholders::_1 ); break; + case comma::csv::format::int8: block_from_bin_ = std::bind( &Values::from_bin_< char >, std::placeholders::_1 ); break; + case comma::csv::format::uint8: block_from_bin_ = std::bind( &Values::from_bin_< unsigned char >, std::placeholders::_1 ); break; + case comma::csv::format::int16: block_from_bin_ = std::bind( &Values::from_bin_< comma::int16 >, std::placeholders::_1 ); break; + case comma::csv::format::uint16: block_from_bin_ = std::bind( &Values::from_bin_< comma::uint16 >, std::placeholders::_1 ); break; + case comma::csv::format::int32: block_from_bin_ = std::bind( &Values::from_bin_< comma::int32 >, std::placeholders::_1 ); break; + case comma::csv::format::uint32: block_from_bin_ = std::bind( &Values::from_bin_< comma::uint32 >, std::placeholders::_1 ); break; + case comma::csv::format::int64: block_from_bin_ = std::bind( &Values::from_bin_< comma::int64 >, std::placeholders::_1 ); break; + case comma::csv::format::uint64: block_from_bin_ = std::bind( &Values::from_bin_< comma::uint64 >, std::placeholders::_1 ); break; default: COMMA_THROW( comma::exception, "expected integer for block id, got format " << input_format_.string() ); } } @@ -277,15 +255,15 @@ class Values id_element_ = input_format_.offset( *id_index_ ); switch( id_element_.type ) { - case comma::csv::format::char_t: id_from_bin_ = boost::bind( &Values::from_bin_< char >, _1 ); break; - case comma::csv::format::int8: id_from_bin_ = boost::bind( &Values::from_bin_< char >, _1 ); break; - case comma::csv::format::uint8: id_from_bin_ = boost::bind( &Values::from_bin_< unsigned char >, _1 ); break; - case comma::csv::format::int16: id_from_bin_ = boost::bind( &Values::from_bin_< comma::int16 >, _1 ); break; - case comma::csv::format::uint16: id_from_bin_ = boost::bind( &Values::from_bin_< comma::uint16 >, _1 ); break; - case comma::csv::format::int32: id_from_bin_ = boost::bind( &Values::from_bin_< comma::int32 >, _1 ); break; - case comma::csv::format::uint32: id_from_bin_ = boost::bind( &Values::from_bin_< comma::uint32 >, _1 ); break; - case comma::csv::format::int64: id_from_bin_ = boost::bind( &Values::from_bin_< comma::int64 >, _1 ); break; - case comma::csv::format::uint64: id_from_bin_ = boost::bind( &Values::from_bin_< comma::uint64 >, _1 ); break; + case comma::csv::format::char_t: id_from_bin_ = std::bind( &Values::from_bin_< char >, std::placeholders::_1 ); break; + case comma::csv::format::int8: id_from_bin_ = std::bind( &Values::from_bin_< char >, std::placeholders::_1 ); break; + case comma::csv::format::uint8: id_from_bin_ = std::bind( &Values::from_bin_< unsigned char >, std::placeholders::_1 ); break; + case comma::csv::format::int16: id_from_bin_ = std::bind( &Values::from_bin_< comma::int16 >, std::placeholders::_1 ); break; + case comma::csv::format::uint16: id_from_bin_ = std::bind( &Values::from_bin_< comma::uint16 >, std::placeholders::_1 ); break; + case comma::csv::format::int32: id_from_bin_ = std::bind( &Values::from_bin_< comma::int32 >, std::placeholders::_1 ); break; + case comma::csv::format::uint32: id_from_bin_ = std::bind( &Values::from_bin_< comma::uint32 >, std::placeholders::_1 ); break; + case comma::csv::format::int64: id_from_bin_ = std::bind( &Values::from_bin_< comma::int64 >, std::placeholders::_1 ); break; + case comma::csv::format::uint64: id_from_bin_ = std::bind( &Values::from_bin_< comma::uint64 >, std::placeholders::_1 ); break; default: COMMA_THROW( comma::exception, "expected integer for block id, got format " << input_format_.string() ); } } @@ -309,7 +287,7 @@ class ascii_input return values_.get(); } - const std::string line() { return line_; } + const std::string& line() const { return line_; } private: comma::csv::options csv_; @@ -334,10 +312,15 @@ class binary_input { while( true ) { + //std::cin.read( &buffer_[0], csv_.format().size() ); + //if( std::cin.gcount() == 0 ) { return NULL; } + //if( std::cin.gcount() != int( csv_.format().size() ) ) { COMMA_THROW( comma::exception, "expected " << csv_.format().size() << " bytes; got " << std::cin.gcount() ); } + //values_.set( &buffer_[0] ); + //return &values_; if( offset_ >= csv_.format().size() ) { values_.set( cur_ ); - line_ = std::string(cur_, csv_.format().size()); + line_ = std::string( cur_, csv_.format().size() ); cur_ += csv_.format().size(); offset_ -= csv_.format().size(); if( cur_ == end_ ) { cur_ = &buffer_[0]; offset_ = 0; } @@ -348,7 +331,8 @@ class binary_input offset_ += count; } } - const std::string line() { return line_; } + + const std::string& line() const { return line_; } private: comma::csv::options csv_; @@ -369,11 +353,11 @@ template < typename T, typename V > struct map_traits template < typename V > struct map_traits< boost::posix_time::ptime, V > { - struct hash : public std::unary_function< boost::posix_time::ptime, std::size_t > + struct hash : public std::function< boost::posix_time::ptime( std::size_t ) > { std::size_t operator()( const boost::posix_time::ptime& t ) const { - BOOST_STATIC_ASSERT( sizeof( t ) == sizeof( comma::uint64 ) ); + static_assert( sizeof( t ) == sizeof( comma::uint64 ), "expected 8-byte time" ); std::size_t seed = 0; boost::hash_combine( seed, reinterpret_cast< const comma::uint64& >( t ) ); // quick and dirty return seed; @@ -431,6 +415,7 @@ namespace Operations struct base { virtual ~base() {} + virtual void reset() = 0; virtual void push( const char* ) = 0; virtual void calculate( char* ) = 0; virtual base* clone() const = 0; @@ -445,6 +430,7 @@ namespace Operations class Min : public base { public: + void reset() { min_ = comma::silent_none< T >(); } void push( const char* buf ) { const T& t = comma::csv::format::traits< T, F >::from_bin( buf ); @@ -456,13 +442,14 @@ namespace Operations friend class Centre< T, F >; friend class Diameter< T, F >; friend class Radius< T, F >; - boost::optional< T > min_; + boost::optional< T > min_{ comma::silent_none< T >() }; }; template < typename T, comma::csv::format::types_enum F = comma::csv::format::type_to_enum< T >::value > class Max : public base { public: + void reset() { max_ = comma::silent_none< T >(); } void push( const char* buf ) { T t = comma::csv::format::traits< T, F >::from_bin( buf ); @@ -474,13 +461,14 @@ namespace Operations friend class Centre< T, F >; friend class Diameter< T, F >; friend class Radius< T, F >; - boost::optional< T > max_; + boost::optional< T > max_{ comma::silent_none< T >() }; }; template < typename T, comma::csv::format::types_enum F = comma::csv::format::type_to_enum< T >::value > class Sum : public base { public: + void reset() { sum_ = comma::silent_none< T >(); } void push( const char* buf ) { T t = comma::csv::format::traits< T, F >::from_bin( buf ); @@ -489,12 +477,13 @@ namespace Operations void calculate( char* buf ) { if( sum_ ) { comma::csv::format::traits< T, F >::to_bin( *sum_, buf ); } } base* clone() const { return new Sum< T, F >( *this ); } private: - boost::optional< T > sum_; + boost::optional< T > sum_{ comma::silent_none< T >() }; }; template < comma::csv::format::types_enum F > class Sum< boost::posix_time::ptime, F > : public base { + void reset() { COMMA_THROW( comma::exception, "sum not defined for time" ); } void push( const char* ) { COMMA_THROW( comma::exception, "sum not defined for time" ); } void calculate( char* ) { COMMA_THROW( comma::exception, "sum not defined for time" ); } base* clone() const { COMMA_THROW( comma::exception, "sum not defined for time" ); } @@ -504,6 +493,7 @@ namespace Operations class Centre : public base { public: + void reset() { min_ = Min< T, F >(); max_ = Max< T, F >(); } void push( const char* buf ) { min_.push( buf ); max_.push( buf ); } void calculate( char* buf ) { if( min_.min_ ) { comma::csv::format::traits< T, F >::to_bin( *min_.min_ + ( *max_.max_ - *min_.min_ ) / 2, buf ); } } base* clone() const { return new Centre< T, F >( *this ); } @@ -519,6 +509,7 @@ namespace Operations class Mode : public base { public: + void reset() { value_count_ = impl::value_count< T >(); } void push( const char* buf ) { value_count_.update( comma::csv::format::traits< T, F >::from_bin( buf ) ); } void calculate( char* buf ) { if( !value_count_.map().empty() ) { comma::csv::format::traits< T, F >::to_bin( static_cast< T >( value_count_.mode().first ), buf ); } } base* clone() const { return new Mode< T, F >( *this ); } @@ -531,6 +522,7 @@ namespace Operations { public: Mean() : count_( 0 ) {} + void reset() { mean_.reset(); count_ = 0; } void push( const char* buf ) { T t = comma::csv::format::traits< T, F >::from_bin( buf ); @@ -540,7 +532,7 @@ namespace Operations void calculate( char* buf ) { if( count_ > 0 ) { comma::csv::format::traits< T, F >::to_bin( static_cast< T >( *mean_ ), buf ); } } base* clone() const { return new Mean< T, F >( *this ); } private: - boost::optional< typename result_traits< T >::type > mean_; + boost::optional< typename result_traits< T >::type > mean_{ comma::silent_none< typename result_traits< T >::type >() }; std::size_t count_; }; @@ -552,89 +544,76 @@ namespace Operations Percentile() : percentile_( 0.0 ), method_( nearest ) {} - void push( const char* buf ) - { - values_.insert( comma::csv::format::traits< T, F >::from_bin( buf )); - } + void push( const char* buf ) { values_.insert( comma::csv::format::traits< T, F >::from_bin( buf ) ); } void set_options( const std::vector< std::string >& options ) { - if( options.size() == 0 ) { - std::cerr << comma::verbose.app_name() << ": percentile operation requires a percentile" << std::endl; - exit( 1 ); - } - + if( options.empty() ) { std::cerr << comma::verbose.app_name() << ": percentile operation requires a percentile" << std::endl; exit( 1 ); } percentile_ = boost::lexical_cast< double >( options[0] ); - if( percentile_ < 0.0 || percentile_ > 1.0 ) { - std::cerr << comma::verbose.app_name() << ": percentile value should be between 0 and 1, got " << percentile_ << std::endl; - exit( 1 ); - } - - if( options.size() == 2 ) { - if( options[1] == "nearest" ) method_ = nearest; - else if( options[1] == "interpolate" ) method_ = interpolate; - else { - std::cerr << comma::verbose.app_name() << ": expected percentile method, got " << options[1] << std::endl; - exit( 1 ); - } - } + if( percentile_ < 0.0 || percentile_ > 1.0 ) { std::cerr << comma::verbose.app_name() << ": percentile value should be between 0 and 1, got " << percentile_ << std::endl; exit( 1 ); } + if( options.size() < 2 ) { return; } + if( options[1] == "nearest" ) { method_ = nearest; } + else if( options[1] == "interpolate" ) { method_ = interpolate; } + else { std::cerr << comma::verbose.app_name() << ": expected percentile method, got '" << options[1] << "'" << std::endl; exit( 1 ); } } void calculate( char* buf ) { + if( values_.empty() ) { return; } std::size_t count = values_.size(); - - if( count > 0 ) + comma::verbose << "calculating " << percentile_*100 << "th percentile using "; + T value = comma::csv::format::traits< T, F >::zero(); + typename std::multiset< T >::iterator it = values_.begin(); + switch( method_ ) { - comma::verbose << "calculating " << percentile_*100 << "th percentile using "; - T value; - typename std::multiset< T >::iterator it = values_.begin(); - switch( method_ ) - { - std::size_t rank; - - case nearest: - // https://en.wikipedia.org/wiki/Percentile#The_Nearest_Rank_method - comma::verbose << "nearest rank method" << std::endl; - comma::verbose << "see https://en.wikipedia.org/wiki/Percentile#The_Nearest_Rank_method" << std::endl; - rank = ( percentile_ == 0.0 ? 1 : std::ceil( count * percentile_ )); - comma::verbose << "n = " << rank << std::endl; - std::advance( it, rank - 1 ); + std::size_t rank; + + case nearest: + // https://en.wikipedia.org/wiki/Percentile#The_Nearest_Rank_method + comma::verbose << "nearest rank method" << std::endl; + comma::verbose << "see https://en.wikipedia.org/wiki/Percentile#The_Nearest_Rank_method" << std::endl; + rank = ( percentile_ == 0.0 ? 1 : std::ceil( count * percentile_ )); + comma::verbose << "n = " << rank << std::endl; + std::advance( it, rank - 1 ); + value = *it; + break; + + case interpolate: + // https://en.wikipedia.org/wiki/Percentile#The_Linear_Interpolation_Between_Closest_Ranks_method + // (third method in that section) + comma::verbose << "NIST linear interpolation method" << std::endl; + comma::verbose << "see http://www.itl.nist.gov/div898/handbook/prc/section2/prc262.htm" << std::endl; + double x = percentile_ * ( count + 1 ); + comma::verbose << "p = " << percentile_ << "; N = " << count << "; p(N + 1) = " << x; + if( x <= 1.0 ) + { + comma::verbose << "; below 1 - choosing smallest value" << std::endl; value = *it; - break; - - case interpolate: - // https://en.wikipedia.org/wiki/Percentile#The_Linear_Interpolation_Between_Closest_Ranks_method - // (third method in that section) - comma::verbose << "NIST linear interpolation method" << std::endl; - comma::verbose << "see http://www.itl.nist.gov/div898/handbook/prc/section2/prc262.htm" << std::endl; - double x = percentile_ * ( count + 1 ); - comma::verbose << "p = " << percentile_ << "; N = " << count - << "; p(N + 1) = " << x; - if( x <= 1.0 ) { - comma::verbose << "; below 1 - choosing smallest value" << std::endl; - value = *it; - } else if( x >= count ) { - comma::verbose << "; above N - choosing largest value" << std::endl; - value = *( values_.rbegin() ); - } else { - rank = x; - double remainder = x - rank; - comma::verbose << "; k = " << rank << "; d = " << remainder << std::endl; - std::advance( it, rank - 1 ); - double v1 = *it; - double v2 = *++it; - value = v1 + ( v2 - v1 ) * remainder; - comma::verbose << "v1 = " << v1 << "; v2 = " << v2 - << "; result = " << value << std::endl; - } - break; - } - comma::csv::format::traits< T, F >::to_bin( static_cast< T >( value ), buf ); + } + else if( x >= count ) + { + comma::verbose << "; above N - choosing largest value" << std::endl; + value = *( values_.rbegin() ); + } + else + { + rank = x; + double remainder = x - rank; + comma::verbose << "; k = " << rank << "; d = " << remainder << std::endl; + std::advance( it, rank - 1 ); + double v1 = *it; + double v2 = *++it; + value = v1 + ( v2 - v1 ) * remainder; + comma::verbose << "v1 = " << v1 << "; v2 = " << v2 << "; result = " << value << std::endl; + } + break; } + comma::csv::format::traits< T, F >::to_bin( static_cast< T >( value ), buf ); } base* clone() const { return new Percentile< T, F >( *this ); } + + void reset() { values_.clear(); } private: std::multiset< T > values_; @@ -645,6 +624,7 @@ namespace Operations template < comma::csv::format::types_enum F > class Percentile< boost::posix_time::ptime, F > : public base { + void reset() { COMMA_THROW( comma::exception, "percentile not implemented for time, todo" ); } void push( const char* ) { COMMA_THROW( comma::exception, "percentile not implemented for time, todo" ); } void calculate( char* ) { COMMA_THROW( comma::exception, "percentile not implemented for time, todo" ); } base* clone() const { COMMA_THROW( comma::exception, "percentile not implemented for time, todo" ); } @@ -722,6 +702,8 @@ namespace Operations typename result_traits< T >::type mean() const { return previous_.mean(); } + void reset() { previous_.reset(); value_ = 0; count_ = 0; } + private: Moment< T, M - 1 > previous_; typename result_traits< T >::type value_; @@ -733,6 +715,7 @@ namespace Operations { public: Moment() : value_( 0 ), count_( 0 ) {} + void update ( const T t ) { ++count_; @@ -741,6 +724,8 @@ namespace Operations typename result_traits< T >::type mean() const { return value_; } + void reset() { value_ = 0; count_ = 0; } + private: typename result_traits< T >::type value_; std::size_t count_; @@ -762,9 +747,10 @@ namespace Operations void update( const T t ) { moments_.update(t); } void calculate( char* buf ) { if( moments_.count() > 0 ) { comma::csv::format::traits< T, F >::to_bin( static_cast< T >( std::sqrt( static_cast< long double >( moments_.value() / ( sample_ ? moments_.count() - 1 : moments_.count() ) ) ) ), buf ); } } base* clone() const { return new Stddev< T, F >( *this ); } + void reset() { moments_.reset(); first_ = boost::none; } private: Moment< T, 2 > moments_; - boost::optional first_; + boost::optional< T > first_{ comma::silent_none< T >() }; bool sample_; }; @@ -782,9 +768,10 @@ namespace Operations } void calculate( char* buf ) { stddev_.calculate(buf); } base* clone() const { return new Stddev< boost::posix_time::ptime, F >( *this ); } + void reset() { stddev_.reset(); first_ = boost::none; } private: Stddev< double, F > stddev_; - boost::optional first_; + boost::optional< boost::posix_time::ptime > first_{ comma::silent_none< boost::posix_time::ptime >() }; }; template < typename T, comma::csv::format::types_enum F = comma::csv::format::type_to_enum< T >::value > @@ -803,9 +790,10 @@ namespace Operations void update( const T t ) { moments_.update(t); } void calculate( char* buf ) { if( moments_.count() > 0 ) { comma::csv::format::traits< T, F >::to_bin( static_cast< T >( moments_.value() / ( sample_ ? moments_.count() - 1 : moments_.count() ) ), buf ); } } base* clone() const { return new Variance< T, F >( *this ); } + void reset() { moments_.reset(); first_ = boost::none; } private: Moment< T, 2 > moments_; - boost::optional first_; + boost::optional< T > first_{ comma::silent_none< T >() }; bool sample_; }; @@ -823,9 +811,10 @@ namespace Operations } void calculate( char* buf ) { variance_.calculate(buf); } base* clone() const { return new Variance< boost::posix_time::ptime, F >( *this ); } + void reset() { variance_.reset(); first_ = boost::none; } private: - Variance< double, F> variance_; - boost::optional first_; + Variance< double, F > variance_; + boost::optional< boost::posix_time::ptime > first_{ comma::silent_none< boost::posix_time::ptime >() }; }; template < typename T, comma::csv::format::types_enum F = comma::csv::format::type_to_enum< T >::value > @@ -846,7 +835,6 @@ namespace Operations if( moments_.count() > 0 ) { typename result_traits< T >::type n = moments_.count(); - // corrected sample skew requires at least 3 samples typename result_traits< T >::type correction = sample_ ? sqrt( n * ( n - 1 ) ) / ( n - 2 ) : 1 ; typename result_traits< T >::type m2 = moments_.previous().value(); @@ -855,9 +843,10 @@ namespace Operations } } base* clone() const { return new Skew< T, F >( *this ); } + void reset() { moments_.reset(); first_ = boost::none; } private: Moment< T, 3 > moments_; - boost::optional first_; + boost::optional< T > first_{ comma::silent_none< T >() }; bool sample_; }; @@ -875,9 +864,10 @@ namespace Operations } void calculate( char* buf ) { skew_.calculate(buf); } base* clone() const { return new Skew< boost::posix_time::ptime, F >( *this ); } + void reset() { skew_.reset(); first_ = boost::none; } private: - Skew< double, F> skew_; - boost::optional first_; + Skew< double, F > skew_; + boost::optional< boost::posix_time::ptime > first_{ comma::silent_none< boost::posix_time::ptime >() }; }; template < typename T, comma::csv::format::types_enum F = comma::csv::format::type_to_enum< T >::value > @@ -889,8 +879,8 @@ namespace Operations { for (std::size_t i = 0; i < options.size(); i++) { - if ( options[i] == "sample" ) { sample_ = true; } - else if ( options[i] == "excess" ) { excess_ = true; } + if( options[i] == "sample" ) { sample_ = true; } + else if( options[i] == "excess" ) { excess_ = true; } } } void push( const char* buf ) @@ -917,9 +907,10 @@ namespace Operations } } base* clone() const { return new Kurtosis< T, F >( *this ); } + void reset() { moments_.reset(); first_ = boost::none; } private: Moment< T, 4 > moments_; - boost::optional first_; + boost::optional< T > first_{ comma::silent_none< T >() }; bool sample_; bool excess_; }; @@ -938,9 +929,10 @@ namespace Operations } void calculate( char* buf ) { kurtosis_.calculate(buf); } base* clone() const { return new Kurtosis< boost::posix_time::ptime, F >( *this ); } + void reset() { kurtosis_.reset(); first_ = boost::none; } private: - Kurtosis< double, F> kurtosis_; - boost::optional first_; + Kurtosis< double, F > kurtosis_; + boost::optional< boost::posix_time::ptime > first_{ comma::silent_none< boost::posix_time::ptime >() }; }; template < typename T > struct Diff @@ -962,6 +954,7 @@ namespace Operations void push( const char* buf ) { min_.push( buf ); max_.push( buf ); } void calculate( char* buf ) { if( min_.min_ ) { comma::csv::format::traits< typename Diff< T >::Type >::to_bin( Diff< T >::subtract( *max_.max_, *min_.min_ ), buf ); } } base* clone() const { return new Diameter< T, F >( *this ); } + void reset() { min_ = Min< T, F >(); max_ = Max< T, F >(); } private: Min< T, F > min_; Max< T, F > max_; @@ -974,6 +967,7 @@ namespace Operations void push( const char* buf ) { min_.push( buf ); max_.push( buf ); } void calculate( char* buf ) { if( min_.min_ ) { comma::csv::format::traits< typename Diff< T >::Type >::to_bin( Diff< T >::subtract( *max_.max_, *min_.min_ ) / 2, buf ); } } base* clone() const { return new Radius< T, F >( *this ); } + void reset() { min_ = Min< T, F >(); max_ = Max< T, F >(); } private: Min< T, F > min_; Max< T, F > max_; @@ -987,6 +981,7 @@ namespace Operations void push( const char* ) { ++count_; } void calculate( char* buf ) { comma::csv::format::traits< comma::uint32 >::to_bin( count_, buf ); } base* clone() const { return new Size< T, F >( *this ); } + void reset() { count_ = 0; } private: std::size_t count_; }; @@ -1035,13 +1030,14 @@ namespace Operations template <> struct traits< Enum::kurtosis > { template < typename T, comma::csv::format::types_enum F > struct FromEnum { typedef Kurtosis< T, F > Type; }; }; } // namespace Operations -class Operationbase +class operation_base { public: - virtual ~Operationbase() {} + virtual ~operation_base() {} virtual void push( const char* buf ) = 0; virtual void calculate() = 0; - virtual Operationbase* clone() const = 0; + virtual operation_base* clone() const = 0; + virtual void reset() = 0; const comma::csv::format& output_format() const { return output_format_; } const char* buffer() const { return &buffer_[0]; } @@ -1053,20 +1049,20 @@ class Operationbase std::vector< comma::csv::format::element > output_elements_; std::vector< char > buffer_; - Operationbase* deep_copy_to_( Operationbase* lhs ) const + operation_base* deep_copy_to_( operation_base* lhs ) const { lhs->input_format_ = input_format_; lhs->input_elements_ = input_elements_; lhs->output_format_ = output_format_; lhs->output_elements_ = output_elements_; lhs->buffer_ = buffer_; - for( std::size_t i = 0; i < operations_.size(); ++i ) { lhs->operations_.push_back( operations_[i].clone() ); } + for( auto& o: operations_ ) { lhs->operations_.push_back( o.clone() ); } return lhs; } }; template < Operations::Enum::Values E > -struct Operation : public Operationbase +struct Operation : public operation_base { Operation() {} Operation( const comma::csv::format& format @@ -1079,7 +1075,7 @@ struct Operation : public Operationbase for( std::size_t i = 0; i < input_elements_.size(); ++i ) { comma::csv::format::types_enum output_type = input_elements_[i].type; - switch( E ) // quick and dirty, implement in operations::traits, just no time + switch( E ) // quick and dirty, operations::traits would be better, but likely to be optimized by compiler anyway { case Operations::Enum::radius: case Operations::Enum::diameter: @@ -1131,57 +1127,84 @@ struct Operation : public Operationbase { for( std::size_t i = 0; i < operations_.size(); ++i ) { operations_[i].calculate( &buffer_[0] + output_elements_[i].offset ); } } + + void reset() { for( auto& o: operations_ ) { o.reset(); } } - Operationbase* clone() const { Operation< E >* op = new Operation< E >; return deep_copy_to_( op ); } + operation_base* clone() const { Operation< E >* op = new Operation< E >; return deep_copy_to_( op ); } }; -typedef boost::unordered_map< comma::uint32, boost::ptr_vector< Operationbase >* > OperationsMap; -typedef boost::unordered_map< comma::uint32, std::string > ResultsMap; -typedef std::vector< std::pair < comma::uint32, std::string > > Inputs; +typedef boost::unordered_map< comma::uint32, std::vector< operation_base* >* > operations_map_t; +typedef boost::unordered_map< comma::uint32, std::string > results_map_t; +typedef std::deque< std::pair < comma::uint32, std::string > > inputs_t; -static void init_operations( boost::ptr_vector< Operationbase >& operations - , const std::vector< Operations::operation_parameters >& operations_parameters - , const comma::csv::format& format ) +class operations_battery_farm_t // all this pain is because operations polymorhism is too slow when there are a lot of ids { - static boost::ptr_vector< Operationbase > sample; - if( sample.empty() ) - { - sample.reserve( operations_parameters.size() ); - for( std::size_t i = 0; i < operations_parameters.size(); ++i ) + public: + typedef std::vector< operation_base* > operations_t; + + operations_battery_farm_t(): end_( 0 ) {} + + ~operations_battery_farm_t() + { + for( auto& operation: operations_ ) { for( auto& o: operation ) { delete o; } } // quick and dirty; shame on me + } + + operations_t& make( const std::vector< Operations::operation_parameters >& operations_parameters, const comma::csv::format& format ) { - switch( operations_parameters[i].type ) + if( operations_.empty() ) { - case Operations::Enum::min: sample.push_back( new Operation< Operations::Enum::min >( format ) ); break; - case Operations::Enum::max: sample.push_back( new Operation< Operations::Enum::max >( format ) ); break; - case Operations::Enum::centre: sample.push_back( new Operation< Operations::Enum::centre >( format ) ); break; - case Operations::Enum::mean: sample.push_back( new Operation< Operations::Enum::mean >( format ) ); break; - case Operations::Enum::mode: sample.push_back( new Operation< Operations::Enum::mode >( format ) ); break; - case Operations::Enum::percentile: sample.push_back( new Operation< Operations::Enum::percentile >( format, operations_parameters[i].options ) ); break; - case Operations::Enum::radius: sample.push_back( new Operation< Operations::Enum::radius >( format ) ); break; - case Operations::Enum::diameter: sample.push_back( new Operation< Operations::Enum::diameter >( format ) ); break; - case Operations::Enum::variance: sample.push_back( new Operation< Operations::Enum::variance >( format, operations_parameters[i].options ) ); break; - case Operations::Enum::stddev: sample.push_back( new Operation< Operations::Enum::stddev >( format, operations_parameters[i].options ) ); break; - case Operations::Enum::skew: sample.push_back( new Operation< Operations::Enum::skew >( format, operations_parameters[i].options ) ); break; - case Operations::Enum::kurtosis: sample.push_back( new Operation< Operations::Enum::kurtosis >( format, operations_parameters[i].options ) ); break; - case Operations::Enum::sum: sample.push_back( new Operation< Operations::Enum::sum >( format ) ); break; - case Operations::Enum::size: sample.push_back( new Operation< Operations::Enum::size >( format ) ); break; + operations_.push_back( operations_t() ); + operations_[0].reserve( operations_parameters.size() ); + for( std::size_t i = 0; i < operations_parameters.size(); ++i ) + { + switch( operations_parameters[i].type ) + { + case Operations::Enum::min: operations_[0].push_back( new Operation< Operations::Enum::min >( format ) ); break; + case Operations::Enum::max: operations_[0].push_back( new Operation< Operations::Enum::max >( format ) ); break; + case Operations::Enum::centre: operations_[0].push_back( new Operation< Operations::Enum::centre >( format ) ); break; + case Operations::Enum::mean: operations_[0].push_back( new Operation< Operations::Enum::mean >( format ) ); break; + case Operations::Enum::mode: operations_[0].push_back( new Operation< Operations::Enum::mode >( format ) ); break; + case Operations::Enum::percentile: operations_[0].push_back( new Operation< Operations::Enum::percentile >( format, operations_parameters[i].options ) ); break; + case Operations::Enum::radius: operations_[0].push_back( new Operation< Operations::Enum::radius >( format ) ); break; + case Operations::Enum::diameter: operations_[0].push_back( new Operation< Operations::Enum::diameter >( format ) ); break; + case Operations::Enum::variance: operations_[0].push_back( new Operation< Operations::Enum::variance >( format, operations_parameters[i].options ) ); break; + case Operations::Enum::stddev: operations_[0].push_back( new Operation< Operations::Enum::stddev >( format, operations_parameters[i].options ) ); break; + case Operations::Enum::skew: operations_[0].push_back( new Operation< Operations::Enum::skew >( format, operations_parameters[i].options ) ); break; + case Operations::Enum::kurtosis: operations_[0].push_back( new Operation< Operations::Enum::kurtosis >( format, operations_parameters[i].options ) ); break; + case Operations::Enum::sum: operations_[0].push_back( new Operation< Operations::Enum::sum >( format ) ); break; + case Operations::Enum::size: operations_[0].push_back( new Operation< Operations::Enum::size >( format ) ); break; + } + } } + if( end_ == operations_.size() ) + { + operations_.push_back( operations_t( operations_[0].size() ) ); + for( unsigned int i = 0; i < operations_[0].size(); ++i ) { operations_.back()[i] = operations_[0][i]->clone(); } + } + for( auto& s: operations_[end_] ) { s->reset(); } + return operations_[ end_++ ]; } - } - operations.clear(); - for( std::size_t i = 0; i < sample.size(); ++i ) { operations.push_back( sample[i].clone() ); } -} + + void reset() { end_ = 0; } + + private: + typedef std::deque< operations_t > operations_t_; + operations_t_ operations_; + unsigned int end_; +}; -static void output( const comma::csv::options& csv, ResultsMap& results, boost::optional< comma::uint32 > block, bool has_block, bool has_id ) +static operations_battery_farm_t operations_battery_farm; + +static void output( const comma::csv::options& csv, results_map_t& results, boost::optional< comma::uint32 > block, bool has_block, bool has_id ) { - for( ResultsMap::iterator it = results.begin(); it != results.end(); ++it ) + for( results_map_t::iterator it = results.begin(); it != results.end(); ++it ) { - std::cout << it->second; + std::cout.write( &it->second[0], it->second.size() ); if( csv.binary() ) { if( has_id ) { std::cout.write( reinterpret_cast< const char* >( &it->first ), sizeof( comma::uint32 ) ); } // quick and dirty if( has_block ) { std::cout.write( reinterpret_cast< const char* >( &( *block ) ), sizeof( comma::uint32 ) ); } // quick and dirty - std::cout.flush(); + if( csv.flush ) { std::cout.flush(); } } else { @@ -1193,34 +1216,50 @@ static void output( const comma::csv::options& csv, ResultsMap& results, boost:: results.clear(); } -static void append_and_output( const comma::csv::options& csv, Inputs& inputs, ResultsMap& results ) -{ +static void append_and_output( const comma::csv::options& csv, inputs_t& inputs, results_map_t& results, std::unordered_set< comma::uint32 >& ids ) +{ for ( size_t i = 0; i < inputs.size(); ++i ) { std::cout << inputs[i].second; - if (!csv.binary()) { std::cout << csv.delimiter; } - std::cout << results.find(inputs[i].first)->second; - if (!csv.binary()) { std::cout << std::endl; } + if( !csv.binary() ) { std::cout << csv.delimiter; } + const auto& r = results.find( inputs[i].first )->second; + std::cout.write( &r[0], r.size() ); + if( !csv.binary() ) { std::cout << std::endl; } } + if( csv.flush ) { std::cout.flush(); } results.clear(); inputs.clear(); + ids.clear(); } -static void calculate( const comma::csv::options& csv, OperationsMap& operations, ResultsMap& results ) +static void calculate( const comma::csv::options& csv, operations_map_t& operations, results_map_t& results ) { - for( OperationsMap::iterator it = operations.begin(); it != operations.end(); ++it ) + for( operations_map_t::iterator it = operations.begin(); it != operations.end(); ++it ) { std::string r; + if( csv.binary() ) + { + unsigned int size = 0; + for( std::size_t i = 0; i < it->second->size(); ++i ) { size += ( *it->second )[i]->output_format().size(); } + r.reserve( size ); + } for( std::size_t i = 0; i < it->second->size(); ++i ) { - ( *it->second )[i].calculate(); - if( csv.binary() ) { r.append( ( *it->second )[i].buffer(), ( *it->second )[i].output_format().size() ); } - else { if( i > 0 ) { r += csv.delimiter; } r.append(( *it->second )[i].output_format().bin_to_csv( ( *it->second )[i].buffer(), csv.delimiter, 12 )); } + ( *it->second )[i]->calculate(); + if( csv.binary() ) + { + r.append( ( *it->second )[i]->buffer(), ( *it->second )[i]->output_format().size() ); + } + else + { + if( i > 0 ) { r += csv.delimiter; } + r.append( ( *it->second )[i]->output_format().bin_to_csv( ( *it->second )[i]->buffer(), csv.delimiter, csv.precision ) ); + } } - results[it->first] = r; + results[ it->first ] = r; } - for( OperationsMap::iterator it = operations.begin(); it != operations.end(); ++it ) { delete it->second; } // quick and dirty operations.clear(); + operations_battery_farm.reset(); } int main( int ac, char** av ) @@ -1229,11 +1268,14 @@ int main( int ac, char** av ) { comma::command_line_options options( ac, av, usage ); if( options.exists( "--bash-completion" ) ) bash_completion( ac, av ); - std::vector< std::string > unnamed = options.unnamed( "", "--binary,-b,--delimiter,-d,--format,--fields,-f,--output-fields" ); + std::vector< std::string > unnamed = options.unnamed( "--append,--append-once,--append-to-first,--flush,--output-fields,--output-format", "--binary,-b,--delimiter,-d,--format,--fields,-f,--output-fields" ); comma::csv::options csv( options ); + csv.full_xpath = false; + std::cout.precision( csv.precision ); #ifdef WIN32 if( csv.binary() ) { _setmode( _fileno( stdin ), _O_BINARY ); _setmode( _fileno( stdout ), _O_BINARY ); } #endif + if( !csv.flush && csv.binary() ) { std::cin.tie( NULL ); std::ios_base::sync_with_stdio( false ); } // todo? quick and dirty, redesign binary_input instead? if( unnamed.empty() ) { std::cerr << comma::verbose.app_name() << ": please specify operations" << std::endl; exit( 1 ); } std::vector< std::string > v = comma::split( unnamed[0], ',' ); std::vector< Operations::operation_parameters > operations_parameters( v.size() ); @@ -1243,21 +1285,22 @@ int main( int ac, char** av ) operations_parameters[i].type = Operations::from_name( p[0] ); if( p.size() == 2 ){ operations_parameters[i].options = comma::split( p[1], ':' ); } } - boost::optional< comma::csv::format > format; + boost::optional< comma::csv::format > format = { comma::silent_none< comma::csv::format >() }; if( csv.binary() ) { format = csv.format(); } else if( options.exists( "--format" ) ) { format = comma::csv::format( options.value< std::string >( "--format" ) ); } boost::scoped_ptr< ascii_input > ascii; boost::scoped_ptr< binary_input > binary; if( csv.binary() ) { binary.reset( new binary_input( csv ) ); } else { ascii.reset( new ascii_input( csv, format ) ); } - OperationsMap operations; - ResultsMap results; - Inputs inputs; + operations_map_t operations; + results_map_t results; + inputs_t inputs; + std::unordered_set< comma::uint32 > ids; // quick and dirty boost::optional< comma::uint32 > block = boost::make_optional< comma::uint32 >( false, 0 ); bool has_block = csv.has_field( "block" ); bool has_id = csv.has_field( "id" ); - bool append = options.exists("--append"); - + bool append_once = options.exists( "--append-once,--append-to-first" ); + bool append = options.exists( "--append" ) || append_once; if( options.exists( "--output-fields" ) ) { std::vector < std::string > fields = comma::split(csv.fields, ','); @@ -1267,32 +1310,28 @@ int main( int ac, char** av ) std::replace(v[op].begin(), v[op].end(), '=', '_'); std::replace(v[op].begin(), v[op].end(), '.', '_'); std::replace(v[op].begin(), v[op].end(), ':', '_'); - for (std::size_t f = 0; f < fields.size(); f++ ) + for( std::size_t f = 0; f < fields.size(); f++ ) { - if (fields[f] == "" || fields[f] == "id" || fields[f] == "block") { continue; } - output_fields.push_back(fields[f] + "/" + v[op]); + if( fields[f] == "" || fields[f] == "id" || fields[f] == "block" ) { continue; } + output_fields.push_back( fields[f] + "/" + v[op] ); } } - if (has_id && !append) { output_fields.push_back("id"); } - if (has_block && !append ) { output_fields.push_back("block"); } - std::cout << comma::join(output_fields, ',') << std::endl; + if( has_id && !append ) { output_fields.push_back( "id" ); } + if( has_block && !append ) { output_fields.push_back( "block" ); } + std::cout << comma::join( output_fields, ',' ) << std::endl; return 0; } - if (options.exists("--output-format")) + if( options.exists( "--output-format" ) ) { if ( !format ) { std::cerr << comma::verbose.app_name() << ": option --output-format requires input format to be specified, please use --format or --binary" << std::endl; return 1; } - boost::ptr_vector< Operationbase > ops; - init_operations(ops, operations_parameters, Values(csv, *format).format()); - for ( std::size_t i = 0; i < ops.size(); ++i ) - { - if ( i > 0 ) { std::cout << csv.delimiter; } - std::cout << ops[i].output_format().string(); - } - if (has_id && !append) { std::cout << csv.delimiter << "ui"; } - if (has_block && !append) { std::cout << csv.delimiter << "ui"; } + auto ops = operations_battery_farm.make( operations_parameters, Values( csv, *format ).format() ); + std::cout << ops[0]->output_format().string(); + for( std::size_t i = 1; i < ops.size(); ++i ) { std::cout << ',' << ops[i]->output_format().string(); } + if( has_id && !append ) { std::cout << ",ui"; } + if( has_block && !append ) { std::cout << ",ui"; } std::cout << std::endl; return 0; - } + } while( std::cin.good() && !std::cin.eof() ) { const Values* v = csv.binary() ? binary->read() : ascii->read(); @@ -1300,28 +1339,27 @@ int main( int ac, char** av ) if( has_block ) { if( block && *block != v->block() ) - { - calculate(csv, operations, results); - if ( append ) { append_and_output(csv, inputs, results); inputs.clear(); } - else { output( csv, results, block, has_block, has_id ); } + { + calculate( csv, operations, results ); + if ( append ) { append_and_output( csv, inputs, results, ids ); } else { output( csv, results, block, has_block, has_id ); } } block = v->block(); } - OperationsMap::iterator it = operations.find( v->id() ); - if( it == operations.end() ) + operations_map_t::iterator it = operations.find( v->id() ); + if( it == operations.end() ) { it = operations.insert( std::make_pair( v->id(), &operations_battery_farm.make( operations_parameters, v->format() ) ) ).first; } + if( append ) { - it = operations.insert( std::make_pair( v->id(), new boost::ptr_vector< Operationbase > ) ).first; - init_operations( *it->second, operations_parameters, v->format() ); + if( !append_once || ids.find( v->id() ) == ids.end() ) { inputs.push_back( std::make_pair( v->id(), csv.binary() ? binary->line() : ascii->line() ) ); } + ids.insert( v->id() ); // quick and dirty } - if (append) { inputs.push_back( std::make_pair( v->id(), csv.binary() ? binary->line() : ascii->line() ) ); } - for( std::size_t i = 0; i < it->second->size(); ++i ) { ( *it->second )[i].push( v->buffer() ); } + for( std::size_t i = 0; i < it->second->size(); ++i ) { ( *it->second )[i]->push( v->buffer() ); } } - calculate(csv, operations, results); - if ( append ) { append_and_output(csv, inputs, results); } + calculate( csv, operations, results ); + if ( append ) { append_and_output( csv, inputs, results, ids ); } else { output( csv, results, block, has_block, has_id ); } return 0; } - catch( std::exception& ex ) { std::cerr << comma::verbose.app_name() << ": " << ex.what() << std::endl; } - catch( ... ) { std::cerr << comma::verbose.app_name() << ": unknown exception" << std::endl; } + catch( std::exception& ex ) { std::cerr << "csv-calc: " << ex.what() << std::endl; } + catch( ... ) { std::cerr << "csv-calc: unknown exception" << std::endl; } return 1; } diff --git a/csv/applications/csv-calc.new.cpp b/csv/applications/csv-calc.new.cpp index e9bf661d9..84642d4b4 100644 --- a/csv/applications/csv-calc.new.cpp +++ b/csv/applications/csv-calc.new.cpp @@ -1,32 +1,7 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney // All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/// @author vsevolod vlaskine #include #include @@ -36,10 +11,9 @@ #include #include #include -#include +#include #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../base/types.h" #include "../../csv/stream.h" #include "../../csv/impl/unstructured.h" @@ -82,7 +56,6 @@ static void usage() std::cerr << "examples" << std::endl; std::cerr << " todo" << std::endl; std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; std::cerr << std::endl; exit( 1 ); } @@ -182,7 +155,6 @@ static std::pair< entry_t, comma::csv::options > make_input_( const comma::csv:: else { vf.push_back( "" ); } } p.second.fields = comma::join( vf, ',' ); - p.second.full_xpath = true; return p; } @@ -360,7 +332,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // { // get_< R >( *result_ ).push_back( R() ); // bound_functors< T > b( boost::bind( f.reset, boost::ref( get_< R >( *result_ ).back() ) ) -// , boost::bind( f.update, boost::ref( get_< R >( *result_ ).back(), _1 ) ) +// , boost::bind( f.update, boost::ref( get_< R >( *result_ ).back(), boost::placeholders::_1 ) ) // , boost::bind( f.finalize, boost::ref( get_< R >( *result_ ).back() ) ) ); // get_< T >().push_back( b ); // } @@ -385,7 +357,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef T type; // typedef T result_type; // static void apply( result_type& lhs, const T& rhs ) { if( rhs < lhs ) { lhs = rhs; } } -// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &min::apply, _1, _2 ), std::numeric_limits< T >::max() ); } +// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &min::apply, boost::placeholders::_1, _2 ), std::numeric_limits< T >::max() ); } // }; // // template <> struct min< std::string > @@ -393,7 +365,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef std::string type; // typedef std::string result_type; // static void apply( result_type&, const std::string& ) {} -// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &min::apply, _1, _2 ) ); } +// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &min::apply, boost::placeholders::_1, _2 ) ); } // }; // // template <> struct min< boost::posix_time::ptime > @@ -401,7 +373,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef boost::posix_time::ptime type; // typedef boost::posix_time::ptime result_type; // static void apply( boost::posix_time::ptime& lhs, const boost::posix_time::ptime& rhs ) { if( lhs.is_not_a_date_time() || rhs < lhs ) { lhs = rhs; } } -// static operation::options< result_type, boost::posix_time::ptime > options() { return operation::options< result_type, boost::posix_time::ptime >( boost::bind( &min::apply, _1, _2 ) ); } +// static operation::options< result_type, boost::posix_time::ptime > options() { return operation::options< result_type, boost::posix_time::ptime >( boost::bind( &min::apply, boost::placeholders::_1, _2 ) ); } // }; // // template < typename T > struct max @@ -409,7 +381,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef T type; // typedef T result_type; // static void apply( result_type& lhs, const T& rhs ) { if( lhs < rhs ) { lhs = rhs; } } -// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &max::apply, _1, _2 ), std::numeric_limits< T >::min() ); } +// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &max::apply, boost::placeholders::_1, _2 ), std::numeric_limits< T >::min() ); } // }; // // template <> struct max< std::string > @@ -417,7 +389,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef std::string type; // typedef std::string result_type; // static void apply( result_type& lhs, const std::string& rhs ) { if( lhs < rhs ) { lhs = rhs; } } -// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &max::apply, _1, _2 ) ); } +// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &max::apply, boost::placeholders::_1, _2 ) ); } // }; // // template <> struct min< boost::posix_time::ptime > @@ -425,7 +397,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef boost::posix_time::ptime type; // typedef boost::posix_time::ptime result_type; // static void apply( boost::posix_time::ptime& lhs, const boost::posix_time::ptime& rhs ) { if( lhs.is_not_a_date_time() || rhs < lhs ) { lhs = rhs; } } -// static operation::options< result_type, boost::posix_time::ptime > options() { return operation::options< result_type, boost::posix_time::ptime >( boost::bind( &max::apply, _1, _2 ) ); } +// static operation::options< result_type, boost::posix_time::ptime > options() { return operation::options< result_type, boost::posix_time::ptime >( boost::bind( &max::apply, boost::placeholders::_1, _2 ) ); } // }; // // template < typename T > struct size @@ -433,7 +405,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef T type; // typedef double result_type; // static void apply( result_type& lhs, const T& rhs ) { ++rhs; } -// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &size::apply, _1, _2 ), 0 ); } +// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &size::apply, boost::placeholders::_1, _2 ), 0 ); } // }; // // template < typename T > struct sum @@ -441,7 +413,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef T type; // typedef T result_type; // static void apply( result_type& lhs, const T& rhs ) { lhs += rhs; } -// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &sum::apply, _1, _2 ), 0 ); } +// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &sum::apply, boost::placeholders::_1, _2 ), 0 ); } // }; // // template <> struct sum< std::string > @@ -449,7 +421,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef std::string type; // typedef std::string result_type; // static void apply( result_type&, const std::string& ) {} -// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &sum::apply, _1, _2 ) ); } +// static operation::options< result_type, T > options() { return operation::options< result_type, T >( boost::bind( &sum::apply, boost::placeholders::_1, _2 ) ); } // }; // // template <> struct sum< boost::posix_time::ptime > @@ -457,7 +429,7 @@ template <> struct bound_result_traits< boost::posix_time::ptime > // typedef boost::posix_time::ptime type; // typedef boost::posix_time::ptime result_type; // static void apply( boost::posix_time::ptime&, const boost::posix_time::ptime& ) {} -// static operation::options< result_type, boost::posix_time::ptime > options() { return operation::options< result_type, boost::posix_time::ptime >( boost::bind( &sum::apply, _1, _2 ) ); } +// static operation::options< result_type, boost::posix_time::ptime > options() { return operation::options< result_type, boost::posix_time::ptime >( boost::bind( &sum::apply, boost::placeholders::_1, _2 ) ); } // }; // // } // namespace operations { diff --git a/csv/applications/csv-cast.cpp b/csv/applications/csv-cast.cpp index a1b1a83ac..bec891db7 100644 --- a/csv/applications/csv-cast.cpp +++ b/csv/applications/csv-cast.cpp @@ -39,13 +39,12 @@ #include #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../base/exception.h" #include "../../csv/format.h" static const std::string app_name = "csv-cast"; -static void usage() +static void usage( bool verbose = false ) { std::cerr << "reads binary in the given input format and writes binary in the given output format" << std::endl; std::cerr << std::endl; @@ -55,6 +54,7 @@ static void usage() std::cerr << std::endl; std::cerr << " --binary,-b,--from: input binary format" << std::endl; std::cerr << " --output-binary,--output,-o,--to: output binary format" << std::endl; + std::cerr << " --flush: flush stdout after each record" << std::endl; std::cerr << " --force: allow narrowing conversions" << std::endl; std::cerr << std::endl; std::cerr << comma::csv::format::usage() << std::endl; @@ -75,8 +75,8 @@ static void usage() std::cerr << " lexical cast, convert s[22],s[10],s[6] to t,2d (creates sample binary data)" << std::endl; std::cerr << " echo {0..9}.2345789,3.1415 | fmt -1 | csv-time-stamp | csv-to-bin s[22],s[10],s[6] | csv-cast s[22],s[10],s[6] t,2d | csv-from-bin t,2d" << std::endl; std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; std::cerr << std::endl; + exit(0); } static void check_conversions( const comma::csv::format& iformat, const comma::csv::format& oformat, const bool force ) @@ -256,21 +256,23 @@ int main( int ac, char** av ) #endif try { - comma::command_line_options options( ac, av ); - if( options.exists( "--help,-h" ) ) { usage(); return 0; } + comma::command_line_options options( ac, av, usage ); if( ac < 3 ) { usage(); return 1; } comma::csv::format iformat( options.value< std::string >( "--binary,-b,--from", av[1] ) ); comma::csv::format oformat( options.value< std::string >( "--output-binary,--output,-o,--to", av[2] ) ); check_conversions( iformat, oformat, options.exists( "--force" ) ); + bool flush = options.exists( "--flush" ); std::vector< char > in( iformat.size() ); std::vector< char > out( oformat.size() ); + if( !flush ) { std::cin.tie( NULL ); } while( std::cin.good() ) { std::cin.read( &in[0], iformat.size() ); if( std::cin.gcount() == 0 ) { break; } if( std::cin.gcount() < static_cast< int >( iformat.size() ) ) { COMMA_THROW( comma::exception, "expected " << iformat.size() << " bytes, got only " << std::cin.gcount() ); } cast( iformat, in, oformat, out ); - std::cout.write( &out[0], oformat.size() ).flush(); + std::cout.write( &out[0], oformat.size() ); + if( flush ) { std::cout.flush(); } } return 0; } diff --git a/csv/applications/csv-crc.cpp b/csv/applications/csv-crc.cpp index cdaabdc39..703bc14a2 100644 --- a/csv/applications/csv-crc.cpp +++ b/csv/applications/csv-crc.cpp @@ -27,7 +27,6 @@ // OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine /// @author james underwood @@ -42,55 +41,56 @@ #include #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../base/types.h" static void usage( bool ) { - std::cerr << std::endl; - std::cerr << "wrap/check crc on fixed-width input (ascii or binary)" << std::endl; - std::cerr << std::endl; - std::cerr << "usage: csv-crc []" << std::endl; - std::cerr << std::endl; - std::cerr << "" << std::endl; - std::cerr << " wrap: add crc" << std::endl; - std::cerr << " check: check crc; exit, if check fails" << std::endl; - std::cerr << " recover: recover with given parameters (see below)" << std::endl; - std::cerr << std::endl; - std::cerr << "data options" << std::endl; - std::cerr << " --crc-size; output given crc size to stdout and exit" << std::endl; - std::cerr << " --delimiter,-d=: ascii csv delimiter" << std::endl; - std::cerr << " --size=: binary data size; if absent, expect ascii csv" << std::endl; - std::cerr << " for wrap: payload size" << std::endl; - std::cerr << " for check/recover size including crc" << std::endl; - std::cerr << std::endl; - std::cerr << "crc options" << std::endl; - std::cerr << " --crc=:" << std::endl; - std::cerr << " 16: 16-bit, generator 0x8805" << std::endl; - std::cerr << " ccitt: 16-bit, generator 0x1021" << std::endl; - std::cerr << " xmodem: 16-bit, generator 0x1021" << std::endl; - std::cerr << " 32: 32-bit, generator 0x04C11DB7" << std::endl; - //std::cerr << " checksum16: simple 16-bit checksum (todo)" << std::endl; - //std::cerr << " checksum32: simple 32-bit checksum (todo)" << std::endl; - std::cerr << " default: ccitt" << std::endl; - std::cerr << " --big-endian,--net-byte-order: if binary, crc is big endian" << std::endl; - std::cerr << " --verbose,-v: more output" << std::endl; - std::cerr << std::endl; - std::cerr << " For a definitive list of 16 bits CRC algorithms see:" << std::endl; - std::cerr << " http://reveng.sourceforge.net/crc-catalogue/16.htm" << std::endl; - std::cerr << std::endl; - std::cerr << "recovery options" << std::endl; - std::cerr << " --give-up-after=: if check fails, give up after bytes" << std::endl; - std::cerr << " if absent and no --recover-after: default 0" << std::endl; - std::cerr << " if absent and --recover-after: default infinity (don't give up)" << std::endl; - std::cerr << " --recover-after=: if check fails and then new valid crc found" << std::endl; - std::cerr << " make sure that at least subsequent lines (ascii) or" << std::endl; - std::cerr << " packets (binary) are valid, before output; default 0, i.e. recover on the next valid" << std::endl; - std::cerr << " --discard-on-recovery,--discard: discard those packets accumulated during the recovery check" << std::endl; - std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; - std::cerr << std::endl; - exit( 1 ); + std::cerr << R"( +wrap/check crc on fixed-width input (ascii or binary) + +usage: csv-crc [] + + + wrap: add crc + check: check crc; exit if check fails + recover: recover with given parameters (see below) + +general options + --help,-h; this help + --verbose,-v: more output + +data options + --crc-size; output given crc size to stdout and exit + --delimiter,-d=[]: ascii csv delimiter + --size=[]: binary data size; if absent, expect ascii csv + for wrap: payload size + for check/recover: size including crc + +crc options + --crc=: + 16: 16-bit, generator 0x8805 + ccitt: 16-bit, generator 0x1021 + xmodem: 16-bit, generator 0x1021 + 32: 32-bit, generator 0x04C11DB7 + default: ccitt + --big-endian,--net-byte-order: if binary, crc is big endian + +recover options + --give-up-after=: if check fails, give up after bytes + default: infinity; don't give up + --recover-after=: if check fails and then new valid crc found + make sure that at least subsequent lines (ascii) + or packets (binary) are valid, before output; + default: 0; recover on the next valid packet + --discard-on-recovery,--discard: discard packets accumulated when recovering + + Note that the check command is equivalent to + csv-crc recover --give-up-after 0 + +For a definitive list of 16 bit CRC algorithms see: +http://reveng.sourceforge.net/crc-catalogue/16.htm +)"; + exit( 0 ); } static bool verbose; @@ -144,6 +144,7 @@ static bool run_() bool recovered = true; std::size_t recovered_count = 0; std::size_t recovered_byte_count = 0; + std::size_t current_recovered_byte_count = 0; std::vector< char > recovery_buffer( recover_after * size ); while( std::cin.good() && !std::cin.eof() ) { @@ -165,35 +166,44 @@ static bool run_() if( big_endian ) { expected = traits< typename Crc::value_type >::hton( expected ); } if( crc == expected ) { + bool output_input_buffer = true; if( !recovered ) { if( recovered_count == recover_after ) { - std::cerr << "csv-crc: recovered after " << recovered_byte_count << " byte(s)" << std::endl; + comma::say() << "recovered after " << recovered_byte_count << " byte(s)" << std::endl; if( !discard_on_recovery ) { std::cout.write( &recovery_buffer[0], recovery_buffer.size() ); } recovered = true; recovered_count = 0; recovered_byte_count = 0; + current_recovered_byte_count = 0; } else { ::memcpy( &recovery_buffer[ recovered_count * size ], p, size ); ++recovered_count; + output_input_buffer = false; // we're just stashing them until we check } } - std::cout.write( p, size ); + if( output_input_buffer ) { std::cout.write( p, size ); } std::cout.flush(); } else // quick and dirty: lots of code duplication, but just to make it working { - if( recovered ) { std::cerr << "csv-crc: crc check failed" << ( !give_up_after || *give_up_after > 0 ? "; recovering..." : "" ) << std::endl; } + if( current_recovered_byte_count / size > recovered_count ) + { + recovered_count = 0; + current_recovered_byte_count = 0; + } + if( recovered ) { comma::say() << "crc check failed" << ( !give_up_after || *give_up_after > 0 ? "; recovering..." : "" ) << std::endl; } recovered = false; - ++recovered_byte_count; + if( give_up_after && recovered_byte_count >= *give_up_after ) { break; } } } unsigned int step = recovered ? size : 1; p += step; offset -= step; + if( !recovered ) { recovered_byte_count += step; current_recovered_byte_count += step; } if( end - p < int( size ) ) { ::memcpy( begin, p, offset ); // todo: quick and dirty, check if works in case of overlapping @@ -205,7 +215,7 @@ static bool run_() if( r <= 0 ) { break; } offset += r; } - if( offset > 0 && offset < size ) { std::cerr << "csv-crc: expected at least " << size << " byte(s), got only " << offset << std::endl; return 1; } + COMMA_ASSERT_BRIEF( offset <= 0 || offset >= size, "expected at least " << size << " byte(s), got only " << offset ); } else { @@ -231,7 +241,7 @@ static bool run_() } else { - std::cerr << "csv-crc: check failed (recovery is not implemented for ascii mode, todo)" << std::endl; + comma::say() << "check failed (recovery is not implemented for ascii mode, todo)" << std::endl; return 1; } } @@ -253,10 +263,10 @@ int main( int ac, char** av ) else if( crc == "ccitt" ) { std::cout << sizeof( boost::crc_ccitt_type::value_type ) << std::endl; } else if( crc == "xmodem" ) { std::cout << sizeof( boost::crc_xmodem_type::value_type ) << std::endl; } else if( crc == "xmodem-boost" ) { std::cout << sizeof( boost::crc_xmodem_type::value_type ) << std::endl; } - else { std::cerr << "csv-crc: expected crc type, got \"" << crc << "\"" << std::endl; return 1; } + else { comma::say() << "expected crc type, got \"" << crc << "\"" << std::endl; return 1; } return 0; } - if( wrap && recover ) { std::cerr << "csv-crc: if 'wrap', then no 'check' or 'recover'" << std::endl; return 1; } + COMMA_ASSERT_BRIEF( !wrap || !recover, "if 'wrap', then no 'check' or 'recover'" ); verbose = options.exists( "--verbose,-v" ); give_up_after = options.optional< unsigned int >( "--give-up-after" ); recover_after = options.value( "--recover-after", 0 ); @@ -266,13 +276,13 @@ int main( int ac, char** av ) big_endian = options.exists( "--big-endian,--net-byte-order" ); delimiter = options.value< char >( "--delimiter,-d", ',' ); std::vector< std::string > commands = options.unnamed( "--discard-on-recovery,--discard,--verbose,-v,--big-endian,--net-byte-order", "--size,--delimiter,-d,--crc,--give-up-after,--recover-after" ); - if( commands.empty() ) { std::cerr << "csv-crc: specify a command" << std::endl; return 1; } + COMMA_ASSERT_BRIEF( !commands.empty(), "please specify a command" ); for( std::size_t i = 0; i < commands.size(); ++i ) { if( commands[i] == "wrap" ) { wrap = true; } else if( commands[i] == "check" ) { recover = true; give_up_after = 0; } else if( commands[i] == "recover" ) { recover = true; } - else { std::cerr << "csv-crc: expected command, got '" << commands[i] << "'" << std::endl; return 1; } + else { comma::say() << "expected command, got '" << commands[i] << "'" << std::endl; return 1; } } // The list of crc versions predefined by boost is given at // http://www.boost.org/doc/libs/1_58_0/libs/crc/crc.html#crc_ex @@ -288,16 +298,9 @@ int main( int ac, char** av ) // the following is designated boost::crc_xmodem_t in the git repo for boost/crc.hpp else if( crc == "xmodem" ) { return run_< boost::crc_optimal< 16, 0x1021, 0, 0, false, false > >(); } else if( crc == "xmodem-boost" ) { return run_< boost::crc_xmodem_type >(); } - std::cerr << "csv-crc: expected crc type, got \"" << crc << "\"" << std::endl; - return 1; - } - catch( std::exception& ex ) - { - std::cerr << "csv-crc: " << ex.what() << std::endl; - } - catch( ... ) - { - std::cerr << "csv-crc: unknown exception" << std::endl; + comma::say() << "expected crc type, got '" << crc << "'" << std::endl; } + catch( std::exception& ex ) { comma::say() << ex.what() << std::endl; } + catch( ... ) { comma::say() << "unknown exception" << std::endl; } return 1; } diff --git a/csv/applications/csv-enumerate.cpp b/csv/applications/csv-enumerate.cpp index a58cfc4f7..10b15b6c7 100644 --- a/csv/applications/csv-enumerate.cpp +++ b/csv/applications/csv-enumerate.cpp @@ -29,7 +29,7 @@ /// @author vsevolod vlaskine -#include +#include #include "../../application/command_line_options.h" #include "../../base/exception.h" #include "../../base/types.h" @@ -39,42 +39,58 @@ static void usage( bool verbose ) { - std::cerr << std::endl; - std::cerr << "append unique id to csv records with the same values; support integer, time, and string fields" << std::endl; - std::cerr << std::endl; - std::cerr << "usage: cat data.csv | csv-enumerate " << std::endl; - std::cerr << std::endl; - std::cerr << "todo: support floating point values as input keys" << std::endl; - std::cerr << std::endl; - std::cerr << "options" << std::endl; - std::cerr << " --fields,-f=; fields of interest, actual field names do not matter; e.g: --fields ,,,a,,b,,,c" << std::endl; - std::cerr << " --format=; if input is ascii and deducing data types may be ambiguous, define field types explicitly, value as in --binary" << std::endl; - std::cerr << " --output-map,--map: do not output input records, only an unsorted records" << std::endl; - std::cerr << " output fields" << std::endl; - std::cerr << " - list of input key values; in same binary as input" << std::endl; - std::cerr << " - corresponding enumeration index as ui" << std::endl; - std::cerr << " - number of values for this enumeration index as ui" << std::endl; - std::cerr << std::endl; - std::cerr << "csv options" << std::endl; - if( verbose ) { std::cerr << comma::csv::options::usage() << std::endl; } else { std::cerr << " run csv-enumerate --help --verbose for more..." << std::endl; } - std::cerr << std::endl; + std::cerr << R"( +append unique id to csv records with the same values; support integer, time, and string fields + +usage: cat data.csv | csv-enumerate + +todo: support floating point values as input keys + +options + --fields,-f=; fields of interest, actual field names do not matter + e.g: --fields ,,,a,,b,,,c + --format=; if input is ascii and deducing data types may be ambiguous, + define field types explicitly, value as in --binary + --output-map,--map: do not output input records, only an unsorted list of keys + output fields + - list of input key values; in same binary as input + - corresponding enumeration index as ui + - number of values for this enumeration index as ui + --verbose,-v: more output to stderr +)" << std::endl; +std::cerr << "csv options" << std::endl; + std::cerr << comma::csv::options::usage( verbose ) << std::endl; exit( 0 ); } +struct output +{ + comma::uint32 id; + output( comma::uint32 id = 0 ): id( id ) {} +}; + +namespace comma { namespace visiting { + +template <> struct traits< output > +{ + template < typename K, typename V > static void visit( const K&, const output& p, V& v ) { v.apply( "id", p.id ); } + template < typename K, typename V > static void visit( const K&, output& p, V& v ) { v.apply( "id", p.id ); } +}; + +} } // namespace comma { namespace visiting { + int main( int ac, char** av ) { typedef comma::csv::impl::unstructured input_t; - typedef boost::unordered_map< comma::csv::impl::unstructured, std::pair< comma::uint32, comma::uint32 >, comma::csv::impl::unstructured::hash > map_t; + typedef std::unordered_map< comma::csv::impl::unstructured, std::pair< comma::uint32, comma::uint32 >, comma::csv::impl::unstructured::hash > map_t; try { comma::command_line_options options( ac, av, usage ); - bool verbose = options.exists( "--verbose,-v" ); bool output_map = options.exists( "--output-map,--map" ); comma::csv::options csv( options ); bool has_non_empty_field = false; for( const auto& f: comma::split( csv.fields, ',' ) ) { if( !f.empty() ) { has_non_empty_field = true; break; } } - if( !has_non_empty_field ) { std::cerr << "csv-enumerate: please specify at least one key in fields" << std::endl; return 1; } - csv.full_xpath = true; + COMMA_ASSERT_BRIEF( has_non_empty_field, "please specify at least one key in fields" ); std::string first_line; comma::csv::format f; if( csv.binary() ) { f = csv.format(); } @@ -84,21 +100,22 @@ int main( int ac, char** av ) while( std::cin.good() && first_line.empty() ) { std::getline( std::cin, first_line ); } if( first_line.empty() ) { return 0; } f = comma::csv::impl::unstructured::guess_format( first_line, csv.delimiter ); - if( verbose ) { std::cerr << "csv-enumerate: guessed format: " << f.string() << std::endl; } + comma::saymore() << "guessed format: " << f.string() << std::endl; } input_t default_input; std::vector< std::string > v = comma::split( csv.fields, ',' ); + std::vector< std::string > format; // quick and dirty + std::vector< std::string > s; + if( csv.binary() ) { format = comma::split( csv.format().expanded_string(), ',' ); } for( unsigned int i = 0; i < v.size(); ++i ) { if( v[i].empty() ) { continue; } v[i] = default_input.append( f.offset( i ).type ); + if( csv.binary() ) { s.push_back( format[i] ); } } - if( verbose ) { std::cerr << "csv-enumerate: fields " << csv.fields << " interpreted as: " << comma::join( v, ',' ) << std::endl; } + std::string map_output_binary_format = comma::join( s, ',' ); + comma::saymore() << "fields " << csv.fields << " interpreted as: " << comma::join( v, ',' ) << std::endl; csv.fields = comma::join( v, ',' ); - comma::csv::input_stream< input_t > istream( std::cin, csv, default_input ); - #ifdef WIN32 - if( istream.is_binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); } - #endif static map_t map; comma::uint32 id = 0; if( !first_line.empty() ) @@ -107,6 +124,15 @@ int main( int ac, char** av ) map[ comma::csv::ascii< input_t >( csv, default_input ).get( first_line ) ] = std::make_pair( id++, 1 ); if( !output_map ) { std::cout << first_line << csv.delimiter << 0 << std::endl; } } + comma::csv::options output_csv; + output_csv.delimiter = csv.delimiter; + if( csv.binary() ) { output_csv.format( comma::csv::format::value< output >() ); } + comma::csv::input_stream< input_t > istream( std::cin, csv, default_input ); + comma::csv::output_stream< output > ostream( std::cout, output_csv ); + comma::csv::tied< input_t, output > tied( istream, ostream ); + #ifdef WIN32 + if( istream.is_binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); } + #endif while( istream.ready() || std::cin.good() ) { const input_t* p = istream.read(); @@ -114,30 +140,21 @@ int main( int ac, char** av ) map_t::iterator it = map.find( *p ); comma::uint32 cur = id; if( it == map.end() ) { map[ *p ] = std::make_pair( id++, 1 ); } else { cur = it->second.first; ++( it->second.second ); } - if( !output_map ) - { - if( csv.binary() ) - { - std::cout.write( istream.binary().last(), csv.format().size() ); - std::cout.write( reinterpret_cast< const char* >( &cur ), sizeof( comma::uint32 ) ); - if( csv.flush ) { std::cout.flush(); } - } - else - { - std::cout << comma::join( istream.ascii().last(), csv.delimiter ) << csv.delimiter << cur << std::endl; - } - } + if( !output_map ) { tied.append( output( cur ) ); } } if( !output_map ) { return 0; } - comma::csv::options output_csv; - output_csv.delimiter = csv.delimiter; - output_csv.full_xpath = true; - if( csv.binary() ) { output_csv.format( comma::csv::format::value< input_t >( default_input ) + ",2ui" ); } - comma::csv::output_stream< map_t::value_type > ostream( std::cout, output_csv, std::make_pair( default_input, std::make_pair( 0, 0 ) ) ); - for( map_t::const_iterator it = map.begin(); it != map.end(); ++it ) { ostream.write( *it ); } + comma::csv::options output_map_csv; + output_map_csv.delimiter = csv.delimiter; + if( csv.binary() ) + { + output_map_csv.format( map_output_binary_format + ",2ui" ); //output_map_csv.format( comma::csv::format::value< input_t >( default_input ) + ",2ui" ); + comma::say() << "binary output format for map: '" << output_map_csv.format().string() << "'" << std::endl; + } + comma::csv::output_stream< map_t::value_type > omstream( std::cout, output_map_csv, std::make_pair( default_input, std::make_pair( 0, 0 ) ) ); + for( map_t::const_iterator it = map.begin(); it != map.end(); ++it ) { omstream.write( *it ); } return 0; } - catch( std::exception& ex ) { std::cerr << "csv-enumerate: " << ex.what() << std::endl; } - catch( ... ) { std::cerr << "csv-enumerate: unknown exception" << std::endl; } + catch( std::exception& ex ) { comma::say() << ex.what() << std::endl; } + catch( ... ) { comma::say() << "unknown exception" << std::endl; } return 1; } diff --git a/csv/applications/csv-fields.cpp b/csv/applications/csv-fields.cpp index 6cea12a13..f2f705901 100644 --- a/csv/applications/csv-fields.cpp +++ b/csv/applications/csv-fields.cpp @@ -29,16 +29,15 @@ /// @author vsevolod vlaskine +#include #include #include #include #include -#include "../../application/contact_info.h" +#include #include "../../application/command_line_options.h" #include "../../string/string.h" -using namespace comma; - static void usage( bool ) { std::cerr << std::endl; @@ -53,6 +52,7 @@ static void usage( bool ) std::cerr << " numbers (default): convert comma-separated field names to field numbers" << std::endl; std::cerr << " e.g. for combining with cut or csv-bin-cut" << std::endl; std::cerr << " --count,--size: output the total number of fields" << std::endl; + std::cerr << " --fields=[]: number only fields with given names, same as csv-fields clear --except ... | csv-fields numbers" << std::endl; std::cerr << " --fill: number even empty fields, e.g. try: echo ,, | csv-fields numbers --fill" << std::endl; std::cerr << " --from=: start field numbering from ; default=1" << std::endl; std::cerr << " to keep it consistent with linux cut utility" << std::endl; @@ -95,6 +95,8 @@ static void usage( bool ) std::cerr << " make-fixed: normalise input to a fixed number of fields" << std::endl; std::cerr << " --count,--size=: number of output fields" << std::endl; std::cerr << " --force: chop input to fields if larger" << std::endl; + std::cerr << " --values=[]: fill missing fields with given values" << std::endl; + std::cerr << " if --count not specified, use number of as desired number of fields" << std::endl; std::cerr << std::endl; std::cerr << "examples" << std::endl; std::cerr << " numbers" << std::endl; @@ -155,29 +157,35 @@ static void usage( bool ) std::cerr << " a,b,c,d,," << std::endl; std::cerr << " x,y,z,,," << std::endl; std::cerr << std::endl; + std::cerr << " make-fixed" << std::endl; + std::cerr << " { echo a,b; echo x,y,z; } | csv-fields make-fixed --count=6 --fields=A,B,C,D,E,F" << std::endl; + std::cerr << " a,b,C,D,E,F" << std::endl; + std::cerr << " x,y,z,D,E,F" << std::endl; + std::cerr << std::endl; std::cerr << " { echo a,b,c,d; echo x,y,z; } | csv-fields make-fixed --count=3 --force" << std::endl; std::cerr << " a,b,c" << std::endl; std::cerr << " x,y,z" << std::endl; std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; - std::cerr << std::endl; - exit( 1 ); + exit( 0 ); } int main( int ac, char** av ) { try { - command_line_options options( ac, av, usage ); + comma::command_line_options options( ac, av, usage ); std::string operation = "numbers"; const std::vector< std::string > unnamed = options.unnamed( "--help,-h", "-.*" ); char delimiter = options.value( "--delimiter,-d", ',' ); if( !unnamed.empty() ) { operation = unnamed[0]; } - if( operation == "numbers" ) + auto numbers = [&]()->int { int from = options.value( "--from", 1 ); bool fill = options.exists( "--fill" ); - std::string prefix = options.value< std::string >( "--prefix", "" ); + options.assert_mutually_exclusive( "--fill,--fields" ); + const auto& v = comma::split( options.value< std::string >( "--fields", "" ), ',', true ); + std::set< std::string > fields( v.begin(), v.end() ); + std::string prefix = options.value< std::string >( "--prefix", "" ); while( std::cin.good() ) { std::string line; @@ -189,14 +197,15 @@ int main( int ac, char** av ) for( unsigned int i = 0; i < v.size(); ++i ) { if( v[i].empty() && !fill ) { continue; } + if( !fields.empty() && fields.find( v[i] ) == fields.end() ) { continue; } std::cout << comma << prefix << ( i + from ); comma = ','; } std::cout << std::endl; } return 0; - } - if( operation == "clear" ) + }; + auto clear = [&]()->int { options.assert_mutually_exclusive( "--except,--keep,--mask,--remove,--inverted-mask,--complement-mask,--unmask,--unmasked" ); std::string keep = options.value< std::string >( "--keep,--except", "" ); @@ -249,8 +258,8 @@ int main( int ac, char** av ) std::cout << std::endl; } return 0; - } - if( operation == "default" ) + }; + auto default_operation = [&]()->int { options.assert_mutually_exclusive( "--value,--values" ); std::vector< std::string > defaults; @@ -276,8 +285,8 @@ int main( int ac, char** av ) std::cout << std::endl; } return 0; - } - if( operation == "prefix" ) + }; + auto prefix = [&]()->int { options.assert_mutually_exclusive( "--fields,--except" ); const std::string& e = options.value< std::string >( "--except", "" ); @@ -309,8 +318,8 @@ int main( int ac, char** av ) std::cout << std::endl; } return 0; - } - if( operation == "rename" ) + }; + auto rename = [&]()->int { const std::vector< std::string >& fields = comma::split( options.value< std::string >( "--fields" ), ',' ); const std::vector< std::string >& to = comma::split( options.value< std::string >( "--to" ), ',' ); @@ -334,8 +343,8 @@ int main( int ac, char** av ) std::cout << std::endl; } return 0; - } - if( operation == "strip" ) + }; + auto strip = [&]()->int { options.assert_mutually_exclusive( "--fields,--except" ); const std::string& e = options.value< std::string >( "--except", "" ); @@ -364,8 +373,8 @@ int main( int ac, char** av ) std::cout << std::endl; } return 0; - } - if( operation == "cut" ) + }; + auto cut = [&]()->int { options.assert_mutually_exclusive( "--except,--fields", "--empty" ); bool except = options.exists( "--except" ); @@ -394,8 +403,8 @@ int main( int ac, char** av ) std::cout << std::endl; } return 0; - } - if( operation == "has" ) + }; + auto has = [&]()->int { const std::string& f = options.value< std::string >( "--fields" ); const std::vector< std::string >& v = comma::split( f, delimiter ); @@ -411,10 +420,12 @@ int main( int ac, char** av ) if( !matches ) { return 1; } if( any ) { return 0; } return matches == fields.size() ? 0 : 1; - } - if( operation == "make-fixed" ) + }; + auto make_fixed = [&]()->int { - const unsigned int count = options.value< unsigned int >( "--count,--size" ); + const std::vector< std::string >& values = comma::split( options.value< std::string >( "--values", "" ), ',', true ); + const unsigned int count = options.value< unsigned int >( "--count,--size", values.size() ); + if( count == 0 ) { std::cerr << "csv-fields: make-fixed: please specify either --count or --values" << std::endl; } bool force = options.exists( "--force" ); while( std::cin.good() ) { @@ -425,7 +436,7 @@ int main( int ac, char** av ) if( v.size() <= count ) { std::cout << line; - for( unsigned int i = v.size(); i < count; i++ ) { std::cout << delimiter; } + for( unsigned int i = v.size(); i < count; i++ ) { std::cout << delimiter << ( i < values.size() ? values[i] : std::string() ); } } else { @@ -436,8 +447,19 @@ int main( int ac, char** av ) std::cout << std::endl; } return 0; - } - std::cerr << "csv-fields: expected operation, got: \"" << operation << "\"" << std::endl; + }; + std::unordered_map< std::string, std::function< int() > > operations = { { "clear", clear } + , { "cut", cut } + , { "default", default_operation } + , { "has", has } + , { "make-fixed", make_fixed } + , { "numbers", numbers } + , { "prefix", prefix } + , { "rename", rename } + , { "strip", strip } }; + auto o = operations.find( operation ); + if( o != operations.end() ) { return o->second(); } + std::cerr << "csv-fields: expected operation, got '" << operation << "'" << std::endl; return 1; } catch( std::exception& ex ) { std::cerr << "csv-fields: " << ex.what() << std::endl; } diff --git a/csv/applications/csv-format.cpp b/csv/applications/csv-format.cpp index ab0b7bff6..f9cbc8bd4 100644 --- a/csv/applications/csv-format.cpp +++ b/csv/applications/csv-format.cpp @@ -33,20 +33,15 @@ #include #include #include -#include #include -#include "../../application/contact_info.h" #include "../../application/command_line_options.h" #include "../../csv/format.h" #include "../../string/string.h" -using namespace comma; -static const char *app_name = "csv-format"; - -static void usage() +static void usage( bool verbose = false ) { std::cerr << std::endl; - std::cerr << "usage: echo \"3f,2f,d\" | " << app_name << " [options] (expand|collapse|count|repeat)" << std::endl; + std::cerr << "usage: echo \"3f,2f,d\" | csv-format [options] (expand|collapse|count|repeat)" << std::endl; std::cerr << std::endl; std::cerr << "perform various operations on format strings" << std::endl; std::cerr << std::endl; @@ -56,6 +51,7 @@ static void usage() std::cerr << " guess: take a csv string, output (roughly) guessed format; e.g: echo 20170101T000000,1,2,3 | csv-format guess --format ,,ui" << std::endl; std::cerr << " expand: output fully expand format, e.g. 2i,3f -> i,i,f,f,f" << std::endl; std::cerr << " repeat: replicate the format n times, e.g. 2i,3f --count 2 -> 2i,3f,2i,3f" << std::endl; + std::cerr << " size: output format size in bytes, e.g: echo 2i,3f | csv-format size would output 20" << std::endl; std::cerr << std::endl; std::cerr << "options" << std::endl; std::cerr << " guess:" << std::endl; @@ -64,9 +60,7 @@ static void usage() std::cerr << " repeat:" << std::endl; std::cerr << " --count=n: replicate the format n times" << std::endl; std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; - std::cerr << std::endl; - exit( 1 ); + exit( 0 ); } static std::string incomplete_expanded( const std::string& s ) // quick and dirty @@ -116,19 +110,18 @@ int main( int ac, char** av ) { try { - command_line_options options( ac, av ); - if( options.exists( "--help,-h" ) ) { usage(); } + comma::command_line_options options( ac, av, usage ); const std::vector< std::string > unnamed = options.unnamed( "--help,-h", "-.*" ); - if( unnamed.empty() ) - { std::cerr << app_name << ": expected an operation (e.g. \"expand\")" << std::endl; return 1; } - if( unnamed.size() != 1 ) { usage(); } + if( unnamed.empty() ) { std::cerr << "csv-format: please specify operation" << std::endl; return 1; } + if( unnamed.size() > 1 ) { std::cerr << "csv-format: please only one operation" << std::endl; return 1; } std::string operation = unnamed[0]; std::string line; std::function< void( const std::string& ) > handle; if( operation == "expand" ) { handle = [&]( const std::string& s ) { std::cout << comma::csv::format( s ).expanded_string() << std::endl; }; } else if( operation == "collapse" ) { handle = [&]( const std::string& s ) { std::cout << comma::csv::format( s ).collapsed_string() << std::endl; }; } else if( operation == "count" ) { handle = [&]( const std::string& s ) { std::cout << comma::csv::format( s ).count() << std::endl; }; } - else if( operation == "guess" ) { handle = [&]( const std::string& s ) { + else if( operation == "size" ) { handle = [&]( const std::string& s ) { std::cout << comma::csv::format( s ).size() << std::endl; }; } + else if( operation == "guess" ) { handle = [&]( const std::string& s ) { static const std::vector< std::string >& e = comma::split( incomplete_expanded( options.value< std::string >( "--format", "" ) ), ',' ); static char delimiter = options.value( "--delimiter,-d", ',' ); const std::vector< std::string >& v = comma::split( s, delimiter ); @@ -150,7 +143,7 @@ int main( int ac, char** av ) std::cout << std::endl; } }; } - else { std::cerr << app_name << ": expected operation; got: \"" << operation << "\"" << std::endl; return 1; } + else { std::cerr << "csv-format: expected operation; got: \"" << operation << "\"" << std::endl; return 1; } while( std::getline( std::cin, line ) ) { const std::string& stripped = comma::strip( line ); @@ -158,7 +151,7 @@ int main( int ac, char** av ) } return 0; } - catch( std::exception& ex ) { std::cerr << app_name << ": " << ex.what() << std::endl; } - catch( ... ) { std::cerr << app_name << ": unknown exception" << std::endl; } + catch( std::exception& ex ) { std::cerr << "csv-format: " << ex.what() << std::endl; } + catch( ... ) { std::cerr << "csv-format: unknown exception" << std::endl; } return 1; } diff --git a/csv/applications/csv-from-bin.cpp b/csv/applications/csv-from-bin.cpp index 644dc4524..f901114c7 100644 --- a/csv/applications/csv-from-bin.cpp +++ b/csv/applications/csv-from-bin.cpp @@ -39,7 +39,6 @@ #include #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../base/exception.h" #include "../../csv/format.h" #include "../../string/string.h" @@ -54,7 +53,6 @@ static void usage() std::cerr << "--precision: set precision (number of mantissa digits) for floating point types" << std::endl; std::cerr << csv::format::usage() << std::endl; std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; std::cerr << std::endl; exit( 0 ); } diff --git a/csv/applications/csv-interval.cpp b/csv/applications/csv-intervals.cpp similarity index 53% rename from csv/applications/csv-interval.cpp rename to csv/applications/csv-intervals.cpp index 5fa81d6a2..0042e47c9 100644 --- a/csv/applications/csv-interval.cpp +++ b/csv/applications/csv-intervals.cpp @@ -27,26 +27,29 @@ // OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author Vinny Do +#include #include #include #include #include +#include #include #include #include "../../application/command_line_options.h" #include "../../base/exception.h" #include "../../csv/stream.h" -#include "../../visiting/traits.h" +#include "../../csv/traits.h" #include "../../csv/impl/unstructured.h" - -static const std::string app_name = "csv-interval"; +#include "../../io/stream.h" +#include "../../name_value/map.h" +#include "../../name_value/parser.h" +#include "../../string/string.h" +#include "../../visiting/traits.h" static bool verbose; static bool debug; -static std::string first_line; static bool append; template < typename T > struct limits @@ -91,21 +94,23 @@ static void usage( bool verbose = false ) { std::cerr << "takes csv intervals and separates them at points of overlap if any" << std::endl; std::cerr << std::endl; - std::cerr << "usage: cat intervals.csv | " << app_name << " [OPTIONS...]" << std::endl; + std::cerr << "usage: cat intervals.csv | csv-intervals []" << std::endl; + std::cerr << std::endl; + std::cerr << "operations" << std::endl; + std::cerr << " contain: given a set of intervals, take scalars on stdin, append 1 if contained in the intervals, 0 if not" << std::endl; + std::cerr << " join: given a set of intervals, take scalars on stdin, append payloads of the intervals the scalars are contained in" << std::endl; + std::cerr << " make: make intervals" << std::endl; std::cerr << std::endl; std::cerr << "options" << std::endl; std::cerr << " --help,-h: show help; --help --verbose for more help" << std::endl; std::cerr << " --verbose,-v: more info" << std::endl; - std::cerr << " --append,-a: append output intervals instead of outputting them in place" << std::endl; - std::cerr << " --debug: print debug" << std::endl; + std::cerr << " --debug: more debug output" << std::endl; std::cerr << " --input-fields: print input fields and exit" << std::endl; // std::cerr << " --input-format: print input format and exit" << std::endl; std::cerr << " --output-fields: print output fields and exit" << std::endl; // std::cerr << " --output-format: print output format and exit" << std::endl; - std::cerr << " --empty: empty value used to signify unbounded intervals" << std::endl; + std::cerr << " --empty=[]: empty value used to signify unbounded intervals" << std::endl; std::cerr << " default for time is \"not-a-date-time\"" << std::endl; - std::cerr << " --format: input format (ascii only), also affects the --limits option; if not given the format is guessed" << std::endl; - std::cerr << " --intervals-only: only output the intervals, ignore payload if any" << std::endl; std::cerr << " --limits,-l: replace empty bounds with type limits" << std::endl; std::cerr << " b : " << (int)limits< char >::lowest() << " " << (int)limits< char >::max() << std::endl; std::cerr << " ub : " << (int)limits< unsigned char >::lowest() << " " << (int)limits< unsigned char >::max() << std::endl; @@ -121,14 +126,31 @@ static void usage( bool verbose = false ) std::cerr << " s : \"" << limits< std::string >::lowest() << "\" \"" << limits< std::string >::max() << "\"" << std::endl; std::cerr << " t : " << limits< boost::posix_time::ptime >::lowest() << " " << limits< boost::posix_time::ptime >::max() << std::endl; std::cerr << " lt : " << limits< boost::posix_time::ptime >::lowest() << " " << limits< boost::posix_time::ptime >::max() << std::endl; - std::cerr << " --overlap-count=[]; output only intervals with overlaps" << std::endl; - std::cerr << " --overlap-count-min,--min-overlap-count=[]; output only intervals with at least overlaps" << std::endl; - std::cerr << " --overlap-count-max,--max-overlap-count=[]; output only intervals with not more than overlaps" << std::endl; + std::cerr << std::endl; + std::cerr << "operation details" << std::endl; + std::cerr << std::endl; + std::cerr << " contain" << std::endl; + std::cerr << " options" << std::endl; + std::cerr << " --intervals=: file or stream name" << std::endl; + std::cerr << " join" << std::endl; + std::cerr << " options" << std::endl; + std::cerr << " --intervals=: file or stream name" << std::endl; + std::cerr << " --matching: output matching input records, do not append the intervals" << std::endl; + std::cerr << " --not-matching: output not matching input records" << std::endl; + std::cerr << std::endl; + std::cerr << " make" << std::endl; + std::cerr << " options" << std::endl; + std::cerr << " --append,-a: append output intervals instead of outputting them in place" << std::endl; + std::cerr << " --format=[]: input format (ascii only), also affects the --limits option; if not given the format is guessed" << std::endl; + std::cerr << " --intervals-only: only output the intervals, ignore payload if any" << std::endl; + std::cerr << " --overlap-count=[]; output only intervals with overlaps" << std::endl; + std::cerr << " --overlap-count-min,--min-overlap-count=[]; output only intervals with at least overlaps" << std::endl; + std::cerr << " --overlap-count-max,--max-overlap-count=[]; output only intervals with not more than overlaps" << std::endl; std::cerr << std::endl; std::cerr << "ascii notes" << std::endl; std::cerr << " unbounded intervals may be indicated by no value (e.g. ,3 \u2261 -\u221e,3), both sides unbounded is also supported" << std::endl; std::cerr << std::endl; - std::cerr << "for examples see verbose help: " << app_name << " --help --verbose" << std::endl; + std::cerr << "for examples see verbose help: csv-intervals --help --verbose" << std::endl; std::cerr << std::endl; if( verbose ) { @@ -143,7 +165,7 @@ static void usage( bool verbose = false ) std::cerr << " B: [2 4]" << std::endl; std::cerr << " C: [3 6]" << std::endl; std::cerr << std::endl; - std::cerr << " echo -e '1,5,A\\n2,4,B\\n3,6,C' | " << app_name << std::endl; + std::cerr << " echo -e '1,5,A\\n2,4,B\\n3,6,C' | csv-intervals make" << std::endl; std::cerr << std::endl; std::cerr << " A: [1 2][2 3][3 4][4 5]" << std::endl; std::cerr << " B: [2 3][3 4]" << std::endl; @@ -157,7 +179,7 @@ static void usage( bool verbose = false ) std::cerr << " D: [3 8]" << std::endl; std::cerr << " Z: [-\u221e +\u221e]" << std::endl; std::cerr << std::endl; - std::cerr << " echo -e ',4,A\\n2,4,B\\n3,6,C\\n3,8,D\\n,,Z' | " << app_name << " --format 2i" << std::endl; + std::cerr << " echo -e ',4,A\\n2,4,B\\n3,6,C\\n3,8,D\\n,,Z' | csv-intervals make" << " --format 2i" << std::endl; std::cerr << std::endl; std::cerr << " A: [-\u221e 2][2 3][3 4]" << std::endl; std::cerr << " B: [2 3][3 4]" << std::endl; @@ -172,7 +194,7 @@ static void usage( bool verbose = false ) std::cerr << " C: [20140916T190000 +\u221e]" << std::endl; std::cerr << " Z: [-\u221e +\u221e]" << std::endl; std::cerr << std::endl; - std::cerr << " echo -e ',20140916T030000.000000,A\\n20140916T010000.000000,20140916T190000.000000,B\\n20140916T190000.000000,,C\\n,,Z' | " << app_name << " --format 2t" << std::endl; + std::cerr << " echo -e ',20140916T030000.000000,A\\n20140916T010000.000000,20140916T190000.000000,B\\n20140916T190000.000000,,C\\n,,Z' | csv-intervals make" << " --format 2t" << std::endl; std::cerr << std::endl; std::cerr << " A: [-\u221e 20140916T010000][20140916T010000 20140916T030000]" << std::endl; std::cerr << " B: [20140916T010000 20140916T030000][20140916T030000 20140916T190000]" << std::endl; @@ -248,6 +270,18 @@ struct interval_t to_t< To > to; }; + +template < typename T > struct scalar_traits { static T zero() { return 0; } }; +template <> struct scalar_traits< boost::posix_time::ptime > { static boost::posix_time::ptime zero() { return boost::posix_time::ptime(); } }; +template <> struct scalar_traits< std::string > { static std::string zero() { return ""; } }; + +template < typename T > struct scalar_t +{ + T scalar{ scalar_traits< T >::zero() }; + scalar_t() {} + scalar_t( const T& t ): scalar( t ) {} +}; + namespace comma { namespace visiting { template < typename T > struct traits< from_t< T > > @@ -268,6 +302,12 @@ template < typename From, typename To > struct traits< interval_t< From, To > > template < typename K, typename V > static void visit( const K&, const interval_t< From, To >& p, V& v ) { v.apply( "", p.from ); v.apply( "", p.to ); } }; +template < typename T > struct traits< scalar_t< T > > +{ + template < typename K, typename V > static void visit( const K&, scalar_t< T >& p, V& v ) { v.apply( "scalar", p.scalar ); } + template < typename K, typename V > static void visit( const K&, const scalar_t< T >& p, V& v ) { v.apply( "scalar", p.scalar ); } +}; + } } // namespace comma { namespace visiting { template < typename From, typename To = From > @@ -287,14 +327,16 @@ struct intervals unsigned int min_overlap_count; unsigned int max_overlap_count; - intervals( const comma::command_line_options& options ) : options( options ) - , csv( options ) - , ocsv( options ) - , ascii_csv( options ) - , empty( traits< bound_type >::cast( options.optional< std::string >( "--empty" ) ) ) - , intervals_only( options.exists( "--intervals-only" ) ) - , use_limits( options.exists( "--limits,-l" ) ) + intervals( const comma::command_line_options& options, const comma::csv::options& csv_ ) + : options( options ) + , csv( csv_ ) + , ocsv( csv ) + , ascii_csv( csv ) + , empty( traits< bound_type >::cast( options.optional< std::string >( "--empty" ) ) ) + , intervals_only( options.exists( "--intervals-only" ) ) + , use_limits( options.exists( "--limits,-l" ) ) { + csv.full_xpath = false; if( csv.fields.empty() ) { csv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' ); } if( ocsv.fields.empty() || intervals_only ) { @@ -303,7 +345,7 @@ struct intervals } ascii_csv.fields = ocsv.fields; ascii_csv.quote = boost::none; - if( verbose ) { std::cerr << app_name << ": empty: "; empty ? std::cerr << *empty : std::cerr << ""; std::cerr << std::endl; } + if( verbose ) { std::cerr << "csv-intervals: empty: "; empty ? std::cerr << *empty : std::cerr << ""; std::cerr << std::endl; } options.assert_mutually_exclusive( "overlap-count-min,overlap-count-max", "overlap-count" ); if( options.exists( "--overlap-count" ) ) { @@ -394,9 +436,9 @@ struct intervals } } - void run() + void read( std::istream& is, const std::string& first_line ) // preparing for adding operations { - comma::csv::input_stream< interval_t< From, To > > istream( std::cin, csv ); + comma::csv::input_stream< interval_t< From, To > > istream( is, csv ); comma::csv::ascii< interval_t< std::string > > ascii( csv.fields ); if( !first_line.empty() ) { @@ -410,10 +452,10 @@ struct intervals if( !first.to.value.empty() && ( !empty || interval.to.value != *empty ) ) { to.value = interval.to.value; } payload = first_line; if( !intervals_only && !append ) { ascii.put( interval_t< std::string >(), payload ); } // blank out interval from payload - if( verbose ) { std::cerr << app_name << ": from: " << from << " to: " << to << " payload: " << payload << std::endl; } + if( verbose ) { std::cerr << "csv-intervals: from: " << from << " to: " << to << " payload: " << payload << std::endl; } add( from, to, payload ); } - while( istream.ready() || std::cin.good() ) + while( istream.ready() || is.good() ) { const interval_t< From, To >* interval = istream.read(); if( !interval ) { break; } @@ -438,108 +480,239 @@ struct intervals if( !intervals_only && !append ) { ascii.put( interval_t< std::string >(), buf ); } // blank out interval from payload payload = comma::join( buf, csv.delimiter ); } - if( verbose ) { std::cerr << app_name << ": from: " << from << " to: " << to << " payload: " << ( csv.binary() ? "" : payload ) << std::endl; } + if( verbose ) { std::cerr << "csv-intervals: from: " << from << " to: " << to << " payload: " << ( csv.binary() ? "" : payload ) << std::endl; } add( from, to, payload ); } - write(); + } + + int contain( std::istream& is, const std::string& first_line ) + { + comma::csv::options icsv( options ); + icsv.full_xpath = false; + comma::csv::input_stream< scalar_t< From > > istream( std::cin, icsv ); + comma::csv::output_stream< scalar_t< bool > > ostream( std::cout, icsv.binary() ); + auto tied = comma::csv::make_tied( istream, ostream ); + this->read( is, first_line ); // todo: support block + while( istream.ready() || std::cin.good() ) + { + auto p = istream.read(); + if( !p ) { break; } + bool contained = false; + for( typename map_t::iterator it = map.begin(); it != map.end() && !contained; ++it ) // todo! quadratic complexity; how the heck to query icl map? use boost::...::query? + { + const bound_t< bound_type >& from = it->first.lower(); + const bound_t< bound_type >& to = it->first.upper(); + contained = ( !from.value || p->scalar >= *from.value ) && ( !to.value || p->scalar < *to.value ); + } + tied.append( scalar_t< bool >( contained ) ); + if( icsv.flush ) { std::cout.flush(); } + } + return 0; + } + + int join( std::istream& is, const std::string& first_line ) + { + options.assert_mutually_exclusive( "--matching,--not-matching" ); + bool matching = options.exists( "--matching" ); + bool not_matching = options.exists( "--not-matching" ); + bool output_joined = !matching && !not_matching; + comma::csv::options icsv( options ); + if( output_joined && csv.binary() != icsv.binary() ) { std::cerr << "csv-intervals: join: expected both inputs ascii or both binary; got stdin " << ( icsv.binary() ? "binary" : "ascii" ) << " while --intervals " << ( csv.binary() ? "binary" : "ascii" ) << std::endl; return 1; } + icsv.full_xpath = false; + comma::csv::input_stream< scalar_t< From > > istream( std::cin, icsv ); + append = true; + this->read( is, first_line ); // todo: support block + while( istream.ready() || std::cin.good() ) + { + auto p = istream.read(); + if( !p ) { break; } + bool found = false; + typename map_t::iterator it; + for( it = map.begin(); it != map.end(); ++it ) // todo! quadratic complexity; how the heck to query icl map? use boost::...::query? + { + const bound_t< bound_type >& from = it->first.lower(); + const bound_t< bound_type >& to = it->first.upper(); + found = ( !from.value || p->scalar >= *from.value ) && ( !to.value || p->scalar < *to.value ); + if( found ) { break; } + } + if( output_joined ) + { + if( found ) + { + std::string joined = csv.binary() ? "" : comma::join( istream.ascii().last(), icsv.delimiter ); + for( const auto& s: it->second ) + { + if( csv.binary() ) + { + std::cout.write( istream.binary().last(), icsv.format().size() ); + std::cout.write( &s[0], s.size() ); + } + else + { + std::cout << joined << icsv.delimiter << s << std::endl; + } + } + } + } + else if( matching == found ) + { + if( icsv.binary() ) { std::cout.write( istream.binary().last(), icsv.format().size() ); } + else { std::cout << comma::join( istream.ascii().last(), icsv.delimiter ) << std::endl; } + } + if( icsv.flush ) { std::cout.flush(); } + } + return 0; + } + + int make( const std::string& first_line ) + { + this->read( std::cin, first_line ); + this->write(); + return 0; } }; -// template < typename From > static void run( const comma::command_line_options& options, const comma::csv::format::types_enum to_type ) -// { -// switch( to_type ) -// { -// case comma::csv::format::int8: intervals< From, char >( options ).run(); break; -// case comma::csv::format::uint8: intervals< From, unsigned char >( options ).run(); break; -// case comma::csv::format::int16: intervals< From, comma::int16 >( options ).run(); break; -// case comma::csv::format::uint16: intervals< From, comma::uint16 >( options ).run(); break; -// case comma::csv::format::int32: intervals< From, comma::int32 >( options ).run(); break; -// case comma::csv::format::uint32: intervals< From, comma::uint32 >( options ).run(); break; -// case comma::csv::format::int64: intervals< From, comma::int64 >( options ).run(); break; -// case comma::csv::format::uint64: intervals< From, comma::uint64 >( options ).run(); break; -// case comma::csv::format::char_t: intervals< From, char >( options ).run(); break; -// case comma::csv::format::float_t: intervals< From, float >( options ).run(); break; -// case comma::csv::format::double_t: intervals< From, double >( options ).run(); break; -// default: COMMA_THROW( comma::exception, "from/to type mismatch" ); break; -// } -// } +static std::tuple< comma::csv::format::types_enum, std::string > interval_type( std::istream& is, comma::csv::options csv, const std::string& format ) +{ + if( csv.fields.empty() ) { csv.fields = comma::join( comma::csv::names< interval_t< double > >(), ',' ); } + if( !csv.has_field( "from,to" ) ) { COMMA_THROW( comma::exception, "expected from and to fields" ); } + std::string first_line; + csv.full_xpath = false; + if( !csv.binary() ) + { + if( format.empty() ) + { + while( is.good() && first_line.empty() ) { std::getline( is, first_line ); } + if( first_line.empty() ) { exit( 0 ); } // quick and dirty + csv.format( comma::csv::impl::unstructured::guess_format( first_line, csv.delimiter ) ); + if( verbose ) { std::cerr << "csv-intervals: guessed format: " << csv.format().string() << std::endl;; } + } + else + { + csv.format( format ); + } + } + const std::vector< std::string >& fields = comma::split( csv.fields, ',' ); + unsigned int from_index = 0; + unsigned int to_index = 1; + for( unsigned int i = 0; i < fields.size(); ++i ) { if( fields[i] == "from" ) { from_index = i; break; } } + for( unsigned int i = 0; i < fields.size(); ++i ) { if( fields[i] == "to" ) { to_index = i; break; } } + const comma::csv::format::types_enum from_type = csv.format().offset( from_index ).type; + const comma::csv::format::types_enum to_type = csv.format().offset( to_index ).type; + if( ( ( from_type == comma::csv::format::time || from_type == comma::csv::format::long_time ) && ( to_type != comma::csv::format::time && to_type != comma::csv::format::long_time ) ) || + ( ( ( from_type != comma::csv::format::time && from_type != comma::csv::format::long_time ) && ( to_type == comma::csv::format::time || to_type == comma::csv::format::long_time ) ) ) ) + { COMMA_THROW( comma::exception, "from/to type mismatch; time" ); } + if( ( from_type == comma::csv::format::fixed_string || to_type == comma::csv::format::fixed_string ) && from_type != to_type ) + { COMMA_THROW( comma::exception, "from/to type mismatch; string" ); } + if( from_type != to_type ) { std::cerr << "csv-intervals: support only from and to of the same type, got from: " << comma::csv::format::to_format( from_type ) << ", to: " << comma::csv::format::to_format( to_type ) << std::endl; exit( 1 ); } + return std::tie( to_type, first_line ); +} int main( int ac, char** av ) { try { - comma::command_line_options options( ac, av ); + comma::command_line_options options( ac, av, usage ); verbose = options.exists( "--verbose,-v" ); debug = options.exists( "--debug" ); - append = options.exists( "--append,-a" ); - if( options.exists( "--help,-h" ) ) { usage( verbose ); } - if( options.exists( "--input-fields" ) ) { std::cout << comma::join( comma::csv::names< interval_t< double > >(), ',' ) << std::endl; return 0; } - if( options.exists( "--output-fields" ) ) { std::cout << comma::join( comma::csv::names< interval_t< double > >(), ',' ) << std::endl; return 0; } - comma::csv::options csv( options ); - if( csv.fields.empty() ) { csv.fields = comma::join( comma::csv::names< interval_t< double > >(), ',' ); } - if( !csv.has_field( "from,to" ) ) { COMMA_THROW( comma::exception, "expected from and to fields" ); } options.assert_mutually_exclusive( "--binary,--format" ); - if( options.exists( "--binary,-b" ) ) {} - else if( options.exists( "--format" ) ) { csv.format( options.value< std::string >( "--format" ) ); } - else + const auto& unnamed = options.unnamed( "--append,-a,--debug,--flush,--input-fields,--matching,--not-matching,--output-fields,--intervals-only,--limits,-l", "-.*" ); + if( unnamed.empty() ) { std::cerr << "csv-intervals: please specify operation" << std::endl; return 1; } + std::string operation = unnamed[0]; + if( operation == "make" ) { - while( std::cin.good() && first_line.empty() ) { std::getline( std::cin, first_line ); } - if( first_line.empty() ) { return 0; } - csv.format( comma::csv::impl::unstructured::guess_format( first_line, csv.delimiter ) ); - if( verbose ) { std::cerr << app_name << ": guessed format: " << csv.format().string() << std::endl;; } + append = options.exists( "--append,-a" ); + if( options.exists( "--input-fields" ) ) { std::cout << comma::join( comma::csv::names< interval_t< double > >(), ',' ) << std::endl; return 0; } + if( options.exists( "--output-fields" ) ) { std::cout << comma::join( comma::csv::names< interval_t< double > >(), ',' ) << std::endl; return 0; } + comma::csv::options csv( options ); + auto t = interval_type( std::cin, comma::csv::options( options ), options.value< std::string >( "--format,-f", "" ) ); + const comma::csv::format::types_enum to_type = std::get< 0 >( t ); + std::string first_line = std::get< 1 >( t ); + switch( to_type ) + { + case comma::csv::format::int8: intervals< char >( options, csv ).make( first_line ); return 0; + case comma::csv::format::uint8: intervals< unsigned char >( options, csv ).make( first_line ); return 0; + case comma::csv::format::int16: intervals< comma::int16 >( options, csv ).make( first_line ); return 0; + case comma::csv::format::uint16: intervals< comma::uint16 >( options, csv ).make( first_line ); return 0; + case comma::csv::format::int32: intervals< comma::int32 >( options, csv ).make( first_line ); return 0; + case comma::csv::format::uint32: intervals< comma::uint32 >( options, csv ).make( first_line ); return 0; + case comma::csv::format::int64: intervals< comma::int64 >( options, csv ).make( first_line ); return 0; + case comma::csv::format::uint64: intervals< comma::uint64 >( options, csv ).make( first_line ); return 0; + case comma::csv::format::char_t: intervals< char >( options, csv ).make( first_line ); return 0; + case comma::csv::format::float_t: intervals< float >( options, csv ).make( first_line ); return 0; + case comma::csv::format::double_t: intervals< double >( options, csv ).make( first_line ); return 0; + case comma::csv::format::time: + case comma::csv::format::long_time: intervals< boost::posix_time::ptime >( options, csv ).make( first_line ); return 0; + case comma::csv::format::fixed_string: intervals< std::string >( options, csv ).make( first_line ); return 0; + default: COMMA_THROW( comma::exception, "invalid type" ); return 0; // never here + } + return 0; } - const std::vector< std::string >& fields = comma::split( csv.fields, ',' ); - unsigned int from_index = 0; - unsigned int to_index = 1; - for( unsigned int i = 0; i < fields.size(); ++i ) { if( fields[i] == "from" ) { from_index = i; break; } } - for( unsigned int i = 0; i < fields.size(); ++i ) { if( fields[i] == "to" ) { to_index = i; break; } } - const comma::csv::format::types_enum from_type = csv.format().offset( from_index ).type; - const comma::csv::format::types_enum to_type = csv.format().offset( to_index ).type; - if( ( ( from_type == comma::csv::format::time || from_type == comma::csv::format::long_time ) && ( to_type != comma::csv::format::time && to_type != comma::csv::format::long_time ) ) || - ( ( ( from_type != comma::csv::format::time && from_type != comma::csv::format::long_time ) && ( to_type == comma::csv::format::time || to_type == comma::csv::format::long_time ) ) ) ) - { COMMA_THROW( comma::exception, "from/to type mismatch; time" ); } - if( ( from_type == comma::csv::format::fixed_string || to_type == comma::csv::format::fixed_string ) && from_type != to_type ) - { COMMA_THROW( comma::exception, "from/to type mismatch; string" ); } -// switch( from_type ) -// { -// case comma::csv::format::int8: run< char >( options, to_type ); break; -// case comma::csv::format::uint8: run< unsigned char >( options, to_type ); break; -// case comma::csv::format::int16: run< comma::int16 >( options, to_type ); break; -// case comma::csv::format::uint16: run< comma::uint16 >( options, to_type ); break; -// case comma::csv::format::int32: run< comma::int32 >( options, to_type ); break; -// case comma::csv::format::uint32: run< comma::uint32 >( options, to_type ); break; -// case comma::csv::format::int64: run< comma::int64 >( options, to_type ); break; -// case comma::csv::format::uint64: run< comma::uint64 >( options, to_type ); break; -// case comma::csv::format::char_t: run< char >( options, to_type ); break; -// case comma::csv::format::float_t: run< float >( options, to_type ); break; -// case comma::csv::format::double_t: run< double >( options, to_type ); break; -// case comma::csv::format::time: -// case comma::csv::format::long_time: intervals< boost::posix_time::ptime >( options ).run(); break; -// case comma::csv::format::fixed_string: intervals< std::string >( options ).run(); break; -// default: COMMA_THROW( comma::exception, "unknown type" ); break; -// } - if( from_type != to_type ) { std::cerr << app_name << ": support only from and to of the same type, got from: " << comma::csv::format::to_format( from_type ) << ", to: " << comma::csv::format::to_format( to_type ) << std::endl; return 1; } - switch( to_type ) + if( operation == "contain" ) { - case comma::csv::format::int8: intervals< char >( options ).run(); break; - case comma::csv::format::uint8: intervals< unsigned char >( options ).run(); break; - case comma::csv::format::int16: intervals< comma::int16 >( options ).run(); break; - case comma::csv::format::uint16: intervals< comma::uint16 >( options ).run(); break; - case comma::csv::format::int32: intervals< comma::int32 >( options ).run(); break; - case comma::csv::format::uint32: intervals< comma::uint32 >( options ).run(); break; - case comma::csv::format::int64: intervals< comma::int64 >( options ).run(); break; - case comma::csv::format::uint64: intervals< comma::uint64 >( options ).run(); break; - case comma::csv::format::char_t: intervals< char >( options ).run(); break; - case comma::csv::format::float_t: intervals< float >( options ).run(); break; - case comma::csv::format::double_t: intervals< double >( options ).run(); break; - case comma::csv::format::time: - case comma::csv::format::long_time: intervals< boost::posix_time::ptime >( options ).run(); break; - case comma::csv::format::fixed_string: intervals< std::string >( options ).run(); break; - default: COMMA_THROW( comma::exception, "from/to type mismatch" ); break; + if( options.exists( "--input-fields" ) ) { std::cout << comma::join( comma::csv::names< scalar_t< double > >(), ',' ) << std::endl; return 0; } + if( options.exists( "--output-fields" ) ) { std::cout << comma::join( comma::csv::names< scalar_t< double > >(), ',' ) << std::endl; return 0; } + auto i = options.value< std::string >( "--intervals" ); + comma::csv::options csv = comma::name_value::parser( "filename" ).get< comma::csv::options >( i ); + std::string format = comma::name_value::map( i ).value< std::string >( "format", "" ); + comma::io::istream is( csv.filename ); + auto t = interval_type( *is, csv, format ); + const comma::csv::format::types_enum to_type = std::get< 0 >( t ); + std::string first_line = std::get< 1 >( t ); + switch( to_type ) + { + case comma::csv::format::int8: intervals< char >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::uint8: intervals< unsigned char >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::int16: intervals< comma::int16 >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::uint16: intervals< comma::uint16 >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::int32: intervals< comma::int32 >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::uint32: intervals< comma::uint32 >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::int64: intervals< comma::int64 >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::uint64: intervals< comma::uint64 >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::char_t: intervals< char >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::float_t: intervals< float >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::double_t: intervals< double >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::time: + case comma::csv::format::long_time: intervals< boost::posix_time::ptime >( options, csv ).contain( *is, first_line ); return 0; + case comma::csv::format::fixed_string: intervals< std::string >( options, csv ).contain( *is, first_line ); return 0; + default: COMMA_THROW( comma::exception, "invalid type" ); // never here + } + return 0; } - return 0; + if( operation == "join" ) + { + if( options.exists( "--input-fields" ) ) { std::cout << comma::join( comma::csv::names< scalar_t< double > >(), ',' ) << std::endl; return 0; } + if( options.exists( "--output-fields" ) ) { std::cerr << "csv-intervals join: does not have --output-fields" << std::endl; return 1; } + auto i = options.value< std::string >( "--intervals" ); + comma::csv::options csv = comma::name_value::parser( "filename" ).get< comma::csv::options >( i ); + std::string format = comma::name_value::map( i ).value< std::string >( "format", "" ); + comma::io::istream is( csv.filename ); + auto t = interval_type( *is, csv, format ); + const comma::csv::format::types_enum to_type = std::get< 0 >( t ); + std::string first_line = std::get< 1 >( t ); + switch( to_type ) + { + case comma::csv::format::int8: intervals< char >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::uint8: intervals< unsigned char >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::int16: intervals< comma::int16 >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::uint16: intervals< comma::uint16 >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::int32: intervals< comma::int32 >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::uint32: intervals< comma::uint32 >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::int64: intervals< comma::int64 >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::uint64: intervals< comma::uint64 >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::char_t: intervals< char >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::float_t: intervals< float >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::double_t: intervals< double >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::time: + case comma::csv::format::long_time: intervals< boost::posix_time::ptime >( options, csv ).join( *is, first_line ); return 0; + case comma::csv::format::fixed_string: intervals< std::string >( options, csv ).join( *is, first_line ); return 0; + default: COMMA_THROW( comma::exception, "invalid type" ); // never here + } + return 0; + } + std::cerr << "csv-intervals: expected operation, got: '" << operation << "'" << std::endl; } - catch( std::exception& ex ) { std::cerr << app_name << ": " << ex.what() << std::endl; } - catch( ... ) { std::cerr << app_name << ": unknown exception" << std::endl; } + catch( std::exception& ex ) { std::cerr << "csv-intervals: " << ex.what() << std::endl; } + catch( ... ) { std::cerr << "csv-intervals: unknown exception" << std::endl; } return 1; } diff --git a/csv/applications/csv-join.cpp b/csv/applications/csv-join.cpp index 161a7df33..39fab1839 100644 --- a/csv/applications/csv-join.cpp +++ b/csv/applications/csv-join.cpp @@ -1,49 +1,21 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney // All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// @author vsevolod vlaskine #include +#include #include #include #include #include +#include +#include #include -#include #include #include -#include #include -#include -#include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../application/signal_flag.h" #include "../../base/exception.h" #include "../../base/types.h" @@ -68,14 +40,17 @@ static void usage( bool more ) std::cerr << std::endl; std::cerr << "options" << std::endl; std::cerr << " --help,-h: help; --help --verbose: more help" << std::endl; + std::cerr << " --block-less; todo! better option name! input and filter block ids expected sorted" << std::endl; + std::cerr << " todo! document" << std::endl; + std::cerr << " --drop-id-fields,--drop-id; remove id and block fields from filter output (same as if you did csv-join|csv-shuffle)" << std::endl; std::cerr << " --first-matching: output only the first matching record (a bit of hack for now, but we needed it)" << std::endl; std::cerr << " --flag-matching: output all records, with 1 appended to matching records and 0 appended to not-matching records" << std::endl; std::cerr << " --matching: output only matching records from stdin" << std::endl; std::cerr << " --nearest: if --radius specified, output only nearest record" << std::endl; std::cerr << " --not-matching: not matching records as read from stdin, no join performed" << std::endl; - std::cerr << " --strict: fail, if id on stdin is not found, or there are multiple filter keys on --unique, etc" << std::endl; + std::cerr << " --output-swap,--swap-output,--swap; output filter records first with the stdin record appended, a convenience option" << std::endl; std::cerr << " --radius,--epsilon=; compare keys in given radius; the keys will be interpreted as floating point numbers" << std::endl; - std::cerr << " --swap-output,--swap; output filter records first with the stdin record appended, a convenience option" << std::endl; + std::cerr << " --strict: fail, if id on stdin is not found, or there are multiple filter keys on --unique, etc" << std::endl; std::cerr << " --unique,--unique-matches: expect only unique matches, exit with error otherwise" << std::endl; std::cerr << " --verbose,-v: more output to stderr" << std::endl; std::cerr << std::endl; @@ -105,40 +80,59 @@ static void usage( bool more ) std::cerr << " any other field name: key" << std::endl; std::cerr << std::endl; std::cerr << " block acts as a key but stream processing occurs at the end of each" << std::endl; - std::cerr << " block. If no block field is given the entire input is considered to be" << std::endl; - std::cerr << " one block. Blocks are required to be contiguous in the input stream." << std::endl; + std::cerr << " block; if no block field is given the entire input is considered to be" << std::endl; + std::cerr << " one block; blocks are required to be contiguous in the input stream" << std::endl; + } + else + { + std::cerr << " run csv-join --help --verbose for more..." << std::endl; } std::cerr << std::endl; - std::cerr << "Examples (try them):" << std::endl; - std::cerr << " on the following data file:" << std::endl; - std::cerr << " echo 1,1,2,hello > data.csv" << std::endl; - std::cerr << " echo 1,2,3,hello >> data.csv" << std::endl; - std::cerr << " echo 3,3,4,world >> data.csv" << std::endl; - std::cerr << " echo 3,4,3,world >> data.csv" << std::endl; - std::cerr << std::endl; - std::cerr << " join with a matching record" << std::endl; - std::cerr << " echo 1,blah | csv-join --fields=id \"data.csv;fields=id\"" << std::endl; - std::cerr << " echo 3,blah | csv-join --fields=id \"data.csv;fields=,,id\"" << std::endl; - std::cerr << " echo 5,blah | csv-join --fields=id \"data.csv;fields=,,id\"" << std::endl; - std::cerr << " echo 5,blah | csv-join --fields=id \"data.csv;fields=,,id\" --not-matching" << std::endl; - std::cerr << " echo 5,blah | csv-join --fields=id \"data.csv;fields=,,id\" --strict" << std::endl; - std::cerr << std::endl; - std::cerr << " join by key which is a string" << std::endl; - std::cerr << " echo 1,hello | csv-join --fields=,id \"data.csv;fields=,,,id\" --string" << std::endl; - std::cerr << " echo 1,world | csv-join --fields=,id \"data.csv;fields=,,,id\" --string" << std::endl; - std::cerr << " echo 1,blah | csv-join --fields=,id \"data.csv;fields=,,,id\" --string" << std::endl; - std::cerr << " echo 1,blah | csv-join --fields=,id \"data.csv;fields=,,,id\" --string --not-matching" << std::endl; - std::cerr << " echo 1,blah | csv-join --fields=,id \"data.csv;fields=,,,id\" --string --strict" << std::endl; - std::cerr << std::endl; - std::cerr << " finite state machine" << std::endl; - std::cerr << " csv-join --fields=event \"data.csv;fields=event,state,next_state\" --initial-state 1" << std::endl; - std::cerr << " " << std::endl; - std::cerr << " " << std::endl; - std::cerr << " " << std::endl; - std::cerr << " " << std::endl; - std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; - std::cerr << std::endl; + if( more ) + { + std::cerr << "examples (try them)" << std::endl; + std::cerr << " on the following data file:" << std::endl; + std::cerr << " echo 1,1,2,hello > data.csv" << std::endl; + std::cerr << " echo 1,2,3,hello >> data.csv" << std::endl; + std::cerr << " echo 3,3,4,world >> data.csv" << std::endl; + std::cerr << " echo 3,4,3,world >> data.csv" << std::endl; + std::cerr << std::endl; + std::cerr << " join with a matching record" << std::endl; + std::cerr << " echo 1,blah | csv-join --fields=id \"data.csv;fields=id\"" << std::endl; + std::cerr << " echo 3,blah | csv-join --fields=id \"data.csv;fields=,,id\"" << std::endl; + std::cerr << " echo 5,blah | csv-join --fields=id \"data.csv;fields=,,id\"" << std::endl; + std::cerr << " echo 5,blah | csv-join --fields=id \"data.csv;fields=,,id\" --not-matching" << std::endl; + std::cerr << " echo 5,blah | csv-join --fields=id \"data.csv;fields=,,id\" --strict" << std::endl; + std::cerr << std::endl; + std::cerr << " join by key which is a string" << std::endl; + std::cerr << " echo 1,hello | csv-join --fields=,id \"data.csv;fields=,,,id\" --string" << std::endl; + std::cerr << " echo 1,world | csv-join --fields=,id \"data.csv;fields=,,,id\" --string" << std::endl; + std::cerr << " echo 1,blah | csv-join --fields=,id \"data.csv;fields=,,,id\" --string" << std::endl; + std::cerr << " echo 1,blah | csv-join --fields=,id \"data.csv;fields=,,,id\" --string --not-matching" << std::endl; + std::cerr << " echo 1,blah | csv-join --fields=,id \"data.csv;fields=,,,id\" --string --strict" << std::endl; + std::cerr << std::endl; + std::cerr << " block id ordered, gaps in filter blocks allowed" << std::endl; + std::cerr << " csv-paste line-number value=0 | head \\" << std::endl; + std::cerr << " | csv-join --fields block,id <( echo 3,0; echo 6,0 )';fields=block,id' --block-less" << std::endl; + std::cerr << " finite state machine" << std::endl; + std::cerr << " csv-join --fields=event \"data.csv;fields=event,state,next_state\" --initial-state 1" << std::endl; + std::cerr << " " << std::endl; + std::cerr << " " << std::endl; + std::cerr << " " << std::endl; + std::cerr << " " << std::endl; + std::cerr << std::endl; + std::cerr << " --drop-id (same would work in binary as well)" << std::endl; + std::cerr << " > echo 0,1,2,3 | csv-join --fields ,x,,y <( echo 1,A,B,3 )';fields=x,,,y'" << std::endl; + std::cerr << " 0,1,2,3,1,A,B,3" << std::endl; + std::cerr << " > echo 0,1,2,3 | csv-join --fields ,x,,y <( echo 1,A,B,3 )';fields=x,,,y' --drop-id" << std::endl; + std::cerr << " 0,1,2,3,A,B" << std::endl; + std::cerr << std::endl; + } + else + { + std::cerr << "examples" << std::endl; + std::cerr << " run csv-join --help --verbose for more..." << std::endl; + } exit( 0 ); } @@ -153,14 +147,18 @@ static bool flag_matching; static bool swap_output; static comma::csv::options stdin_csv; static comma::csv::options filter_csv; +static bool filter_id_fields_discard; // todo: super-quick and dirty, put in a separate class +static std::vector< unsigned int > filter_id_fields_flags; // todo: super-quick and dirty, put in a separate class +static std::vector< std::pair< unsigned int, unsigned int > > filter_id_fields_offsets; // todo: super-quick and dirty, put in a separate class +static unsigned int filter_id_fields_size{0}; // todo: super-quick and dirty, put in a separate class boost::scoped_ptr< comma::io::istream > filter_transport; static comma::uint32 block = 0; static boost::optional< double > radius; static void hash_combine_( std::size_t& seed, boost::posix_time::ptime key ) { - BOOST_STATIC_ASSERT( sizeof( boost::posix_time::ptime ) == 8 ); - boost::hash_combine( seed, *reinterpret_cast< const long long* >( &key ) ); + static_assert( sizeof( boost::posix_time::ptime ) == 8, "expected time of size 8" ); + boost::hash_combine( seed, *reinterpret_cast< const comma::uint64* >( &key ) ); } template < typename K > static void hash_combine_( std::size_t& seed, K key ) { boost::hash_combine( seed, key ); } @@ -189,7 +187,7 @@ struct input return comma::math::less( keys[0], rhs.keys[0] ); //, *radius ); } - struct hash : public std::unary_function< input, std::size_t > + struct hash : public std::function< input( std::size_t ) > { std::size_t operator()( input const& p ) const { @@ -199,7 +197,7 @@ struct input } }; - typedef boost::unordered_map< input, std::vector< std::string >, hash > unordered_map; + typedef std::unordered_map< input, std::vector< std::string >, hash > unordered_map; typedef std::map< input, std::vector< std::string > > map; }; @@ -295,6 +293,7 @@ template < typename T > static std::string keys_as_string( const input< T >& i ) { std::ostringstream oss; comma::csv::options csv; + csv.full_xpath = false; csv.fields = "keys"; comma::csv::ascii_output_stream< input< T > > os( oss, csv, i ); os.write( i ); @@ -306,49 +305,79 @@ template < typename K, bool Strict = true > struct join_impl_ // quick and dirty static typename traits< K, Strict >::map filter_map; static input< K > default_input; - static void read_filter_block() + static std::string make_output( const std::vector< std::string >& values ) // todo? implement something like comma::join( values, drop )? + { + if( filter_id_fields_flags.empty() ) { return comma::join( values, stdin_csv.delimiter ); } + std::string s; + std::string delimiter; + unsigned int i = 0; + for( ; i < std::min( filter_id_fields_flags.size(), values.size() ); ++i ) + { + if( filter_id_fields_flags[i] == 0 ) { s += delimiter + values[i]; delimiter = std::string( 1, stdin_csv.delimiter ); } + } + for( ; i < values.size(); ++i ) { s += delimiter + values[i]; delimiter = std::string( 1, stdin_csv.delimiter ); } + return s; + } + + static std::string make_output( const char* values ) // todo? quick and dirty for now; use csv::binary? or implement something like csv::format::drop_fields...? + { + std::string s( filter_csv.format().size() - filter_id_fields_size, '\0' ); + if( filter_id_fields_flags.empty() ) + { + ::memcpy( &s[0], values, filter_csv.format().size() ); + } + else + { + char* p = &s[0]; + unsigned int t = 0; + for( const auto& e: filter_id_fields_offsets ) // todo: quick and dirty, watch performance + { + unsigned int c = e.first - t; + std::memcpy( p, values + t, c ); + p += c; + t = e.first + e.second; + } + std::memcpy( p, values + t, filter_csv.format().size() - t ); // todo: quick and dirty, watch performance + } + return s; + } + + static const input< K >* read_filter_block() { static comma::csv::input_stream< input< K > > filter_stream( **filter_transport, filter_csv, default_input ); static const input< K >* last = filter_stream.read(); filter_map.clear(); - if( !last ) { return; } + if( !last ) { return last; } block = last->block; comma::uint64 count = 0; static comma::signal_flag is_shutdown( comma::signal_flag::hard ); while( last->block == block && !is_shutdown ) { - typename traits< K, Strict >::map::mapped_type& d = filter_map[ *last ]; - if( filter_stream.is_binary() ) - { - typename traits< K, Strict >::map::mapped_type& d = filter_map[ *last ]; - d.push_back( std::string() ); - d.back().resize( filter_csv.format().size() ); - ::memcpy( &d.back()[0], filter_stream.binary().last(), filter_csv.format().size() ); - } - else - { - d.push_back( comma::join( filter_stream.ascii().last(), stdin_csv.delimiter ) ); - } + filter_map[ *last ].push_back( filter_stream.is_binary() ? make_output( filter_stream.binary().last() ) : make_output( filter_stream.ascii().last() ) ); if( verbose ) { ++count; if( count % 10000 == 0 ) { std::cerr << "csv-join: reading block " << block << "; loaded " << count << " point" << ( count == 1 ? "" : "s" ) << "; hash map size: " << filter_map.size() << std::endl; } } //if( ( *filter_transport )->good() && !( *filter_transport )->eof() ) { break; } last = filter_stream.read(); if( !last ) { break; } } if( verbose ) { std::cerr << "csv-join: read block " << block << " of " << count << " point" << ( count == 1 ? "" : "s" ) << "; hash map size: " << filter_map.size() << std::endl; } + return last; } static int run( const comma::command_line_options& options ) { + bool block_less = options.exists( "--block-less" ); std::vector< std::string > v = comma::split( stdin_csv.fields, ',' ); std::vector< std::string > w = comma::split( filter_csv.fields, ',' ); + if( filter_id_fields_discard ) { filter_id_fields_flags.resize( w.size(), 0 ); } bool got_state = false; bool got_next_state = false; - std::size_t filter_state_index; + std::size_t filter_state_index{0}; for( std::size_t k = 0; k < w.size() && ( !got_state || !got_next_state ); ++k ) { if( w[k] == "state" ) { got_state = true; filter_state_index = k; continue; } if( w[k] == "next_state" ) { got_next_state = true; continue; } } + if( ( got_state || got_next_state ) && filter_id_fields_discard ) { std::cerr << "csv-join: --drop-id and 'state' or 'next_field' are mutually exclusive" << std::endl; return 1; } bool is_state_machine = got_state && got_next_state; std::size_t default_input_keys_count = 0; bool no_stdin_key_fields = true; @@ -361,12 +390,26 @@ template < typename K, bool Strict = true > struct join_impl_ // quick and dirty { if( is_state_machine && ( w[k] == "state" || w[k] == "next_state" ) ) { no_filter_key_fields = false; continue; } if( !w[k].empty() && w[k] != "block" ) { no_filter_key_fields = false; } + if( filter_id_fields_discard && w[k] == "block" ) { filter_id_fields_flags[k] = 1; } if( v[i] != w[k] ) { continue; } - v[i] = "keys[" + boost::lexical_cast< std::string >( default_input_keys_count ) + "]"; - w[k] = "keys[" + boost::lexical_cast< std::string >( default_input_keys_count ) + "]"; + v[i] = "keys[" + std::to_string( default_input_keys_count ) + "]"; + w[k] = "keys[" + std::to_string( default_input_keys_count ) + "]"; + if( filter_id_fields_discard ) { filter_id_fields_flags[k] = 1; } ++default_input_keys_count; } } + if( filter_csv.binary() ) // todo: super-quick and dirty; move all the --drop-id stuff to a class + { + for( unsigned int i = 0; i < filter_id_fields_flags.size(); ++i ) + { + if( filter_id_fields_flags[i] ) + { + const auto& e = filter_csv.format().offset( i ); + filter_id_fields_offsets.push_back( { e.offset, e.size } ); + filter_id_fields_size += e.size; + } + } + } bool do_full_join = no_stdin_key_fields && no_filter_key_fields; if( default_input_keys_count == 0 && !do_full_join ) { std::cerr << "csv-join: please specify at least one common key; fields: " << stdin_csv.fields << "; filter fields: " << filter_csv.fields << std::endl; return 1; } //if( default_input_keys_count == 0 ) { std::cerr << "csv-join: please specify at least one common key; fields: " << stdin_csv.fields << "; filter fields: " << filter_csv.fields << std::endl; return 1; } @@ -375,7 +418,7 @@ template < typename K, bool Strict = true > struct join_impl_ // quick and dirty if( is_state_machine ) { state_index = default_input_keys_count; - w[filter_state_index] = "keys[" + boost::lexical_cast< std::string >( state_index ) + "]"; + w[filter_state_index] = "keys[" + std::to_string( state_index ) + "]"; ++default_input_keys_count; } default_input.keys.resize( default_input_keys_count ); @@ -390,7 +433,7 @@ template < typename K, bool Strict = true > struct join_impl_ // quick and dirty filter_transport.reset( new comma::io::istream( filter_csv.filename, filter_csv.binary() ? comma::io::mode::binary : comma::io::mode::ascii ) ); if( filter_transport->fd() == comma::io::invalid_file_descriptor ) { std::cerr << "csv-join: failed to open \"" << filter_csv.filename << "\"" << std::endl; return 1; } std::size_t discarded = 0; - read_filter_block(); + auto last = read_filter_block(); #ifdef WIN32 if( stdin_stream.is_binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); } #endif @@ -398,7 +441,18 @@ template < typename K, bool Strict = true > struct join_impl_ // quick and dirty { const input< K >* p = stdin_stream.read(); if( !p ) { break; } - if( block != p->block ) { read_filter_block(); } + if( block_less ) + { + if( p->block < block ) { continue; } + while( last && p->block >= last->block ) + { + last = read_filter_block(); + } + } + else + { + if( p->block != block ) { last = read_filter_block(); } + } typename traits< K, Strict >::pair pair; if( is_state_machine ) { @@ -430,6 +484,7 @@ template < typename K, bool Strict = true > struct join_impl_ // quick and dirty if( !strict ) { ++discarded; continue; } std::string s; comma::csv::options c; + c.full_xpath = false; c.fields = "keys"; std::cerr << "csv-join: match not found for key(s): " << comma::csv::ascii< input< K > >( c, default_input ).put( *p, s ) << ", block: " << block << std::endl; return 1; @@ -451,7 +506,7 @@ template < typename K, bool Strict = true > struct join_impl_ // quick and dirty if( is_state_machine ) { state = it->first.next_state; } if( flag_matching ) { char match = 1; std::cout.write( &match, 1 ); break; } if( matching ) { break; } - std::cout.write( &( it->second[i][0] ), filter_csv.format().size() ); + std::cout.write( &( it->second[i][0] ), it->second[i].size() ); if( swap_output ) { std::cout.write( stdin_stream.binary().last(), stdin_csv.format().size() ); } std::cout.flush(); } @@ -499,13 +554,15 @@ int main( int ac, char** av ) flag_matching = options.exists( "--flag-matching" ); radius = options.optional< double >( "--radius,--epsilon" ); nearest = options.exists( "--nearest" ); - swap_output = options.exists( "--swap-output,--swap" ); + swap_output = options.exists( "--output-swap,--swap-output,--swap" ); + filter_id_fields_discard = options.exists( "--drop-id-fields,--drop-id" ); if( nearest && !radius ) { std::cerr << "csv-join: if using --nearest, please specify --radius" << std::endl; return 1; } - options.assert_mutually_exclusive( "--matching,--not-matching,--flag-matching,--swap-output,--swap" ); + options.assert_mutually_exclusive( "--matching,--not-matching,--flag-matching,--swap-output,--swap,--output-swap" ); options.assert_mutually_exclusive( "--radius,--epsilon,--first-matching" ); options.assert_mutually_exclusive( "--radius,--epsilon,--string,-s,--double,--time" ); + options.assert_mutually_exclusive( "--matching,--not-matching", "--drop-id-fields,--drop-id" ); stdin_csv = comma::csv::options( options ); - std::vector< std::string > unnamed = options.unnamed( "--verbose,-v,--first-matching,--matching,--not-matching,--string,-s,--time,--double,--strict,--swap-output,--swap", "-.*" ); + std::vector< std::string > unnamed = options.unnamed( "--verbose,-v,--block-less,--first-matching,--matching,--not-matching,--string,-s,--time,--double,--strict,--swap-output,--swap,--output-swap,--nearest,--drop-id-fields,--drop-id", "-.*" ); if( unnamed.empty() ) { std::cerr << "csv-join: please specify the second source" << std::endl; return 1; } if( unnamed.size() > 1 ) { std::cerr << "csv-join: expected one file or stream to join, got " << comma::join( unnamed, ' ' ) << std::endl; return 1; } comma::name_value::parser parser( "filename", ';', '=', false ); diff --git a/csv/applications/csv-paste.cpp b/csv/applications/csv-paste.cpp index ef711b9da..ec660a6ea 100644 --- a/csv/applications/csv-paste.cpp +++ b/csv/applications/csv-paste.cpp @@ -1,32 +1,4 @@ -// This file is part of comma, a generic and flexible library // Copyright (c) 2011 The University of Sydney -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. Neither the name of the University of Sydney nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE -// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT -// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author vsevolod vlaskine @@ -37,7 +9,6 @@ #include #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../base/exception.h" #include "../../csv/format.h" #include "../../io/stream.h" @@ -71,18 +42,32 @@ static void usage( bool verbose ) std::cerr << std::endl; std::cerr << "options" << std::endl; std::cerr << " --delimiter,-d : default ','" << std::endl; + std::cerr << " --flush; flush stdout on every record" << std::endl; + std::cerr << " --head=[]; output first records and exit; convenience option, e.g. try:" << std::endl; + std::cerr << " csv-paste 'line-number;size=4' 'line-number;size=4;index' --head=16" << std::endl; std::cerr << " --help,-h : help, --help --verbose for more help" << std::endl; std::cerr << " --verbose,-v; more debug output" << std::endl; std::cerr << std::endl; std::cerr << "inputs" << std::endl; - std::cerr << " : [;size=|binary=]: file name or \"-\" for stdin; specify size or format, if binary" << std::endl; + std::cerr << " : [;]: file name or \"-\" for stdin; specify size or format, if binary" << std::endl; + std::cerr << " properties" << std::endl; + std::cerr << " binary=: if input is binary, record binary format; or use 'size'" << std::endl; + std::cerr << " block-size=; repeat each record times" << std::endl; + std::cerr << " size=; if input is binary, record size in bytes; or use 'binary'" << std::endl; std::cerr << " value : value=[;binary=]; specify size or format, if binary" << std::endl; std::cerr << " line-number[;] : add the line number; as ui, if binary (quick and dirty, will override the file named \"line-number\")" << std::endl; std::cerr << " options" << std::endl; - std::cerr << " --begin : start line number count at ; default: 0" << std::endl; + std::cerr << " --begin : start line number count at , can be negative; default: 0" << std::endl; + std::cerr << " --block-size,--size=: number of records with the same line number; default: 1" << std::endl; + std::cerr << " WARNING: --size: deprecated, since it is confusing for files" << std::endl; std::cerr << " --index; instead of block number output record index in the block" << std::endl; + std::cerr << " --repeat=[]; repeat a given pattern times" << std::endl; + std::cerr << " no --size: same as --head" << std::endl; + std::cerr << " --size: repeat block of a given size times" << std::endl; + std::cerr << " --shape: repeat a given shape times, e.g. 'line-number;shape=4,3,5;repeat=2'" << std::endl; std::cerr << " --reverse; if --index, output index in descending order" << std::endl; - std::cerr << " --size,--block-size : number of records with the same line number; default: 1" << std::endl; + std::cerr << " --shape=; iterate through indices of a given shape; : same meaning as in numpy, e.g. 'line-number;shape=10,5,4'" << std::endl; + std::cerr << " --step=; default=1; line number increment/decrement step" << std::endl; std::cerr << " examples (try them)" << std::endl; std::cerr << " line number" << std::endl; std::cerr << " seq 0 20 | csv-paste - line-number --begin 5 --size 3" << std::endl; @@ -94,7 +79,6 @@ static void usage( bool verbose ) std::cerr << "csv format parameters" << std::endl; if( verbose ) { std::cerr << comma::csv::format::usage() << std::endl; } else { std::cerr << " run csv-paste --help --verbose for more..." << std::endl; } std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; std::cerr << std::endl; exit( 0 ); } @@ -102,19 +86,20 @@ static void usage( bool verbose ) class source { public: - source( const std::string& properties = "" ) : properties_( properties ) + source( const std::string& properties = "" ) : properties_( properties ), block_count_( 0 ), buf_( nullptr ) { comma::name_value::map map( properties, ';', '=' ); format_ = comma::csv::format( map.value< std::string >( "binary", "" ) ); unsigned int size = map.value< unsigned int >( "size", format_.size() ); binary_ = size > 0; value_ = std::string( size, 0 ); + block_size_ = map.value< unsigned int >( "block-size", 1 ); } virtual ~source() {} virtual const std::string* read() = 0; virtual const char* read( char* buf ) = 0; bool binary() const { return binary_; } - virtual const bool is_stream() const { return false; } + virtual bool is_stream() const { return false; } const std::string& properties() const { return properties_; } std::size_t size() const { return value_.size(); } @@ -123,6 +108,9 @@ class source bool binary_; comma::csv::format format_; std::string properties_; + unsigned int block_size_; + unsigned int block_count_; + const char* buf_; }; class stream : public source @@ -136,22 +124,35 @@ class stream : public source const std::string* read() { - while( stream_->good() && !stream_->eof() ) + if( block_count_ == block_size_ || value_.empty() ) { - std::getline( *stream_, value_ ); - if( !value_.empty() && *value_.rbegin() == '\r' ) { value_ = value_.substr( 0, value_.length() - 1 ); } // windows... sigh... - if( !value_.empty() ) { return &value_; } + block_count_ = 1; + while( stream_->good() && !stream_->eof() ) + { + std::getline( *stream_, value_ ); + if( !value_.empty() && *value_.rbegin() == '\r' ) { value_ = value_.substr( 0, value_.length() - 1 ); } // windows... sigh... + if( !value_.empty() ) { return &value_; } + } + return nullptr; } - return NULL; + ++block_count_; + return &value_; } const char* read( char* buf ) { - stream_->read( buf, value_.size() ); - return stream_->gcount() == int( value_.size() ) ? buf : NULL; + if( block_count_ == block_size_ || buf_ == nullptr ) + { + block_count_ = 1; + buf_ = buf; // quick and dirty + stream_->read( buf, value_.size() ); + return stream_->gcount() == int( value_.size() ) ? buf : nullptr; + } + ++block_count_; + return buf_; } - const bool is_stream() { return true; } + bool is_stream() const { return true; } private: comma::io::istream stream_; @@ -176,79 +177,120 @@ class line_number : public source class options { public: - comma::uint32 size; - bool index; - bool reverse; - comma::uint32 begin; + comma::uint32 size{0}; + bool index{false}; + bool reverse{false}; + comma::int32 step{0}; + comma::int32 begin{0}; + std::vector< comma::uint32 > shape; + std::string format; + comma::uint32 repeat{0}; - options( boost::optional< comma::uint32 > b = boost::optional< comma::uint32 >(), comma::uint32 size = 1, bool index = false, bool reverse = false ) + options( const boost::optional< comma::int32 >& b = boost::optional< comma::int32 >(), comma::uint32 size = 1, bool index = false, bool reverse = false, int s = 1, unsigned int repeat = 0 ) : size( size ) , index( index ) , reverse( reverse ) + , step( s ) , begin( begin_( b ) ) + , repeat( repeat ) { } options( const std::string& properties, const comma::command_line_options& o ) // quick and dirty: use visiting instead { - options defaults( boost::optional< comma::uint32 >(), o.value< comma::uint32 >( "--size,--block-size", 1 ), o.exists( "--index" ), o.exists( "--reverse" ) ); + o.assert_mutually_exclusive( "--shape", "--block-size,--size,--reverse,--begin" ); + options defaults( boost::optional< comma::int32 >(), o.value< comma::uint32 >( "--block-size,--size", 1 ), o.exists( "--index" ), o.exists( "--reverse" ), o.value< comma::int32 >( "--step", 1 ), o.value< comma::uint32 >( "--repeat", 0 ) ); comma::name_value::map map( properties, ';', '=' ); - size = map.value< comma::uint32 >( "size", defaults.size ); - index = map.value< bool >( "index", defaults.index ); - reverse = map.value< bool >( "reverse", defaults.reverse ); - auto b = map.optional< comma::uint32 >( "begin" ); - if( !b ) { b = o.optional< comma::uint32 >( "--begin" ); } - begin = begin_( b ); + map.assert_mutually_exclusive( "shape", "block-size,size,reverse,begin,step" ); + std::string s = map.value< std::string >( "shape", o.value< std::string >( "--shape", "" ) ); + format = map.value< std::string >( "binary", "" ); + if( s.empty() ) + { + size = map.value< comma::uint32 >( map.get().find( "block-size" ) != map.get().end() ? "block-size" : "size", defaults.size ); // quick and dirty + index = map.value< bool >( "index", defaults.index ); + reverse = map.value< bool >( "reverse", defaults.reverse ); + step = map.value< comma::int32 >( "step", defaults.step ); + auto b = map.optional< comma::int32 >( "begin" ); + if( !b ) { b = o.optional< comma::int32 >( "--begin" ); } + begin = begin_( b ); + if( !format.empty() && format != "ui" ) { std::cerr << "csv-paste: currently only ui supported for line-number; got: '" << format << "'" << std::endl; exit( 1 ); } // quick and dirty for now + } + else + { + auto v = comma::split( s, ',' ); + shape.resize( v.size() ); + for( unsigned int i = 0; i < v.size(); ++i ) { shape[i] = boost::lexical_cast< unsigned int >( v[i] ); } + } + repeat = map.value< comma::int32 >( "repeat", defaults.repeat ); } private: - comma::uint32 begin_( const boost::optional< comma::uint32 >& b ) + comma::int32 begin_( const boost::optional< comma::int32 >& b ) // todo! handle size correctly for negative values for begin and step { - if( index && reverse && b && ( *b + 1 ) < size ) { COMMA_THROW( comma::exception, "for --reverse --index, for --size " << size << " expected --begin not less than " << ( size - 1 ) << "; got: " << *b ); } - return b ? *b : reverse ? size - 1 : 0; + if( index && reverse && b && ( *b + step ) < int( size ) * step ) { COMMA_THROW( comma::exception, "for --reverse --index, for --size " << size << " expected --begin not less than " << ( size - 1 ) << "; got: " << *b ); } + return b ? *b : reverse ? ( size - 1 ) * step : 0; } }; line_number( bool is_binary, const options& options ) - : source( is_binary ? "binary=ui" : "" ) + : source( options.format.empty() ? ( is_binary ? options.shape.empty() ? std::string ( "binary=ui" ) : "binary=" + boost::lexical_cast< std::string >( options.shape.size() ) + "ui" : std::string() ) : "binary=" + options.format ) // quick and dirty , options_( options ) , count_( 0 ) , value_( options_.begin ) + , values_( options_.shape.size(), 0 ) { } const std::string* read() { - serialized_ = boost::lexical_cast< std::string >( value_ ); - update_(); - return &serialized_; + serialized_ = values_.empty() ? boost::lexical_cast< std::string >( value_ ) : comma::join( values_, ',' ); + return update_() ? &serialized_ : nullptr; } const char* read( char* buf ) // quick and dirty { - comma::csv::format::traits< comma::uint32 >::to_bin( value_, buf ); - update_(); - return buf; + if( values_.empty() ) { comma::csv::format::traits< comma::int32 >::to_bin( value_, buf ); } + else { for( unsigned int i = 0; i < values_.size(); ++i, buf += sizeof( comma::int32 ) ) { comma::csv::format::traits< comma::int32 >::to_bin( values_[i], buf ); } } + return update_() ? buf : nullptr; } private: options options_; - comma::uint32 count_; - comma::uint32 value_; + comma::uint32 count_{0}; + comma::int32 value_{0}; + comma::uint32 _repeats{0}; + std::vector< comma::uint32 > values_; std::string serialized_; + bool _done{false}; - void update_() + bool update_() { - ++count_; - if( count_ < options_.size ) + if( _done ) { return false; } + if( values_.empty() ) { - if( options_.index ) { value_ += options_.reverse ? -1 : 1; } + ++count_; //count_ += options_.step; + if( count_ < options_.size ) + { + if( options_.index ) { value_ += options_.reverse ? -options_.step : options_.step; } + } + else + { + value_ = options_.index ? options_.begin : ( value_ + options_.step ); + count_ = 0; + if( options_.repeat > 0 ) { ++_repeats; if( _repeats == options_.repeat ) { _done = true; } } + } } else { - value_ = options_.index ? options_.begin : ( value_ + 1 ); - count_ = 0; + for( int i = values_.size() - 1; i >= 0; --i ) + { + ++values_[i]; + if( values_[i] < options_.shape[i] ) { break; } + if( i == 0 && options_.repeat > 0 ) { ++_repeats; if( _repeats == options_.repeat ) { _done = true; } } + values_[i] = 0; + } } + return true; } }; @@ -258,14 +300,16 @@ int main( int ac, char** av ) { comma::command_line_options options( ac, av, usage ); char delimiter = options.value( "--delimiter,-d", ',' ); - std::vector< std::string > unnamed = options.unnamed( "--flush,--index,--reverse", "--delimiter,-d,--begin,--size,--block-size" ); + std::vector< std::string > unnamed = options.unnamed( "--flush,--index,--reverse", "--delimiter,-d,--begin,--size,--step,--block-size,--head,--repeat" ); + bool flush = options.exists( "--flush" ); boost::ptr_vector< source > sources; bool is_binary = false; - for( unsigned int i = 0; i < unnamed.size(); ++i ) // quick and dirty + boost::optional< comma::uint32 > head = options.optional< comma::uint32 >( "--head" ); + for( unsigned int i = 0; i < unnamed.size(); ++i ) // quick and dirty; really lousy code duplication { if( unnamed[i].substr( 0, 6 ) == "value=" ) { if( value( unnamed[i] ).binary() ) { is_binary = true; } } - else if( unnamed[i] == "line-number" || unnamed[i].substr( 0, 12 ) == "line-number;" ) { continue; } // quick and dirty - if( stream( unnamed[i] ).binary() ) { is_binary = true; } + else if( unnamed[i] == "line-number" || unnamed[i].substr( 0, 12 ) == "line-number;" ) { if( line_number( is_binary, line_number::options( unnamed[i], options ) ).binary() ) { is_binary = true; } } // quick and dirty + else if( stream( unnamed[i] ).binary() ) { is_binary = true; } } for( unsigned int i = 0; i < unnamed.size(); ++i ) { @@ -295,13 +339,13 @@ int main( int ac, char** av ) std::size_t size = 0; for( unsigned int i = 0; i < sources.size(); ++i ) { size += sources[i].size(); } std::vector< char > buffer( size ); - while( true ) + while( !head || ( *head )-- ) { unsigned int streams = 0; char* p = &buffer[0]; for( unsigned int i = 0; i < sources.size(); p += sources[i].size(), ++i ) { - if( sources[i].read( p ) == NULL ) + if( sources[i].read( p ) == nullptr ) { if( streams == 0 ) { return 0; } std::cerr << "csv-paste: unexpected end of file in " << unnamed[i] << std::endl; @@ -310,29 +354,28 @@ int main( int ac, char** av ) if( sources[i].is_stream() ) { ++streams; } } std::cout.write( &buffer[0], buffer.size() ); - std::cout.flush(); + if( flush ) { std::cout.flush(); } } + return 0; } - else + while( !head || ( *head )-- ) { - while( true ) + std::ostringstream oss; + unsigned int streams = 0; + for( unsigned int i = 0; i < sources.size(); ++i ) { - std::ostringstream oss; - unsigned int streams = 0; - for( unsigned int i = 0; i < sources.size(); ++i ) + const std::string* s = sources[i].read(); + if( s == nullptr ) { - const std::string* s = sources[i].read(); - if( s == NULL ) - { - if( streams == 0 ) { return 0; } - std::cerr << "csv-paste: unexpected end of file in " << unnamed[i] << std::endl; return 1; - } - if (sources[i].is_stream()) ++streams; - if( i > 0 ) { oss << delimiter; } - oss << *s; + if( streams == 0 ) { return 0; } + std::cerr << "csv-paste: unexpected end of file in " << unnamed[i] << std::endl; + return 1; } - std::cout << oss.str() << std::endl; + if( sources[i].is_stream() ) { ++streams; } + if( i > 0 ) { oss << delimiter; } + oss << *s; } + std::cout << oss.str() << std::endl; } return 0; } @@ -340,124 +383,3 @@ int main( int ac, char** av ) catch( ... ) { std::cerr << "csv-paste: unknown exception" << std::endl; } return 1; } - - -// int main( int ac, char** av ) -// { -// bool show_usage = true; -// try -// { -// comma::command_line_options options( ac, av ); -// if( options.exists( "--help,-h" ) ) { usage(); } -// char delimiter = options.value( "--delimiter,-d", ',' ); -// std::vector< std::string > unnamed = options.unnamed( "", "--delimiter,-d" ); -// boost::ptr_vector< std::istream > files; -// std::vector< std::pair< std::istream*, std::size_t > > sources; -// bool binary = false; -// for( unsigned int i = 0; i < unnamed.size(); ++i ) -// { -// std::string filename = unnamed[i]; -// std::size_t size = 0; -// std::vector< std::string > v = comma::split( unnamed[i], ';' ); -// filename = v[0]; -// for( std::size_t j = 1; j < v.size(); ++j ) -// { -// std::vector< std::string > w = comma::split( v[j], '=' ); -// if( w.size() != 2 ) { COMMA_THROW( comma::exception, "expected filename and options, got \"" << unnamed[i] << "\"" ); } -// if( w[0] == "binary" ) -// { -// if( i == 0 ) { binary = true; } -// else if( !binary ) { COMMA_THROW( comma::exception, unnamed[0] << " is ascii, but " << filename << " is binary" ); } -// size = comma::csv::format( w[1] ).size(); -// } -// else if( w[0] == "size" ) -// { -// if( i == 0 ) { binary = true; } -// else if( !binary ) { COMMA_THROW( comma::exception, unnamed[0] << " is ascii, but " << filename << " is binary" ); } -// size = boost::lexical_cast< std::size_t >( w[1] ); -// } -// } -// if( binary && size == 0 ) { COMMA_THROW( comma::exception, "in binary mode, please specify size or format for \"" << filename << "\"" ); } -// if( filename == "-" ) -// { -// sources.push_back( std::make_pair( &std::cin, size ) ); -// } -// else -// { -// files.push_back( new std::ifstream( filename.c_str() ) ); -// if( !files.back().good() || files.back().eof() ) { COMMA_THROW( comma::exception, "failed to open " << unnamed[i] ); } -// sources.push_back( std::make_pair( &files.back(), size ) ); -// } -// } -// if( sources.empty() ) { usage(); } -// #ifdef WIN32 -// if( binary ) { _setmode( _fileno( stdin ), _O_BINARY ); } -// #endif -// show_usage = false; -// if( binary ) -// { -// std::size_t size = 0; -// for( unsigned int i = 0; i < sources.size(); ++i ) { size += sources[i].second; } -// while( true ) -// { -// for( unsigned int i = 0; i < sources.size(); ++i ) -// { -// std::string s( sources[i].second, 0 ); -// char* buf = &s[0]; -// sources[i].first->read( buf, sources[i].second ); -// int count = sources[i].first->gcount(); -// if( count != 0 && (unsigned int)count != sources[i].second ) { COMMA_THROW( comma::exception, unnamed[i] << ": expected " << sources[i].second << " bytes, got " << count ); } -// if( !sources[i].first->good() || sources[i].first->eof() ) -// { -// bool ok = true; -// for( unsigned int j = 0; j < sources.size() && ok; ++j ) -// { -// if( j > i ) { sources[j].first->peek(); } -// ok = !sources[j].first->good() || sources[j].first->eof(); -// } -// if( ok ) { return 0; } -// else { COMMA_THROW( comma::exception, unnamed[i] << ": unexpected end of file" ); } -// } -// std::cout << s; -// } -// } -// } -// else -// { -// while( true ) -// { -// bool first = true; -// for( unsigned int i = 0; i < sources.size(); ++i ) -// { -// std::string s; -// std::getline( *sources[i].first, s ); -// if( !sources[i].first->good() || sources[i].first->eof() ) -// { -// bool ok = true; -// for( unsigned int j = 0; j < sources.size() && ok; ++j ) -// { -// if( j > i ) { sources[j].first->peek(); } -// ok = !sources[j].first->good() || sources[j].first->eof(); -// } -// if( ok ) { return 0; } -// else { COMMA_THROW( comma::exception, unnamed[i] << ": unexpected end of file" ); } -// } -// if( !s.empty() && *s.rbegin() == '\r' ) { s = s.substr( 0, s.length() - 1 ); } // windows... sigh... -// if( s.empty() ) { continue; } -// if( !first ) { std::cout << delimiter; } else { first = false; } -// std::cout << s; -// } -// std::cout << std::endl; -// } -// } -// } -// catch( std::exception& ex ) -// { -// std::cerr << "csv-paste: " << ex.what() << std::endl; -// } -// catch( ... ) -// { -// std::cerr << "csv-paste: unknown exception" << std::endl; -// } -// if( show_usage ) { usage(); } -// } diff --git a/csv/applications/csv-play.cpp b/csv/applications/csv-play.cpp index 3d9eb2a69..66e513089 100644 --- a/csv/applications/csv-play.cpp +++ b/csv/applications/csv-play.cpp @@ -27,169 +27,223 @@ // OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /// @author cedric wohlleber #include #include #ifdef WIN32 -#include #include #else #include #include #include -#include #include -#include #include #endif #include #include -#include +#include #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../application/signal_flag.h" #include "../../base/exception.h" #include "../../csv/options.h" -#include "../../csv/stream.h" #include "../../csv/traits.h" #include "../../name_value/parser.h" -#include "../../csv/applications/play/play.h" #include "../../csv/applications/play/multiplay.h" -static void usage() +static void bash_completion( unsigned const ac, char const* const* av ) { - std::cerr << std::endl; - std::cerr << "play back timestamped data from standard input in a real time manner" << std::endl; - std::cerr << "to standard output or optionally into given files/pipes" << std::endl; - std::cerr << std::endl; - std::cerr << "usage: csv-play []" << std::endl; - std::cerr << std::endl; - std::cerr << "options" << std::endl; - std::cerr << " --speed: speed-up playback by a factor, default is 1 (inverse to --slowdown)" << std::endl; - std::cerr << " --slowdown,--slow: slow-down playback by a factor, default is 1 (inverse to --speed)" << std::endl; - std::cerr << " --quiet: don't print warnings when lagging behind" << std::endl; - std::cerr << " --fields : specify where timestamp is" << std::endl; - std::cerr << " e.g., if timestamp is the 4th field: --fields=\",,,t\"" << std::endl; - std::cerr << " default: the timestamp is the first field" << std::endl; - std::cerr << " --binary : use binary format" << std::endl; - std::cerr << " --clients: minimum number of clients to connect to each stream" << std::endl; - std::cerr << " before playback starts; default 0" << std::endl; - std::cerr << " can be specified individually for each client, e.g." << std::endl; - std::cerr << " csv-play file1;pipe;clients=1 file2;tcp:1234;clients=3" << std::endl; - std::cerr << " --interactive,-i: react to key presses:" << std::endl; - std::cerr << " : pause, resume" << std::endl; - std::cerr << " left or down arrow key: output one record at a time" << std::endl; - std::cerr << " shift left or down arrow key: TODO: output one block at a time" << std::endl; - std::cerr << " --no-flush : if present, do not flush the output stream ( use on high bandwidth sources )" << std::endl; - std::cerr << " --paused-at-start,--paused; if --interactive, then start playback as paused" << std::endl; - std::cerr << " --resolution=: timestamp resolution; timestamps closer than this value will be" << std::endl; - std::cerr << " played without delay; the rationale is that microsleep used in csv-play" << std::endl; - std::cerr << " (boost::this_thread::sleep()) is essentially imprecise and may create" << std::endl; - std::cerr << " unnecessary delays in the data" << std::endl; - std::cerr << " default 0.01" << std::endl; - std::cerr << " --from : play back data starting at ( iso format )" << std::endl; - std::cerr << " --to : play back data up to ( iso format )" << std::endl; - std::cerr << comma::csv::format::usage(); - std::cerr << std::endl; - std::cerr << "output" << std::endl; - std::cerr << " -: write to stdout (default)" << std::endl; - std::cerr << " offset=: add seconds to the timestamp of this source" << std::endl; - std::cerr << " : write to file or named pipe, e.g. csv-play \"points.csv;pipe\"" << std::endl; - std::cerr << " tcp:: open tcp server socket on given port and write to the tcp clients" << std::endl; - std::cerr << " local:: same as tcp, but use unix/linux domain sockets" << std::endl; - std::cerr << std::endl; - std::cerr << "examples" << std::endl; - std::cerr << " output timestamped 3d points in real time manner to stdout (e.g. for visualisation)" << std::endl; - std::cerr << " cat points.csv | csv-play | view-points --fields=,x,y,z" << std::endl; - std::cerr << std::endl; - std::cerr << " play back several files and output to, say, named pipes:" << std::endl; - std::cerr << " mkfifo file1.pipe file2.pipe" << std::endl; - std::cerr << " csv-play \"file1.csv;pipe1\" \"file2.csv;pipe2\" &" << std::endl; - std::cerr << " view-points pipe1 pipe2 --fields=,x,y,z" << std::endl; - std::cerr << std::endl; - std::cerr << " same as above, but block, until all the pipes are connected:" << std::endl; - std::cerr << " csv-play \"file1.csv;pipe1\" \"file2.csv;pipe2\" --clients=1 &" << std::endl; - std::cerr << std::endl; - std::cerr << " output multiple inputs of the same format to stdout:" << std::endl; - std::cerr << " csv-play \"file1.csv;-\" \"file2.csv;-\" &" << std::endl; - std::cerr << std::endl; - std::cerr << comma::contact_info << std::endl; - std::cerr << std::endl; - exit( -1 ); + static const char* completion_options = + " --help -h" + " --speed --slowdown --slow" + " --quiet" + " --fields --binary" + " --clients" + " --interactive -i" + " --no-flush " + " --paused-at-start --paused" + " --resolution" + " --from --to" + ; + std::cout << completion_options << std::endl; + exit( 0 ); } +static void interactive_help( std::string prefix ) +{ + prefix.assign( prefix.size(), ' ' ); + std::cerr << prefix << ": pause or resume" << std::endl; + std::cerr << prefix << "right or down arrow key: output one record at a time" << std::endl; + std::cerr << prefix << ": output current timestamp to stderr" << std::endl; + std::cerr << prefix << ": quit" << std::endl; +} + +static void usage( bool verbose ) +{ + std::cerr << R"( +play back timestamped data from standard input in a real time manner +to standard output or optionally into given files/pipes + +usage: csv-play [] + +options + --speed: speed-up playback by a factor, default is 1 (inverse to --slowdown) + --slowdown,--slow: slow-down playback by a factor, default is 1 (inverse to --speed) + --quiet: don't print warnings when lagging behind + --fields : specify where timestamp is + e.g., if timestamp is the 4th field: --fields=',,,t' + default: the timestamp is the first field + --binary : use binary format + --clients: minimum number of clients to connect to each stream + before playback starts; default 0 + can be specified individually for each client, e.g. + csv-play file1;pipe;clients=1 file2;tcp:1234;clients=3 + --interactive,-i: react to key presses:" +)"; + interactive_help( " --interactive,-i: " ); + std::cerr << R"( --no-flush : if present, do not flush the output stream ( use on high bandwidth sources ) + --paused-at-start,--paused: start playback as paused, implies --interactive + --pause-at=[]; pause when timestamp reached, implies --interactive + --resolution=: timestamp resolution; timestamps closer than this value will be + played without delay; the rationale is that microsleep used in csv-play + (boost::this_thread::sleep()) is essentially imprecise and may create + unnecessary delays in the data + default 0.01 + --from : play back data starting at ( iso format ) + --to : play back data up to ( iso format ) +)" << std::endl; + std::cerr << "csv options" << std::endl; + std::cerr << comma::csv::options::usage( verbose ); + std::cerr << R"( +output + -: write to stdout (default) + offset=: add seconds to the timestamp of this source + : write to file or named pipe, e.g. csv-play 'points.csv;pipe' + tcp:: open tcp server socket on given port and write to the tcp clients + local:: same as tcp, but use unix/linux domain sockets + +examples + output timestamped 3d points in real time manner to stdout (e.g. for visualisation) + cat points.csv | csv-play | view-points --fields=,x,y,z + + play back several files and output to, say, named pipes: + mkfifo file1.pipe file2.pipe + csv-play 'file1.csv;pipe1' 'file2.csv;pipe2' & + view-points pipe1 pipe2 --fields=,x,y,z + + same as above, but block, until all the pipes are connected: + csv-play 'file1.csv;pipe1' 'file2.csv;pipe2' --clients=1 & + + output multiple inputs of the same format to stdout: + csv-play 'file1.csv;-' 'file2.csv;-' & + + use binary data (try it) + > csv-play <( csv-paste line-number | csv-repeat --pace --period 1 | csv-time-amp | csv-to-bin t,ui --flush )';-;binary=t,ui' \ + <( csv-paste line-number value=0 | csv-repeat --pace --period 1 | csv-time-stamp | csv-to-bin t,2ui --flush )';tcp:8888;binary=t,2ui' \ + | csv-from-bin t,ui + > #in another shell, run + > socat tcp:localhost:8888 - | csv-from-bin t,2ui + + pause and step through output: + echo 0 | csv-repeat --period 0.1 --yes | csv-paste - line-number | csv-time-stamp | csv-play --interactive + +)" << std::endl; + exit( 0 ); +} + +static boost::scoped_ptr< comma::csv::applications::play::Multiplay > multiplay; +static bool quit = false; + +class playback_state_t +{ +public: + playback_state_t() : state_( state::running ) {} + + bool is_running() const { return state_ == state::running; } + + bool is_paused() const { return state_ == state::paused; } + + void pause( const boost::posix_time::ptime& t = boost::posix_time::not_a_date_time ) + { + if( state_ == state::paused ) { return; } + state_ = state::paused; + paused_time_ = boost::posix_time::microsec_clock::universal_time(); + if( ! t.is_not_a_date_time() ) { std::cerr << "csv-play: paused at " << boost::posix_time::to_iso_string( t ) << std::endl; } + } + + void unpause() + { + multiplay->paused_for( boost::posix_time::microsec_clock::universal_time() - paused_time_ ); + } + + void run() + { + if( state_ == state::running ) { return; } + if( state_ == state::paused ) { unpause(); } + state_ = state::running; + std::cerr << "csv-play: resumed" << std::endl; + } + + void read_once() + { + if( state_ == state::paused ) { unpause(); } + state_ = state::read_once; + } + + void has_read_once() { if( state_ == state::read_once ) { pause(); } } + +private: + enum class state { running, paused, read_once, read_block }; + state state_; + boost::posix_time::ptime paused_time_; +}; + +static playback_state_t playback; + class key_press_handler_t { public: - enum states { running, paused, read_once, read_block }; - - key_press_handler_t( bool interactive, bool paused_at_start ): key_press_( interactive ), paused_( paused_at_start ), state_( paused_ ? paused : running ) { if( paused_at_start ) { std::cerr << "csv-play: paused at start" << std::endl; } } - + key_press_handler_t( bool interactive ) : key_press_( interactive ) {} + void update( boost::posix_time::ptime t ) + { + key k = get_key(); + switch( k ) + { + case key::space: if( playback.is_running() ) { playback.pause( t ); } else { playback.run(); } break; + case key::down_arrow: case key::right_arrow: playback.read_once(); break; + case key::q: quit = true; break; + case key::t: std::cerr << boost::posix_time::to_iso_string( t ) << std::endl; break; + case key::none: case key::other: break; + } + } + +private: + enum class key { none, space, right_arrow, down_arrow, q, t, other }; + + key get_key() { boost::optional< char > c = key_press_.read(); - if( !c ) { return; } + if( !c ) { return key::none; } switch( *c ) { - case 10: - case ' ': - switch( state_ ) - { - case running: - std::cerr << "csv-play: paused at " << boost::posix_time::to_iso_string( t ) << std::endl; - state_ = paused; - break; - case paused: - std::cerr << "csv-play: resumed" << std::endl; - state_ = running; - break; - case read_block: - return; // never here, todo - case read_once: - std::cerr << "csv-play: resumed" << std::endl; - state_ = running; - break; - }; - break; - case 27: + case ' ': return key::space; + case 'q': return key::q; + case 't': return key::t; + case 27: // escape sequence for arrows: ESC-[ c = key_press_.read(); - if( !c || *c != 91 ) { return; } + if( !c || *c != 91 ) { break; } c = key_press_.read(); - if( !c ) { return; } - switch( *c ) - { - case 66: - case 67: - state_ = read_once; - break; - default: - return; - } - default: + if( !c ) { break; } + if( *c == 66 ) { return key::down_arrow; } + if( *c == 67 ) { return key::right_arrow; } break; } + return key::other; } - - states state() const { return state_; } - - void has_read_once() - { - switch( state_ ) - { - case running: - case paused: - case read_block: - return; - case read_once: - state_ = paused; - }; - } - -private: + class key_press_t_ { public: @@ -206,8 +260,7 @@ class key_press_handler_t new_termios.c_iflag &= ~( BRKINT | ICRNL | INPCK | ISTRIP | IXON ); if( ::tcsetattr( fd_, TCSANOW, &new_termios ) < 0 ) { COMMA_THROW( comma::exception, "failed to set '" << tty << "'" ); } std::cerr << "csv-play: running in interactive mode" << std::endl; - std::cerr << " press to pause or resume" << std::endl; - std::cerr << " press left or down arrow key: output one record at a time" << std::endl; + interactive_help( "csv-play: " ); } ~key_press_t_() @@ -231,20 +284,18 @@ class key_press_handler_t int fd_; struct termios old_termios_; }; + key_press_t_ key_press_; - bool paused_; - states state_; }; int main( int argc, char** argv ) { - boost::scoped_ptr< comma::Multiplay > multiplay; try { const boost::array< comma::signal_flag::signals, 2 > signals = { { comma::signal_flag::sigint, comma::signal_flag::sigterm } }; comma::signal_flag shutdown_flag( signals ); - comma::command_line_options options( argc, argv ); - if( options.exists( "--help,-h" ) ) { usage(); } + comma::command_line_options options( argc, argv, usage ); + if( options.exists( "--bash-completion" ) ) bash_completion( argc, argv ); options.assert_mutually_exclusive( "--speed,--slow,--slowdown" ); double speed = options.value( "--speed", 1.0 / options.value< double >( "--slow,--slowdown", 1.0 ) ); double resolution = options.value< double >( "--resolution", 0.01 ); @@ -252,25 +303,34 @@ int main( int argc, char** argv ) std::string to = options.value< std::string>( "--to", "" ); bool quiet = options.exists( "--quiet" ); bool flush = !options.exists( "--no-flush" ); - std::vector< std::string > configstrings = options.unnamed("--interactive,-i,--paused,--paused-at-start,--quiet,--flush,--no-flush","--slow,--slowdown,--speed,--resolution,--binary,--fields,--clients,--from,--to"); + std::vector< std::string > configstrings = options.unnamed( "--verbose,-v,--interactive,-i,--paused,--paused-at-start,--quiet,--flush,--no-flush","--pause-at,--slow,--slowdown,--speed,--resolution,--binary,--fields,--clients,--from,--to" ); if( configstrings.empty() ) { configstrings.push_back( "-;-" ); } - comma::csv::options csvoptions( argc, argv ); + comma::csv::options csv( argc, argv ); + csv.full_xpath = false; comma::name_value::parser name_value("filename,output", ';', '=', false ); - std::vector< comma::Multiplay::SourceConfig > sourceConfigs( configstrings.size() ); - comma::Multiplay::SourceConfig defaultConfig( "-", options.value( "--clients", 0 ), csvoptions ); - for( unsigned int i = 0U; i < configstrings.size(); ++i ) { sourceConfigs[i] = name_value.get< comma::Multiplay::SourceConfig >( configstrings[i], defaultConfig ); } + std::vector< comma::csv::applications::play::Multiplay::SourceConfig > source_configs( configstrings.size() ); + comma::csv::applications::play::Multiplay::SourceConfig defaultConfig( "-", options.value( "--clients", 0 ), csv ); + for( unsigned int i = 0U; i < configstrings.size(); ++i ) { source_configs[i] = name_value.get< comma::csv::applications::play::Multiplay::SourceConfig >( configstrings[i], defaultConfig ); } boost::posix_time::ptime fromtime; if( !from.empty() ) { fromtime = boost::posix_time::from_iso_string( from ); } boost::posix_time::ptime totime; if( !to.empty() ) { totime = boost::posix_time::from_iso_string( to ); } - multiplay.reset( new comma::Multiplay( sourceConfigs, 1.0 / speed, quiet, boost::posix_time::microseconds( static_cast (resolution * 1000000) ), fromtime, totime, flush ) ); - key_press_handler_t key_press_handler( options.exists( "--interactive,-i" ), options.exists( "--paused,--paused-at-start" ) ); - while( !shutdown_flag && std::cout.good() && !std::cout.bad() && !std::cout.eof() ) + multiplay.reset( new comma::csv::applications::play::Multiplay( source_configs, speed, quiet, boost::posix_time::microseconds( static_cast< unsigned int >( resolution * 1000000 )), fromtime, totime, flush )); + if( options.exists( "--paused,--paused-at-start" )) { playback.pause(); } + boost::optional< std::string > pause_at_option = options.optional< std::string >( "--pause-at" ); + boost::optional< boost::posix_time::ptime > pause_at_timestamp = boost::make_optional< boost::posix_time::ptime >( false, boost::posix_time::not_a_date_time ); + if( pause_at_option ) { pause_at_timestamp = boost::posix_time::from_iso_string( *pause_at_option ); } + key_press_handler_t key_press_handler( options.exists( "--interactive,-i" ) + || options.exists( "--paused,--paused-at-start" ) + || options.exists( "--pause-at" )); + while( !shutdown_flag && !quit && std::cout.good() ) { - key_press_handler.update( multiplay->now() ); - if( key_press_handler.state() == key_press_handler_t::paused ) { boost::this_thread::sleep( boost::posix_time::millisec( 200 ) ); continue; } + boost::posix_time::ptime now = multiplay->now(); + key_press_handler.update( now ); + if( pause_at_timestamp && !now.is_not_a_date_time() && *pause_at_timestamp < now ) { playback.pause( now ); pause_at_timestamp = boost::none; } + if( playback.is_paused() ) { boost::this_thread::sleep( boost::posix_time::millisec( 200 ) ); continue; } if( !multiplay->read() ) { break; } - key_press_handler.has_read_once(); + playback.has_read_once(); } multiplay->close(); multiplay.reset(); diff --git a/csv/applications/csv-random.cpp b/csv/applications/csv-random.cpp new file mode 100644 index 000000000..745b0c2e8 --- /dev/null +++ b/csv/applications/csv-random.cpp @@ -0,0 +1,531 @@ +// Copyright (c) 2018 Vsevolod Vlaskine + +/// @authors vsevolod vlaskine, kent hu + +#include +#include +#include +#include +#include +#include +#include + +#include "../../application/command_line_options.h" +#include "../../base/exception.h" +#include "../../base/none.h" +#include "../../base/types.h" +#include "../../csv/stream.h" +#include "../../string/string.h" + +// todo +// - seed=true-random +// - make, true-random: --head= +// - sample, shuffle +// - examples +// - regression test! +// - --help vs --help --verbose +// ? wiki: tutorials +// - csv-random +// - csv-repeat: --pace etc +// - make tutorials searchable + +static void usage( bool verbose ) +{ + std::cerr << "\nrandom operations on input stream"; + std::cerr << '\n'; + std::cerr << "\nusage: csv-random []"; + std::cerr << "\n"; + std::cerr << "\n where is one of:"; + std::cerr << "\n pseudo-random, make: output pseudo-random numbers"; + std::cerr << "\n true-random: output non-deterministic uniformly distributed numbers"; + std::cerr << "\n sample: output a uniformly distributed sample of input records"; + std::cerr << "\n shuffle: output input records in pseudo-random order"; + std::cerr << '\n'; + std::cerr << "\noptions"; + std::cerr << "\n --seed=[]; random seed:"; + std::cerr << "\n : integer seed for pseudo-random generator"; + std::cerr << "\n 'true-random': todo: true random number to use as seed"; + std::cerr << '\n'; + std::cerr << "\noperations"; + std::cerr << "\n pseudo-random, make: output pseudo-random numbers"; + std::cerr << '\n'; + std::cerr << "\n usage: csv-random make [] > random.csv"; + std::cerr << "\n cat input.csv | csv-random make --append []"; + std::cerr << '\n'; + std::cerr << "\n options"; + std::cerr << "\n --append; append random numbers to stdin input"; + std::cerr << "\n --distribution=[;]"; + std::cerr << "\n where is one of:"; + std::cerr << "\n uniform[;;] (default)"; + std::cerr << "\n if , not present, --range values will be used"; + std::cerr << "\n gaussian[;;]"; + std::cerr << "\n normal: alias for gaussian"; + std::cerr << "\n todo: more distributions to plug in, just ask"; + std::cerr << "\n --engine="; + std::cerr << "\n where is one of: minstd_rand0, minstd_rand, mt19937,"; + std::cerr << "\n mt19937_64 (default), ranlux24_base, ranlux48_base,"; + std::cerr << "\n ranlux24, ranlux48, knuth_b, default_random_engine"; + std::cerr << "\n --output-binary; output random numbers as binary"; + std::cerr << "\n specify --binary= for stdin input"; + std::cerr << "\n --range=[,]; desired value range"; + std::cerr << "\n attention! will pick value until gets something in range"; + std::cerr << "\n --type=; default=ui; supported values: b,ub,w,uw,i,ui,l,ul,f,d"; + std::cerr << "\n can have more than one i.e. 3ui"; + std::cerr << '\n'; + std::cerr << "\n true-random: output non-deterministic uniformly distributed unsigned int"; + std::cerr << "\n random numbers (if non-deterministic source is not available)"; + std::cerr << "\n e.g. a hardware device, output will be pseudo-random"; + std::cerr << '\n'; + std::cerr << "\n usage: csv-random true-random []"; + std::cerr << "\n cat input.csv | csv-random true-random --append []"; + std::cerr << '\n'; + std::cerr << "\n options"; + std::cerr << "\n --append; append random number to stdin input"; + std::cerr << "\n --once; output random number only once"; + std::cerr << "\n --output-binary; output random numbers as binary"; + std::cerr << "\n specify --binary= for stdin input"; + std::cerr << "\n --range=[,]; desired value range"; + std::cerr << "\n if multiple output values, e.g: --type=f,ui,ub"; + std::cerr << "\n --range will be applied to all output values"; + std::cerr << "\n --type=; default=ui; todo: supported values: ui;"; + std::cerr << "\n e.g: --type=3ui; --type=ui,ui,ui; etc"; + std::cerr << '\n'; + std::cerr << "\n example"; + std::cerr << "\n > csv-random make --seed=$( csv-random true-random --once )"; + std::cerr << '\n'; + std::cerr << "\n sample: output uniformly distributed sample of input records of a given size"; + std::cerr << "\n record order preserved"; + std::cerr << "\n limitation: current implementation accumulates input records before"; + std::cerr << "\n outputting, if records are large, it may be memory-inefficient;"; + std::cerr << "\n can be improved, just ask"; + std::cerr << '\n'; + std::cerr << "\n usage: cat records.csv | csv-random sample [] > sample.csv"; + std::cerr << '\n'; + std::cerr << "\n options"; + std::cerr << "\n --engine="; + std::cerr << "\n where is one of: minstd_rand0, minstd_rand, mt19937,"; + std::cerr << "\n mt19937_64 (default), ranlux24_base, ranlux48_base,"; + std::cerr << "\n ranlux24, ranlux48, knuth_b, default_random_engine"; + std::cerr << "\n --fields=[]; if 'block' field present, sample each block,"; + std::cerr << "\n otherwise read whole input and then sample"; + std::cerr << "\n --ratio=[]; portion of each block to output,"; + std::cerr << "\n if block is too small, nothing will be output for it"; + std::cerr << "\n --size=; default=1; number of records to output in each block,"; + std::cerr << "\n if smaller than block size, output the whole block"; + std::cerr << "\n --sliding-window,--window=[]; todo: sample on sliding window"; + std::cerr << "\n of records"; + std::cerr << '\n'; + std::cerr << "\n shuffle: output input records in pseudo-random order"; + std::cerr << '\n'; + std::cerr << "\n usage: cat records.csv | csv-random shuffle [] > shuffled.csv"; + std::cerr << '\n'; + std::cerr << "\n options"; + std::cerr << "\n --engine="; + std::cerr << "\n where is one of: minstd_rand0, minstd_rand, mt19937,"; + std::cerr << "\n mt19937_64 (default), ranlux24_base, ranlux48_base,"; + std::cerr << "\n ranlux24, ranlux48, knuth_b, default_random_engine"; + std::cerr << "\n --fields=[]; if 'block' field present, shuffle each block,"; + std::cerr << "\n otherwise read whole input and then shuffle"; + std::cerr << "\n --ratio=[]; portion of each block to output,"; + std::cerr << "\n same as for \"sample\" operation, but shuffled"; + std::cerr << "\n --size=; default=1; number of records to output in each block,"; + std::cerr << "\n same as for \"sample\" operation, but shuffled"; + std::cerr << "\n --sliding-window,--window=[]; todo: shuffle on sliding window"; + std::cerr << "\n of records"; + std::cerr << '\n'; + std::cerr << "\ncsv options:"; + std::cerr << comma::csv::options::usage( "", verbose ) << std::endl; + std::cerr << "\nexamples"; + std::cerr << "\n generate three floating point random samples from gaussian distribution"; + std::cerr << "\n csv-random make --distribution gaussian 0,10 --type 3f"; + std::cerr << "\n csv-random make --distribution gaussian 0,10 --type 3f \\"; + std::cerr << "\n | csv-paste line-number - \\"; + std::cerr << "\n | csv-blocks group --fields scalar --span 1000 \\"; + std::cerr << "\n | csv-shuffle --fields=id,x,y,z,block --output-fields=id,block,x,y,z"; + std::cerr << "\n\n"; + exit( 0 ); +} + +static bool verbose; +static comma::csv::options csv; +static boost::optional< int > seed; + +namespace comma { namespace applications { namespace random { namespace shuffle { + +struct input +{ + comma::uint32 block{0}; +}; + +} } } } // namespace comma { namespace applications { namespace random { namespace shuffle { + +namespace comma { namespace visiting { + +template <> struct traits< comma::applications::random::shuffle::input > +{ + template < typename K, typename V > static void visit( const K&, const comma::applications::random::shuffle::input& p, V& v ) { v.apply( "block", p.block ); } + template < typename K, typename V > static void visit( const K&, comma::applications::random::shuffle::input& p, V& v ) { v.apply( "block", p.block ); } +}; + +} } // namespace comma { namespace visiting { + +namespace comma { namespace applications { namespace random { + +template < typename T > struct type_traits { static T cast( const T t ) { return t; } }; +template <> struct type_traits< char > { static int cast( const char t ) { return static_cast< int >( t ); } }; +template <> struct type_traits< unsigned char > { static unsigned int cast( const unsigned char t ) { return static_cast< int >( t ); } }; + +namespace make { + +template < typename T, template < typename > class Distribution, typename Engine > +static int run_impl( Distribution< T >& distribution, bool append, bool binary, std::size_t count, const boost::optional< std::pair< T, T > >& range ) +{ + Engine engine = ::seed ? Engine( *::seed ) : Engine(); + auto _pick = [&]() -> T + { + if( !range ) { return distribution( engine ); } + while( true ) // todo? parametrise? while( true ) is quite cruel + { + T r = distribution( engine ); + if( r >= range->first && r <= range->second ) { return r; } + } + }; + if( !::csv.flush ) { std::cin.tie( nullptr ); } + if( append ) + { + if( ::csv.binary() ) + { + std::vector< char > buf( ::csv.format().size() ); + while( std::cin.good() ) + { + std::cin.read( &buf[0], buf.size() ); + if( std::cin.gcount() == 0 ) { break; } + if( std::cin.gcount() != static_cast< int >( buf.size() ) ) { std::cerr << "csv-random make: expected " << buf.size() << " bytes; got " << std::cin.gcount() << std::endl; return 1; } + std::cout.write( &buf[0], buf.size() ); + for( std::size_t i = 0; i < count; ++i ) + { + T r = _pick(); + std::cout.write( reinterpret_cast< char* >( &r ), sizeof( T ) ); + } + if( ::csv.flush ) { std::cout.flush(); } + } + return 0; + } + while( std::cin.good() ) + { + std::string s; + std::getline( std::cin, s ); + if( s.empty() ) { continue; } + std::cout << s; + for( std::size_t i = 0; i < count; ++i ) { std::cout << ::csv.delimiter << type_traits< T >::cast( _pick() ); } + std::cout << std::endl; + if( ::csv.flush ) { std::cout.flush(); } + } + return 0; + } + if( binary ) + { + while( std::cout.good() ) + { + for( std::size_t i = 0; i < count; ++i ) + { + T r = _pick(); + std::cout.write( reinterpret_cast< char* >( &r ), sizeof( T ) ); + } + if( ::csv.flush ) { std::cout.flush(); } + } + return 0; + } + while( std::cout.good() ) + { + std::string comma; + for( std::size_t i = 0; i < count; ++i ) + { + std::cout << comma << type_traits< T >::cast( _pick() ); + comma = ::csv.delimiter; + } + std::cout << std::endl; + if( ::csv.flush ) { std::cout.flush(); } // todo? remove? std::endl flushes anyway? + } + return 0; +} + +template < typename T > struct cast_traits { typedef T type; }; +template <> struct cast_traits< char > { typedef int type; }; +template <> struct cast_traits< unsigned char > { typedef unsigned int type; }; + +template < typename T > static std::vector< T > _as( const std::vector< std::string >& v, unsigned int begin ) // todo? move to library? +{ + std::vector< T > r( v.size() - begin ); + for( unsigned int i = begin; i < v.size(); r[ i - begin ] = boost::lexical_cast< typename cast_traits< T >::type >( v[i] ), ++i ); + return r; +} + +template < typename T, template < typename > class Distribution, unsigned int Size > struct distribution_traits { static Distribution< T > make( const std::vector< T >& p ); }; // quick and dirty +template < typename T, template < typename > class Distribution > struct distribution_traits< T, Distribution, 0 > { static Distribution< T > make( const std::vector< T >& p ) { return Distribution< T >(); } }; +template < typename T, template < typename > class Distribution > struct distribution_traits< T, Distribution, 1 > { static Distribution< T > make( const std::vector< T >& p ) { return Distribution< T >( p[0] ); } }; +template < typename T, template < typename > class Distribution > struct distribution_traits< T, Distribution, 2 > { static Distribution< T > make( const std::vector< T >& p ) { return Distribution< T >( p[0], p[1] ); } }; + +template < typename T, template < typename > class Distribution > static Distribution< T > make_distribution( const std::vector< std::string >& params ) // quick and dirty +{ + const auto& p = _as< T >( params, 1 ); + switch( p.size() ) // quick and dirty; does not scale, but for now just to make it working + { + case 0: return distribution_traits< T, Distribution, 0 >::make( p ); + case 1: return distribution_traits< T, Distribution, 1 >::make( p ); + case 2: return distribution_traits< T, Distribution, 2 >::make( p ); + default: COMMA_THROW( comma::exception, "distribution traits for " << p.size() << " parameters: not implemented; just ask" ); + } +} + +template < typename T, template < typename > class Distribution > +static int run_impl( const std::vector< std::string >& params, const comma::command_line_options& options ) +{ + const auto& append = options.exists( "--append" ); + const auto& binary = options.exists( "--output-binary" ) || ::csv.binary(); + const auto& engine = options.value< std::string >( "--engine", "mt19937_64" ); + const auto& count = comma::csv::format( options.value< std::string >( "--type", "ui" ) ).count(); + boost::optional< std::pair< T, T > > range; + if( options.exists( "--range" ) && params[0] != "uniform" ) { range = comma::csv::ascii< std::pair< T, T > >().get( options.value< std::string >( "--range" ) ); } // quick and dirty + Distribution< T > distribution = make_distribution< T, Distribution >( params ); + if( engine == "minstd_rand0" ) { return run_impl< T, Distribution, std::minstd_rand0 >( distribution, append, binary, count, range ); } + if( engine == "minstd_rand" ) { return run_impl< T, Distribution, std::minstd_rand >( distribution, append, binary, count, range ); } + if( engine == "mt19937" ) { return run_impl< T, Distribution, std::mt19937 >( distribution, append, binary, count, range ); } + if( engine == "mt19937_64" ) { return run_impl< T, Distribution, std::mt19937_64 >( distribution, append, binary, count, range ); } + if( engine == "ranlux24_base" ) { return run_impl< T, Distribution, std::ranlux24_base >( distribution, append, binary, count, range ); } + if( engine == "ranlux48_base" ) { return run_impl< T, Distribution, std::ranlux48_base >( distribution, append, binary, count, range ); } + if( engine == "ranlux24" ) { return run_impl< T, Distribution, std::ranlux24 >( distribution, append, binary, count, range ); } + if( engine == "ranlux48" ) { return run_impl< T, Distribution, std::ranlux48 >( distribution, append, binary, count, range ); } + if( engine == "knuth_b" ) { return run_impl< T, Distribution, std::knuth_b >( distribution, append, binary, count, range ); } + if( engine == "default_random_engine" ) { return run_impl< T, Distribution, std::default_random_engine >( distribution, append, binary, count, range ); } + std::cerr << "csv-random make: expected engine; got: '" << engine << "'" << std::endl; + return 1; +} + +static int run( const comma::command_line_options& options ) // quick and dirty +{ + auto params = comma::split( options.value< std::string >( "--distribution", "uniform" ), ',' ); + const std::string& distribution = params[0]; + const auto& format = comma::csv::format( options.value< std::string >( "--type", "ui" ) ); + if ( format.collapsed_string().find( ',' ) != std::string::npos ) { std::cerr << "csv-random make: --type must be homogeneous i.e. ui or 2ui or 3ui" << std::endl; return 1; } + if( distribution == "uniform" ) + { + if( options.exists( "--range" ) ) // super-quick and dirty to preserve backward compatibility + { + if( params.size() > 1 ) { std::cerr << "csv-random make: uniform: either use --range or uniform[,,], not both" << std::endl; return 1; } + const auto& r = comma::split( options.value< std::string >( "--range" ), ',' ); + params = { "uniform", r[0], r[1] }; + } + if( params.size() != 1 && params.size() != 3 ) { std::cerr << "csv-random make: uniform: expected uniform[,,]; got: \"" << options.value< std::string >( "--distribution" ) << "\"" << std::endl; return 1; } + switch( format.offset( 0 ).type ) + { + case csv::format::int8: return run_impl< char, std::uniform_int_distribution >( params, options ); + case csv::format::uint8: return run_impl< unsigned char, std::uniform_int_distribution >( params, options ); + case csv::format::int16: return run_impl< comma::int16, std::uniform_int_distribution >( params, options ); + case csv::format::uint16: return run_impl< comma::uint16, std::uniform_int_distribution >( params, options ); + case csv::format::int32: return run_impl< comma::int32, std::uniform_int_distribution >( params, options ); + case csv::format::uint32: return run_impl< comma::uint32, std::uniform_int_distribution >( params, options ); + case csv::format::int64: return run_impl< comma::int64, std::uniform_int_distribution >( params, options ); + case csv::format::uint64: return run_impl< comma::uint64, std::uniform_int_distribution >( params, options ); + case csv::format::float_t: return run_impl< float, std::uniform_real_distribution >( params, options ); + case csv::format::double_t: return run_impl< double, std::uniform_real_distribution >( params, options ); + default: std::cerr << "csv-random make: uniform distribution: expected type; got: '" << format.string() << "'" << std::endl; return 1; + } + } + if( distribution == "gaussian" || distribution == "normal" ) + { + if( params.size() != 1 && params.size() != 3 ) { std::cerr << "csv-random make: gaussian: expected gaussian[,,]; got: \"" << options.value< std::string >( "--distribution" ) << "\"" << std::endl; return 1; } + switch( format.offset( 0 ).type ) + { + case csv::format::float_t: return run_impl< float, std::normal_distribution >( params, options ); + case csv::format::double_t: return run_impl< double, std::normal_distribution >( params, options ); + default: std::cerr << "csv-random make: normal distribution: expected floating point --type; got unsupported type: '" << format.string() << "'" << std::endl; return 1; + } + } + std::cerr << "csv-random make: expected distribution; got: '" << distribution << "'" << std::endl; + return 1; +} + +} // namespace make { + +namespace shuffle { + +template < typename Engine > static int run_impl( const comma::command_line_options& options, bool sample = false ) +{ + auto engine = ::seed ? Engine( *::seed ) : Engine(); + std::deque< std::string > records; + std::vector< unsigned int > indices; // quick and dirty + unsigned int size = options.value( "--size", 1 ); // quick and dirty + auto ratio = options.optional< float >( "--ratio" ); // quick and dirty + auto sliding_window = options.optional< unsigned int >( "--sliding-window,--window" ); + if( sliding_window ) { std::cerr << "csv-random shuffle: --sliding-window: todo" << std::endl; return 1; } + if( sliding_window ) { std::cerr << "csv-random shuffle: expected either block field or --sliding-window; got both" << std::endl; return 1; } + comma::csv::input_stream< input > is( std::cin, ::csv ); + bool has_block = ::csv.has_field( "block" ); + comma::uint32 block{0}; + while( is.ready() || std::cin.good() ) + { + const input* p = is.read(); + if( !p || ( has_block && p->block != block ) ) + { + if( !records.empty() ) + { + //std::uniform_int_distribution< int > distribution( 0, records.size() - 1 ); // quick and dirty + indices.resize( records.size() ); // quick and dirty + for( unsigned int i = 0; i < indices.size(); ++i ) { indices[i] = i; } + // deprecated: std::random_shuffle( indices.begin(), indices.end(), [&]( int ) -> int { return distribution( engine ); } ); // quick and dirty, watch performance + std::shuffle( indices.begin(), indices.end(), engine ); + unsigned int s = sample ? ( ratio ? int( records.size() * *ratio ) : size ) : records.size(); + if( sample ) { std::sort( indices.begin(), indices.begin() + s ); } // quick and dirty + for( unsigned int i = 0; i < s; ++i ) { std::cout.write( &records[indices[i]][0], records[indices[i]].size() ); } + records.clear(); + if( ::csv.flush ) { std::cout.flush(); } + } + if( p ) { block = p->block; } + } + if( !p ) { break; } + if( ::csv.binary() ) + { + records.emplace_back(); + records.back().resize( ::csv.format().size() ); + std::memcpy( &records.back()[0], is.binary().last(), ::csv.format().size() ); + } + else + { + records.push_back( comma::join( is.ascii().last(), ::csv.delimiter ) + "\n" ); + } + } + return 0; +} + +static int run( const comma::command_line_options& options, bool sample = false ) +{ + const auto& engine = options.value< std::string >( "--engine", "mt19937_64" ); + if( engine == "minstd_rand0" ) { return run_impl< std::minstd_rand0 >( options, sample ); } + if( engine == "minstd_rand" ) { return run_impl< std::minstd_rand >( options, sample ); } + if( engine == "mt19937" ) { return run_impl< std::mt19937>( options, sample ); } + if( engine == "mt19937_64" ) { return run_impl< std::mt19937_64 >( options, sample ); } + if( engine == "ranlux24_base" ) { return run_impl< std::ranlux24_base >( options, sample ); } + if( engine == "ranlux48_base" ) { return run_impl< std::ranlux48_base >( options, sample ); } + if( engine == "ranlux24" ) { return run_impl< std::ranlux24 >( options, sample ); } + if( engine == "ranlux48" ) { return run_impl< std::ranlux48 >( options, sample ); } + if( engine == "knuth_b" ) { return run_impl< std::knuth_b >( options, sample ); } + if( engine == "default_random_engine" ) { return run_impl< std::default_random_engine >( options, sample ); } + std::cerr << "csv-random " << ( sample ? "sample" : "shuffle" ) << ": expected engine; got: '" << engine << "'" << std::endl; + return 1; +} + +} // namespace shuffle { + +namespace sample { + +static int run( const comma::command_line_options& options ) { return shuffle::run( options, true ); } // quick and relatively dirty for now + +} // namespace sample { + +namespace true_random { + +template < typename T > +static int run_impl( const comma::command_line_options& options, std::size_t count ) +{ + std::random_device rd; + const bool binary = options.exists( "--output-binary" ) || ::csv.binary(); + const bool flush = options.exists( "--flush" ) || ::csv.flush; + typedef std::pair< double, double > pair_t; + boost::optional< pair_t > range = comma::silent_none< pair_t >(); + double factor{1}; + if( options.exists( "--range" ) ) + { + range = comma::csv::ascii< pair_t >().get( options.value< std::string >( "--range" ) ); + factor = ( range->second - range->first ) / ( rd.max() - rd.min() ); + } + auto output_line_to_stdout = [&]( std::string&& initial_delimiter ) + { + for( std::size_t i = 0; i < count; ++i ) + { + T r = rd(); + if( range ) { r = range->first + ( r - rd.min() ) * factor; } + if( binary ) { std::cout.write( reinterpret_cast< const char* >( &r ), sizeof( T ) ); } + else { std::cout << initial_delimiter << type_traits< T >::cast( r ); initial_delimiter = ::csv.delimiter; } + } + if( !binary ) { std::cout << std::endl; } + if( flush ) { std::cout << std::flush; } + }; + if( options.exists( "--append" ) ) + { + while( std::cin.good() ) + { + auto buf = ::csv.binary() ? std::string( ::csv.format().size(), {} ) : std::string{}; + if( ::csv.binary() ) + { + std::cin.read( &buf[0], buf.size() ); + if( std::cin.gcount() == 0 ) { return 0; } + if( std::cin.gcount() != static_cast< int >( buf.size() ) ) { std::cerr << "csv-random true-random: expected " << buf.size() << " bytes; got " << std::cin.gcount() << std::endl; return 1; } + } + else + { + std::getline( std::cin, buf ); + if( buf.empty() ) { continue; } + } + std::cout.write( &buf[0], buf.size() ); + output_line_to_stdout( { ::csv.delimiter } ); + } + } + else + { + while( std::cout.good() ) + { + output_line_to_stdout( {} ); + if( options.exists( "--once" ) ) { break; } + } + } + return 0; +} + +static int run( const comma::command_line_options& options ) +{ + const auto format = comma::csv::format( options.value< std::string >( "--type", "ui" ) ); + if( format.collapsed_string().find( ',' ) != std::string::npos ) { std::cerr << "csv-random true-random: --type must be homogeneous i.e. ui or 2ui or 3ui" << std::endl; return 1; } + switch( format.offset( 0 ).type ) { + case csv::format::int8: return run_impl< char >( options, format.count() ); + case csv::format::uint8: return run_impl< unsigned char >( options, format.count() ); + case csv::format::int16: return run_impl< comma::int16 >( options, format.count() ); + case csv::format::uint16: return run_impl< comma::uint16 >( options, format.count() ); + case csv::format::int32: return run_impl< comma::int32 >( options, format.count() ); + case csv::format::uint32: return run_impl< comma::uint32 >( options, format.count() ); + case csv::format::int64: return run_impl< comma::int64 >( options, format.count() ); + case csv::format::uint64: return run_impl< comma::uint64 >( options, format.count() ); + case csv::format::float_t: return run_impl< float >( options, format.count() ); + case csv::format::double_t: return run_impl< double >( options, format.count() ); + default: std::cerr << "csv-random true-random: expected type; got: '" << format.string() << "'" << std::endl; + } + return 1; +} + +} // namespace true_random { + +} } } // namespace comma { namespace applications { namespace random { + +int main( int ac, char** av ) +{ + try + { + comma::command_line_options options( ac, av, usage ); + const auto& unnamed = options.unnamed( "--append,--flush,--verbose,-v", "-.*" ); + if( unnamed.empty() ) { std::cerr << "csv-random: please specify operation" << std::endl; return 1; } + ::csv = comma::csv::options( options ); + std::cout.precision( ::csv.precision ); + ::seed = options.optional< comma::uint32 >( "--seed" ); + ::verbose = options.exists( "--verbose,-v" ); + std::string operation = unnamed[0]; + if( operation == "make" || operation == "pseudo-random" ) { return comma::applications::random::make::run( options ); } + if( operation == "sample" ) { return comma::applications::random::sample::run( options ); } + if( operation == "shuffle" ) { return comma::applications::random::shuffle::run( options ); } + if( operation == "true-random" ) { return comma::applications::random::true_random::run( options ); } + std::cerr << "csv-random: expected operation; got: '" << operation << "'" << std::endl; + return 1; + } + catch( std::exception& ex ) { std::cerr << "csv-random: " << ex.what() << std::endl; } + catch( ... ) { std::cerr << "csv-random: unknown exception" << std::endl; } + return 1; +} diff --git a/csv/applications/csv-repeat.cpp b/csv/applications/csv-repeat.cpp index a6fe24e6d..5401c1173 100644 --- a/csv/applications/csv-repeat.cpp +++ b/csv/applications/csv-repeat.cpp @@ -29,11 +29,11 @@ /// @author dave jennings +#include #include #include #include #include "../../application/command_line_options.h" -#include "../../application/contact_info.h" #include "../../application/signal_flag.h" #include "../../csv/options.h" #include "../../csv/stream.h" @@ -73,6 +73,11 @@ void usage( bool verbose = false ) std::cerr << " warning: currently is very simplistic; see todo comments in the code to make it more robust" << std::endl; std::cerr << " --period=[]: period of repeated record" << std::endl; std::cerr << " --timeout,-t=[]: timeout before repeating the last record; if not specified, timeout is set to --period" << std::endl; + std::cerr << " --timestamped: use input timestamp for repeating; currently, would do blocking read" << std::endl; + std::cerr << " convenient for filling holes in data in offline processing" << std::endl; + std::cerr << " --timestamped options" << std::endl; + std::cerr << " --at-least-from,--from=[