1
0
mirror of https://git.suyu.dev/suyu/suyu synced 2025-09-14 18:17:59 -05:00

Initial commit

This commit is contained in:
Crimson-Hawk
2024-03-05 16:42:40 +08:00
commit f1e4595ebf
39576 changed files with 7006612 additions and 0 deletions

View File

View File

View File

@@ -0,0 +1,38 @@
# Use, modification, and distribution are
# subject to the Boost Software License, Version 1.0. (See accompanying
# file LICENSE.txt)
#
# Copyright Rene Rivera 2020.
# For Drone CI we use the Starlark scripting language to reduce duplication.
# As the yaml syntax for Drone CI is rather limited.
#
#
globalenv={'B2_VARIANT': 'variant=release,debug'}
linuxglobalimage="cppalliance/droneubuntu1604:1"
windowsglobalimage="cppalliance/dronevs2019"
def main(ctx):
return [
linux_cxx("B2_TOOLSET=gcc-4.8 B2_CXXSTD=03,11 Job 0", "g++-4.8", packages="g++-4.8", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'gcc-4.8', 'B2_CXXSTD': '03,11', 'DRONE_JOB_UUID': 'b6589fc6ab'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=gcc-5 B2_CXXSTD=11 Job 1", "g++-5", packages="g++-5", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'gcc-5', 'B2_CXXSTD': '11', 'DRONE_JOB_UUID': '356a192b79'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=gcc-6 B2_CXXSTD=11,14 Job 2", "g++-6", packages="g++-6", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'gcc-6', 'B2_CXXSTD': '11,14', 'DRONE_JOB_UUID': 'da4b9237ba'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=gcc-7 B2_CXXSTD=11,14,17 Job 3", "g++-7", packages="g++-7", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'gcc-7', 'B2_CXXSTD': '11,14,17', 'DRONE_JOB_UUID': '77de68daec'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=gcc-8 B2_CXXSTD=14,17,2a Job 4", "g++-8", packages="g++-8", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'gcc-8', 'B2_CXXSTD': '14,17,2a', 'DRONE_JOB_UUID': '1b64538924'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=clang-3.8 B2_CXXSTD=03,11,14 Job 5", "clang++-3.8", packages="clang-3.8 libstdc++-6-dev", llvm_os="xenial", llvm_ver="3.8", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'clang-3.8', 'B2_CXXSTD': '03,11,14', 'DRONE_JOB_UUID': 'ac3478d69a'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=clang-4.0 B2_CXXSTD=11,14 Job 6", "clang++-4.0", packages="clang-4.0 libstdc++-6-dev", llvm_os="xenial", llvm_ver="4.0", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'clang-4.0', 'B2_CXXSTD': '11,14', 'DRONE_JOB_UUID': 'c1dfd96eea'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=clang-5.0 B2_CXXSTD=11,14,17 Job 7", "clang++-5.0", packages="clang-5.0 libstdc++-7-dev", llvm_os="xenial", llvm_ver="5.0", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'clang-5.0', 'B2_CXXSTD': '11,14,17', 'DRONE_JOB_UUID': '902ba3cda1'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=clang-6.0 B2_CXXSTD=14,17,2a Job 8", "clang++-6.0", packages="clang-6.0 libc6-dbg libc++-dev libstdc++-8-dev", llvm_os="xenial", llvm_ver="6.0", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'clang-6.0', 'B2_CXXSTD': '14,17,2a', 'DRONE_JOB_UUID': 'fe5dbbcea5'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=clang-7 B2_CXXSTD=14,17,2a Job 9", "clang++-7", packages="clang-7 libc6-dbg libc++-dev libstdc++-8-dev", llvm_os="xenial", llvm_ver="7", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'clang-7', 'B2_CXXSTD': '14,17,2a', 'DRONE_JOB_UUID': '0ade7c2cf9'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=clang-8 B2_CXXSTD=14,17,2a Job 10", "clang++-8", packages="clang-8 libc6-dbg libc++-dev libstdc++-8-dev", llvm_os="xenial", llvm_ver="8", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'clang-8', 'B2_CXXSTD': '14,17,2a', 'DRONE_JOB_UUID': 'b1d5781111'}, globalenv=globalenv),
linux_cxx("B2_TOOLSET=clang-6.0 B2_CXXSTD=03,11,14,17,2a Job 11", "clang++-6.0", packages="clang-6.0 libc6-dbg libc++-dev libstdc++-8-dev", llvm_os="xenial", llvm_ver="6.0", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'B2_TOOLSET': 'clang-6.0', 'B2_CXXSTD': '03,11,14,17,2a', 'B2_CXXFLAGS': '-stdlib=libc++', 'DRONE_JOB_UUID': '17ba079149'}, globalenv=globalenv),
osx_cxx("B2_TOOLSET=clang B2_CXXSTD=03,11,17 Job 12", "g++", packages="", buildtype="boost", buildscript="drone", environment={'B2_TOOLSET': 'clang', 'B2_CXXSTD': '03,11,17', 'DRONE_JOB_UUID': '7b52009b64'}, globalenv=globalenv),
linux_cxx("COMMENT=codecov.io B2_TOOLSET=gcc-8 B2_DEFINE Job 13", "g++-8", packages="g++-8", buildtype="b5847f804b-bbb3de2b00", buildscript="drone", image=linuxglobalimage, environment={'COMMENT': 'codecov.io', 'B2_TOOLSET': 'gcc-8', 'B2_DEFINES': 'define=BOOST_NO_STRESS_TEST=1', 'DRONE_JOB_UUID': 'bd307a3ec3'}, globalenv=globalenv),
linux_cxx("COMMENT=cppcheck Job 14", "g++", packages="binutils-gold gdb libc6-dbg", buildtype="b5847f804b-ed45733e6c", buildscript="drone", image=linuxglobalimage, environment={'COMMENT': 'cppcheck', 'DRONE_JOB_UUID': 'fa35e19212'}, globalenv=globalenv),
linux_cxx("COMMENT=ubsan B2_VARIANT=variant=debug B2_TOO Job 15", "g++-8", packages="g++-8", buildtype="boost", buildscript="drone", image=linuxglobalimage, environment={'COMMENT': 'ubsan', 'B2_VARIANT': 'variant=debug', 'B2_TOOLSET': 'gcc-8', 'B2_CXXSTD': '03,11,14,17,2a', 'B2_DEFINES': 'define=BOOST_NO_STRESS_TEST=1', 'B2_CXXFLAGS': 'cxxflags=-fno-omit-frame-pointer cxxflags=-fsanitize=undefined cxxflags=-fno-sanitize-recover=undefined', 'B2_LINKFLAGS': 'linkflags=-fsanitize=undefined linkflags=-fno-sanitize-recover=undefined linkflags=-fuse-ld=gold', 'UBSAN_OPTIONS': 'print_stacktrace=1', 'DRONE_JOB_UUID': 'f1abd67035'}, globalenv=globalenv),
linux_cxx("COMMENT=valgrind B2_TOOLSET=clang-6.0 B2_CXXS Job 16", "clang++-6.0", packages="clang-6.0 libc6-dbg libc++-dev libstdc++-8-dev", llvm_os="xenial", llvm_ver="6.0", buildtype="b5847f804b-db180b7bd2", buildscript="drone", image=linuxglobalimage, environment={'COMMENT': 'valgrind', 'B2_TOOLSET': 'clang-6.0', 'B2_CXXSTD': '03,11,14,17,2a', 'B2_DEFINES': 'define=BOOST_NO_STRESS_TEST=1', 'B2_VARIANT': 'variant=debug', 'B2_TESTFLAGS': 'testing.launcher=valgrind', 'VALGRIND_OPTS': '--error-exitcode=1', 'DRONE_JOB_UUID': '1574bddb75'}, globalenv=globalenv),
linux_cxx("COMMENT=Coverity Scan B2_TOOLSET=clang Job 17", "g++", packages="binutils-gold gdb libc6-dbg", buildtype="b5847f804b-cce9827eb5", buildscript="drone", image=linuxglobalimage, environment={'COMMENT': 'Coverity Scan', 'B2_TOOLSET': 'clang', 'DRONE_JOB_UUID': '0716d9708d'}, globalenv=globalenv),
]
# from https://github.com/boostorg/boost-ci
load("@boost_ci//ci/drone/:functions.star", "linux_cxx","windows_cxx","osx_cxx","freebsd_cxx")

View File

@@ -0,0 +1,129 @@
#!/bin/bash
# Copyright 2020 Rene Rivera, Sam Darwin
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE.txt or copy at http://boost.org/LICENSE_1_0.txt)
set -e
export TRAVIS_BUILD_DIR=$(pwd)
export DRONE_BUILD_DIR=$(pwd)
export TRAVIS_BRANCH=$DRONE_BRANCH
export VCS_COMMIT_ID=$DRONE_COMMIT
export GIT_COMMIT=$DRONE_COMMIT
export REPO_NAME=$DRONE_REPO
export PATH=~/.local/bin:/usr/local/bin:$PATH
if [ "$DRONE_JOB_BUILDTYPE" == "boost" ]; then
echo '==================================> INSTALL'
git clone https://github.com/boostorg/boost-ci.git boost-ci
cp -pr boost-ci/ci boost-ci/.codecov.yml .
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
unset -f cd
fi
export SELF=`basename $REPO_NAME`
export BOOST_CI_TARGET_BRANCH="$TRAVIS_BRANCH"
export BOOST_CI_SRC_FOLDER=$(pwd)
. ./ci/common_install.sh
echo '==================================> SCRIPT'
cd $BOOST_ROOT/libs/$SELF
ci/travis/build.sh
elif [ "$DRONE_JOB_BUILDTYPE" == "b5847f804b-bbb3de2b00" ]; then
echo '==================================> INSTALL'
git clone https://github.com/boostorg/boost-ci.git boost-ci
cp -pr boost-ci/ci boost-ci/.codecov.yml .
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
unset -f cd
fi
export SELF=`basename $REPO_NAME`
export BOOST_CI_TARGET_BRANCH="$TRAVIS_BRANCH"
export BOOST_CI_SRC_FOLDER=$(pwd)
. ./ci/common_install.sh
echo '==================================> SCRIPT'
pushd /tmp && git clone https://github.com/linux-test-project/lcov.git && export PATH=/tmp/lcov/bin:$PATH && which lcov && lcov --version && popd
cd $BOOST_ROOT/libs/$SELF
ci/travis/codecov.sh
elif [ "$DRONE_JOB_BUILDTYPE" == "b5847f804b-ed45733e6c" ]; then
echo '==================================> INSTALL'
git clone https://github.com/boostorg/boost-ci.git boost-ci
cp -pr boost-ci/ci boost-ci/.codecov.yml .
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
unset -f cd
fi
export SELF=`basename $REPO_NAME`
export BOOST_CI_TARGET_BRANCH="$TRAVIS_BRANCH"
export BOOST_CI_SRC_FOLDER=$(pwd)
. ./ci/common_install.sh
echo '==================================> SCRIPT'
cd $BOOST_ROOT/libs/$SELF
ci/travis/cppcheck.sh
elif [ "$DRONE_JOB_BUILDTYPE" == "b5847f804b-db180b7bd2" ]; then
echo '==================================> INSTALL'
git clone https://github.com/boostorg/boost-ci.git boost-ci
cp -pr boost-ci/ci boost-ci/.codecov.yml .
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
unset -f cd
fi
export SELF=`basename $REPO_NAME`
export BOOST_CI_TARGET_BRANCH="$TRAVIS_BRANCH"
export BOOST_CI_SRC_FOLDER=$(pwd)
. ./ci/common_install.sh
echo '==================================> SCRIPT'
cd $BOOST_ROOT/libs/$SELF
ci/travis/valgrind.sh
elif [ "$DRONE_JOB_BUILDTYPE" == "b5847f804b-cce9827eb5" ]; then
echo '==================================> INSTALL'
git clone https://github.com/boostorg/boost-ci.git boost-ci
cp -pr boost-ci/ci boost-ci/.codecov.yml .
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
unset -f cd
fi
export SELF=`basename $REPO_NAME`
export BOOST_CI_TARGET_BRANCH="$TRAVIS_BRANCH"
export BOOST_CI_SRC_FOLDER=$(pwd)
. ./ci/common_install.sh
echo '==================================> SCRIPT'
if [ -n "${COVERITY_SCAN_NOTIFICATION_EMAIL}" -a \( "$DRONE_BRANCH" = "develop" -o "$DRONE_BRANCH" = "master" \) -a "$DRONE_BUILD_EVENT" = "push" ] ; then
cd $BOOST_ROOT/libs/$SELF
ci/travis/coverity.sh
fi
fi

View File

@@ -0,0 +1,96 @@
* text=auto !eol svneol=native#text/plain
*.gitattributes text svneol=native#text/plain
# Scriptish formats
*.bat text svneol=native#text/plain
*.bsh text svneol=native#text/x-beanshell
*.cgi text svneol=native#text/plain
*.cmd text svneol=native#text/plain
*.js text svneol=native#text/javascript
*.php text svneol=native#text/x-php
*.pl text svneol=native#text/x-perl
*.pm text svneol=native#text/x-perl
*.py text svneol=native#text/x-python
*.sh eol=lf svneol=LF#text/x-sh
configure eol=lf svneol=LF#text/x-sh
# Image formats
*.bmp binary svneol=unset#image/bmp
*.gif binary svneol=unset#image/gif
*.ico binary svneol=unset#image/ico
*.jpeg binary svneol=unset#image/jpeg
*.jpg binary svneol=unset#image/jpeg
*.png binary svneol=unset#image/png
*.tif binary svneol=unset#image/tiff
*.tiff binary svneol=unset#image/tiff
*.svg text svneol=native#image/svg%2Bxml
# Data formats
*.pdf binary svneol=unset#application/pdf
*.avi binary svneol=unset#video/avi
*.doc binary svneol=unset#application/msword
*.dsp text svneol=crlf#text/plain
*.dsw text svneol=crlf#text/plain
*.eps binary svneol=unset#application/postscript
*.gz binary svneol=unset#application/gzip
*.mov binary svneol=unset#video/quicktime
*.mp3 binary svneol=unset#audio/mpeg
*.ppt binary svneol=unset#application/vnd.ms-powerpoint
*.ps binary svneol=unset#application/postscript
*.psd binary svneol=unset#application/photoshop
*.rdf binary svneol=unset#text/rdf
*.rss text svneol=unset#text/xml
*.rtf binary svneol=unset#text/rtf
*.sln text svneol=native#text/plain
*.swf binary svneol=unset#application/x-shockwave-flash
*.tgz binary svneol=unset#application/gzip
*.vcproj text svneol=native#text/xml
*.vcxproj text svneol=native#text/xml
*.vsprops text svneol=native#text/xml
*.wav binary svneol=unset#audio/wav
*.xls binary svneol=unset#application/vnd.ms-excel
*.zip binary svneol=unset#application/zip
# Text formats
.htaccess text svneol=native#text/plain
*.bbk text svneol=native#text/xml
*.cmake text svneol=native#text/plain
*.css text svneol=native#text/css
*.dtd text svneol=native#text/xml
*.htm text svneol=native#text/html
*.html text svneol=native#text/html
*.ini text svneol=native#text/plain
*.log text svneol=native#text/plain
*.mak text svneol=native#text/plain
*.qbk text svneol=native#text/plain
*.rst text svneol=native#text/plain
*.sql text svneol=native#text/x-sql
*.txt text svneol=native#text/plain
*.xhtml text svneol=native#text/xhtml%2Bxml
*.xml text svneol=native#text/xml
*.xsd text svneol=native#text/xml
*.xsl text svneol=native#text/xml
*.xslt text svneol=native#text/xml
*.xul text svneol=native#text/xul
*.yml text svneol=native#text/plain
boost-no-inspect text svneol=native#text/plain
CHANGES text svneol=native#text/plain
COPYING text svneol=native#text/plain
INSTALL text svneol=native#text/plain
Jamfile text svneol=native#text/plain
Jamroot text svneol=native#text/plain
Jamfile.v2 text svneol=native#text/plain
Jamrules text svneol=native#text/plain
Makefile* text svneol=native#text/plain
README text svneol=native#text/plain
TODO text svneol=native#text/plain
# Code formats
*.c text svneol=native#text/plain
*.cpp text svneol=native#text/plain
*.h text svneol=native#text/plain
*.hpp text svneol=native#text/plain
*.ipp text svneol=native#text/plain
*.tpp text svneol=native#text/plain
*.jam text svneol=native#text/plain
*.java text svneol=native#text/plain

View File

@@ -0,0 +1,185 @@
# Copyright 2016 Peter Dimov
# Copyright 2017 - 2019 James E. King III
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at http://boost.org/LICENSE_1_0.txt)
#
# Generic Travis CI build script for boostorg repositories
# See: https://github.com/boostorg/boost-ci
#
# Instructions for customizing this script for your library:
#
# 1. Customize the compilers and language levels you want in the 'jobs'.
# 2. If you have more than include/, src/, test/, example/, examples/, or
# tools/ directories, modify your Travis CI project and add the environment
# variable DEPINST. For example if your build uses code in "bench/" and
# "fog/" directories, then set DEPINST to the following:
# --include bench --include fog
# 3. If you want to enable Coverity Scan, you need to provide the environment
# variables COVERITY_SCAN_TOKEN and COVERITY_SCAN_NOTIFICATION_EMAIL in
# your github settings.
# 4. Enable pull request builds in your boostorg/<library> account.
#
# That's it - the scripts will do everything else for you.
dist: xenial
language: cpp
env:
global:
# see: http://www.boost.org/build/doc/html/bbv2/overview/invocation.html#bbv2.overview.invocation.properties
# to use the default for a given environment, comment it out; recommend you build debug and release however..
# - B2_ADDRESS_MODEL=address-model=64,32
# - B2_LINK=link=shared,static
# - B2_THREADING=threading=multi,single
- B2_VARIANT=variant=release,debug
install:
- git clone https://github.com/boostorg/boost-ci.git boost-ci
- cp -pr boost-ci/ci boost-ci/.codecov.yml .
- source ci/travis/install.sh
addons:
apt:
packages:
- binutils-gold
- gdb
- libc6-dbg
branches:
only:
- develop
- master
script:
- cd $BOOST_ROOT/libs/$SELF
- ci/travis/build.sh
#
# Default toolsets in Ubuntu
#
# trusty xenial bionic
# 14.04 16.04 18.04
# ------ ------ ------
# clang 3.4 3.8 6.0
# gcc 4.8.2 5.3.1 7.3.0
#
anchors:
clang-38: &clang-38 { apt: { packages: [ "clang-3.8",
"libstdc++-6-dev" ], sources: [ "llvm-toolchain-xenial-3.8",
"ubuntu-toolchain-r-test" ] } }
clang-4: &clang-4 { apt: { packages: [ "clang-4.0",
"libstdc++-6-dev" ], sources: [ "llvm-toolchain-xenial-4.0",
"ubuntu-toolchain-r-test" ] } }
clang-5: &clang-5 { apt: { packages: [ "clang-5.0",
"libstdc++-7-dev" ], sources: [ "llvm-toolchain-xenial-5.0",
"ubuntu-toolchain-r-test" ] } }
clang-6: &clang-6 { apt: { packages: [ "clang-6.0",
"libc6-dbg",
"libc++-dev",
"libstdc++-8-dev" ], sources: [ "llvm-toolchain-xenial-6.0",
"ubuntu-toolchain-r-test" ] } }
clang-7: &clang-7 { apt: { packages: [ "clang-7",
"libc6-dbg",
"libc++-dev",
"libstdc++-8-dev" ], sources: [ "llvm-toolchain-xenial-7",
"ubuntu-toolchain-r-test" ] } }
clang-8: &clang-8 { apt: { packages: [ "clang-8",
"libc6-dbg",
"libc++-dev",
"libstdc++-8-dev" ], sources: [ "llvm-toolchain-xenial-8",
"ubuntu-toolchain-r-test" ] } }
gcc-48: &gcc-48 { apt: { packages: [ "g++-4.8" ] } }
gcc-5: &gcc-5 { apt: { packages: [ "g++-5" ] } }
gcc-6: &gcc-6 { apt: { packages: [ "g++-6" ], sources: [ "ubuntu-toolchain-r-test" ] } }
gcc-7: &gcc-7 { apt: { packages: [ "g++-7" ], sources: [ "ubuntu-toolchain-r-test" ] } }
gcc-8: &gcc-8 { apt: { packages: [ "g++-8" ], sources: [ "ubuntu-toolchain-r-test" ] } }
jobs:
allow_failures:
- env:
- COPY="all the environment settings from your job"
include:
# libstdc++
- { os: "linux", env: [ "B2_TOOLSET=gcc-4.8", "B2_CXXSTD=03,11" ], addons: *gcc-48 }
- { os: "linux", env: [ "B2_TOOLSET=gcc-5", "B2_CXXSTD=11" ], addons: *gcc-5 }
- { os: "linux", env: [ "B2_TOOLSET=gcc-6", "B2_CXXSTD=11,14" ], addons: *gcc-6 }
- { os: "linux", env: [ "B2_TOOLSET=gcc-7", "B2_CXXSTD=11,14,17" ], addons: *gcc-7 }
- { os: "linux", env: [ "B2_TOOLSET=gcc-8", "B2_CXXSTD=14,17,2a" ], addons: *gcc-8 }
- { os: "linux", env: [ "B2_TOOLSET=clang-3.8", "B2_CXXSTD=03,11,14" ], addons: *clang-38 }
- { os: "linux", env: [ "B2_TOOLSET=clang-4.0", "B2_CXXSTD=11,14" ], addons: *clang-4 }
- { os: "linux", env: [ "B2_TOOLSET=clang-5.0", "B2_CXXSTD=11,14,17" ], addons: *clang-5 }
- { os: "linux", env: [ "B2_TOOLSET=clang-6.0", "B2_CXXSTD=14,17,2a" ], addons: *clang-6 }
- { os: "linux", env: [ "B2_TOOLSET=clang-7", "B2_CXXSTD=14,17,2a" ], addons: *clang-7 }
- { os: "linux", env: [ "B2_TOOLSET=clang-8", "B2_CXXSTD=14,17,2a" ], addons: *clang-8 }
# libc++
- { os: "linux", env: [ "B2_TOOLSET=clang-6.0", "B2_CXXSTD=03,11,14,17,2a",
"B2_CXXFLAGS=-stdlib=libc++" ], addons: *clang-6 }
- { os: "osx" , env: [ "B2_TOOLSET=clang", "B2_CXXSTD=03,11,17" ] }
# to enable Intel ICC define INTEL_ICC_SERIAL_NUMBER and the following:
# - { os: "linux", env: [ "B2_TOOLSET=intel-linux", "B2_CXXSTD=11,14,17" ], addons: *gcc-7,
# script: cd $BOOST_ROOT/libs/$SELF && ci/travis/intelicc.sh }
- os: linux
env:
- COMMENT=codecov.io
- B2_TOOLSET=gcc-8
- B2_DEFINES="define=BOOST_NO_STRESS_TEST=1"
addons: *gcc-8
script:
- pushd /tmp && git clone https://github.com/linux-test-project/lcov.git && export PATH=/tmp/lcov/bin:$PATH && which lcov && lcov --version && popd
- cd $BOOST_ROOT/libs/$SELF
- ci/travis/codecov.sh
- os: linux
env:
- COMMENT=cppcheck
script:
- cd $BOOST_ROOT/libs/$SELF
- ci/travis/cppcheck.sh
- os: linux
env:
- COMMENT=ubsan
- B2_VARIANT=variant=debug
- B2_TOOLSET=gcc-8
- B2_CXXSTD=03,11,14,17,2a
- B2_DEFINES="define=BOOST_NO_STRESS_TEST=1"
- B2_CXXFLAGS="cxxflags=-fno-omit-frame-pointer cxxflags=-fsanitize=undefined cxxflags=-fno-sanitize-recover=undefined"
- B2_LINKFLAGS="linkflags=-fsanitize=undefined linkflags=-fno-sanitize-recover=undefined linkflags=-fuse-ld=gold"
- UBSAN_OPTIONS=print_stacktrace=1
addons: *gcc-8
- os: linux
env:
- COMMENT=valgrind
- B2_TOOLSET=clang-6.0
- B2_CXXSTD=03,11,14,17,2a
- B2_DEFINES="define=BOOST_NO_STRESS_TEST=1"
- B2_VARIANT=variant=debug
- B2_TESTFLAGS=testing.launcher=valgrind
- VALGRIND_OPTS=--error-exitcode=1
addons: *clang-6
script:
- cd $BOOST_ROOT/libs/$SELF
- ci/travis/valgrind.sh
#################### Jobs to run on pushes to master, develop ###################
# Coverity Scan
- os: linux
if: (env(COVERITY_SCAN_NOTIFICATION_EMAIL) IS present) AND (branch IN (develop, master)) AND (type IN (cron, push))
env:
- COMMENT="Coverity Scan"
- B2_TOOLSET=clang
script:
- cd $BOOST_ROOT/libs/$SELF
- ci/travis/coverity.sh
notifications:
email:
false

View File

@@ -0,0 +1,30 @@
# Generated by `boostdep --cmake tokenizer`
# Copyright 2020 Peter Dimov
# Distributed under the Boost Software License, Version 1.0.
# https://www.boost.org/LICENSE_1_0.txt
cmake_minimum_required(VERSION 3.5...3.16)
project(boost_tokenizer VERSION "${BOOST_SUPERPROJECT_VERSION}" LANGUAGES CXX)
add_library(boost_tokenizer INTERFACE)
add_library(Boost::tokenizer ALIAS boost_tokenizer)
target_include_directories(boost_tokenizer INTERFACE include)
target_link_libraries(boost_tokenizer
INTERFACE
Boost::assert
Boost::config
Boost::iterator
Boost::mpl
Boost::throw_exception
Boost::type_traits
)
if(BUILD_TESTING AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/CMakeLists.txt")
add_subdirectory(test)
endif()

View File

@@ -0,0 +1,11 @@
# Boost.Tokenizer Library Jamfile
#
# Copyright (c) 2018 James E. King III
#
# Use, modification, and distribution are subject to the
# Boost Software License, Version 1.0. (See accompanying file
# LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
# please order by name to ease maintenance
build-project example ;
build-project test ;

View File

@@ -0,0 +1,23 @@
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

View File

@@ -0,0 +1,94 @@
# [Boost.Tokenizer](http://boost.org/libs/tokenizer)
Boost.Tokenizer is a part of [Boost C++ Libraries](http://github.com/boostorg). The Boost.Tokenizer package provides a flexible and easy-to-use way to break a string or other character sequence into a series of tokens.
## License
Distributed under the [Boost Software License, Version 1.0](http://www.boost.org/LICENSE_1_0.txt).
## Properties
* C++03
* Header-Only
## Build Status
Branch | Travis | Appveyor | Coverity Scan | codecov.io | Deps | Docs | Tests |
:-------------: | ------ | -------- | ------------- | ---------- | ---- | ---- | ----- |
[`master`](https://github.com/boostorg/tokenizer/tree/master) | [![Build Status](https://travis-ci.org/boostorg/tokenizer.svg?branch=master)](https://travis-ci.org/boostorg/tokenizer) | [![Build status](https://ci.appveyor.com/api/projects/status/FIXME-vc81nhd5i2f6hi8y/branch/master?svg=true)](https://ci.appveyor.com/project/jeking3/tokenizer-c6pnd/branch/master) | [![Coverity Scan Build Status](https://scan.coverity.com/projects/15854/badge.svg)](https://scan.coverity.com/projects/boostorg-tokenizer) | [![codecov](https://codecov.io/gh/boostorg/tokenizer/branch/master/graph/badge.svg)](https://codecov.io/gh/boostorg/tokenizer/branch/master)| [![Deps](https://img.shields.io/badge/deps-master-brightgreen.svg)](https://pdimov.github.io/boostdep-report/master/tokenizer.html) | [![Documentation](https://img.shields.io/badge/docs-master-brightgreen.svg)](http://www.boost.org/doc/libs/master/doc/html/tokenizer.html) | [![Enter the Matrix](https://img.shields.io/badge/matrix-master-brightgreen.svg)](http://www.boost.org/development/tests/master/developer/tokenizer.html)
[`develop`](https://github.com/boostorg/tokenizer/tree/develop) | [![Build Status](https://travis-ci.org/boostorg/tokenizer.svg?branch=develop)](https://travis-ci.org/boostorg/tokenizer) | [![Build status](https://ci.appveyor.com/api/projects/status/FIXME-vc81nhd5i2f6hi8y/branch/develop?svg=true)](https://ci.appveyor.com/project/jeking3/tokenizer-c6pnd/branch/develop) | [![Coverity Scan Build Status](https://scan.coverity.com/projects/15854/badge.svg)](https://scan.coverity.com/projects/boostorg-tokenizer) | [![codecov](https://codecov.io/gh/boostorg/tokenizer/branch/develop/graph/badge.svg)](https://codecov.io/gh/boostorg/tokenizer/branch/develop) | [![Deps](https://img.shields.io/badge/deps-develop-brightgreen.svg)](https://pdimov.github.io/boostdep-report/develop/tokenizer.html) | [![Documentation](https://img.shields.io/badge/docs-develop-brightgreen.svg)](http://www.boost.org/doc/libs/develop/doc/html/tokenizer.html) | [![Enter the Matrix](https://img.shields.io/badge/matrix-develop-brightgreen.svg)](http://www.boost.org/development/tests/develop/developer/tokenizer.html)
## Overview
> break up a phrase into words.
<a target="_blank" href="http://melpon.org/wandbox/permlink/kZeKmQAtqDlpn8if">![Try it online][badge.wandbox]</a>
```c++
#include <iostream>
#include <boost/tokenizer.hpp>
#include <string>
int main(){
std::string s = "This is, a test";
typedef boost::tokenizer<> Tok;
Tok tok(s);
for (Tok::iterator beg = tok.begin(); beg != tok.end(); ++beg){
std::cout << *beg << "\n";
}
}
```
> Using Range-based for loop (>c++11)
<a target="_blank" href="http://melpon.org/wandbox/permlink/z94YLs8PdYSh7rXz">![Try it online][badge.wandbox]</a>
```c++
#include <iostream>
#include <boost/tokenizer.hpp>
#include <string>
int main(){
std::string s = "This is, a test";
boost::tokenizer<> tok(s);
for (auto token: tok) {
std::cout << token << "\n";
}
}
```
## Documentation
Documentation can be found at [Boost.Tokenizer](http://boost.org/libs/tokenizer)
## Related Material
[Boost.Tokenizer](http://theboostcpplibraries.com/boost.tokenizer) Chapter 10 at theboostcpplibraries.com, contains several examples including **escaped_list_separator**.
##Contributing
>This library is being maintained as a part of the [Boost Library Official Maintainer Program](http://beta.boost.org/community/official_library_maintainer_program.html)
Open Issues on <a target="_blank" href="https://svn.boost.org/trac/boost/query?status=assigned&status=new&status=reopened&component=tokenizer&col=id&col=summary&col=status&col=owner&col=type&col=milestone&order=priority">![][badge.trac]</a>
##Acknowledgements
>From the author:
>
I wish to thank the members of the boost mailing list, whose comments, compliments, and criticisms during both the development and formal review helped make the Tokenizer library what it is. I especially wish to thank Aleksey Gurtovoy for the idea of using a pair of iterators to specify the input, instead of a string. I also wish to thank Jeremy Siek for his idea of providing a container interface for the token iterators and for simplifying the template parameters for the TokenizerFunctions. He and Daryle Walker also emphasized the need to separate interface and implementation. Gary Powell sparked the idea of using the isspace and ispunct as the defaults for char_delimiters_separator. Jeff Garland provided ideas on how to change to order of the template parameters in order to make tokenizer easier to declare. Thanks to Douglas Gregor who served as review manager and provided many insights both on the boost list and in e-mail on how to polish up the implementation and presentation of Tokenizer. Finally, thanks to Beman Dawes who integrated the final version into the boost distribution.
##License
Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
[badge.Wandbox]: https://img.shields.io/badge/try%20it-online-blue.svg
[badge.Trac]:https://svn.boost.org/htdocs/common/trac_logo_mini.png

View File

@@ -0,0 +1,126 @@
# Copyright 2016, 2017 Peter Dimov
# Copyright (C) 2017 - 2019 James E. King III
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at http://boost.org/LICENSE_1_0.txt)
#
# Generic Appveyor build script for boostorg repositories
# See: https://github.com/boostorg/boost-ci/
#
# Instructions for customizing this script for your library:
#
# 1. Customize the compilers and language levels you want.
# 2. If you have more than include/, src/, test/, example/, examples/,
# benchmark/ or tools/ directories, set the environment variable DEPINST.
# For example if your build uses code in "bench/" and "fog/" directories:
# - DEPINST: --include bench --include fog
# 3. Enable pull request builds in your boostorg/<library> account.
#
# That's it - the script will do everything else for you.
#
version: 1.0.{build}-{branch}
shallow_clone: true
branches:
only:
- develop
- master
matrix:
# Adding MAYFAIL to any matrix job allows it to fail but the build stays green:
allow_failures:
- MAYFAIL: true
environment:
global:
# see: http://www.boost.org/build/doc/html/bbv2/overview/invocation.html#bbv2.overview.invocation.properties
# to use the default for a given environment, comment it out; recommend you build debug and release however:
# on Windows it is important to exercise all the possibilities, especially shared vs static, however most
# libraries that care about this exercise it in their Jamfiles...
# B2_ADDRESS_MODEL: address-model=64,32
# B2_LINK: link=shared,static
# B2_THREADING: threading=multi,single
B2_VARIANT: variant=release,debug
matrix:
- FLAVOR: Visual Studio 2017 C++2a Strict
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
B2_ADDRESS_MODEL: address-model=64
B2_CXXFLAGS: cxxflags=-permissive-
B2_CXXSTD: latest # 2a
B2_TOOLSET: msvc-14.1
- FLAVOR: Visual Studio 2017 C++17
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
B2_ADDRESS_MODEL: address-model=64
B2_CXXSTD: 17
B2_TOOLSET: msvc-14.1
- FLAVOR: Visual Studio 2017 C++14 (Default)
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
B2_ADDRESS_MODEL: address-model=64,32
B2_TOOLSET: msvc-14.1
- FLAVOR: Visual Studio 2015 C++14 (Default)
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
B2_ADDRESS_MODEL: address-model=64,32
B2_TOOLSET: msvc-14.0
- FLAVOR: Visual Studio 2010, 2012, 2013
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
B2_TOOLSET: msvc-10.0,msvc-11.0,msvc-12.0
- FLAVOR: cygwin (32-bit)
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
ADDPATH: C:\cygwin\bin;
B2_ADDRESS_MODEL: address-model=32
B2_CXXSTD: 03,11
# https://github.com/boostorg/test/issues/144
B2_DEFINES: define=_POSIX_C_SOURCE=200112L
B2_THREADING: threadapi=pthread
B2_TOOLSET: gcc
- FLAVOR: cygwin (64-bit)
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
ADDPATH: C:\cygwin64\bin;
B2_ADDRESS_MODEL: address-model=64
B2_CXXSTD: 11,17
# https://github.com/boostorg/test/issues/144
B2_DEFINES: define=_POSIX_C_SOURCE=200112L define=__USE_ISOC99
B2_THREADING: threadapi=pthread
B2_TOOLSET: gcc
- FLAVOR: mingw32
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
ARCH: i686
B2_ADDRESS_MODEL: address-model=32
B2_CXXSTD: 03,11
SCRIPT: ci\appveyor\mingw.bat
- FLAVOR: mingw64
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
ARCH: x86_64
B2_ADDRESS_MODEL: address-model=64
B2_CXXSTD: 11,17
B2_DEFINES: define=__USE_ISOC99
SCRIPT: ci\appveyor\mingw.bat
install:
- set SELF=%APPVEYOR_PROJECT_NAME:-=_%
- git clone https://github.com/boostorg/boost-ci.git C:\boost-ci
- xcopy /s /e /q /i C:\boost-ci\ci .\ci
- ci\appveyor\install.bat
build: off
test_script:
- set SELF=%APPVEYOR_PROJECT_NAME:-=_%
- PATH=%ADDPATH%%PATH%
# The definition of B2_TOOLCXX omits B2_CXXSTD= if it was not defined above
- IF NOT DEFINED B2_CXXSTD (SET B2_TOOLCXX=toolset=%B2_TOOLSET%) ELSE (SET B2_TOOLCXX=toolset=%B2_TOOLSET% cxxstd=%B2_CXXSTD%)
# Echo the complete build command to the build log
- IF NOT DEFINED SCRIPT (ECHO b2 libs/%SELF:\=/% %B2_TOOLCXX% %B2_CXXFLAGS% %B2_DEFINES% %B2_THREADING% %B2_ADDRESS_MODEL% %B2_LINK% %B2_THREADING% %B2_VARIANT% -j3)
# Now go build...
- IF DEFINED SCRIPT (call libs\%SELF%\%SCRIPT%) ELSE (b2 libs/%SELF:\=/% %B2_TOOLCXX% %B2_CXXFLAGS% %B2_DEFINES% %B2_THREADING% %B2_ADDRESS_MODEL% %B2_LINK% %B2_THREADING% %B2_VARIANT% -j3)

View File

@@ -0,0 +1,156 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 6.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Boost Char Delimiters Separator</title>
</head>
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE" vlink="#551A8B" alink=
"#FF0000">
<p><img src="../../../boost.png" alt="C++ Boost" width="277" height=
"86"><br></p><font color="red">Note: This class is deprecated. Please use
<a href="char_separator.htm"><tt>char_separator</tt></a> instead.</font>
<h1 align="center">Char Delimiters Separator</h1>
<pre>
template &lt;class Char, class Traits = std::char_traits&lt;Char&gt; &gt;
class char_delimiters_separator{
</pre>
<p>The char_delimiters_separator class is an implementation of the <a href=
"tokenizerfunction.htm">TokenizerFunction</a> concept that can be used to
break text up into tokens. It is the default TokenizerFunction for
tokenizer and token_iterator_generator. An example is below.</p>
<h2>Example</h2>
<pre>
// simple_example_4.cpp
#include&lt;iostream&gt;
#include&lt;boost/tokenizer.hpp&gt;
#include&lt;string&gt;
int main(){
using namespace std;
using namespace boost;
string s = "This is, a test";
tokenizer&lt;char_delimiters_separator&lt;char&gt; &gt; tok(s);
for(tokenizer&lt;char_delimiters_separator&lt;char&gt; &gt;::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout &lt;&lt; *beg &lt;&lt; "\n";
}
}
</pre>
<h2>Construction and Usage</h2>
<p>There is one constructor of interest. It is as follows</p>
<pre>
explicit char_delimiters_separator(bool return_delims = false,
const Char* returnable = "",const Char* nonreturnable = "" )
</pre>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Parameter</strong></p>
</td>
<td>
<p align="center"><strong>Description</strong></p>
</td>
</tr>
<tr>
<td>return_delims</td>
<td>Whether or not to return the delimiters that have been found. Note
that not all delimiters can be returned. See the other two parameters
for explanation.</td>
</tr>
<tr>
<td>returnable</td>
<td>This specifies the returnable delimiters. These are the delimiters
that can be returned as tokens when return_delims is true. Since these
are typically punctuation, if a 0 is provided as the argument, then the
returnable delmiters will be all characters Cfor which std::ispunct(C)
yields a true value. If an argument of "" is provided, then this is
taken to mean that there are noreturnable delimiters.</td>
</tr>
<tr>
<td>nonreturnable</td>
<td>This specifies the nonreturnable delimiters. These are delimiters
that cannot be returned as tokens. Since these are typically
whitespace, if 0 is specified as an argument, then the nonreturnable
delimiters will be all characters C for which std::isspace(C) yields a
true value. If an argument of "" is provided, then this is taken to
mean that there are no non-returnable delimiters.</td>
</tr>
</table>
<p>The reason there is a distinction between nonreturnable and returnable
delimiters is that some delimiters are just used to split up tokens and are
nothing more. Take for example the following string "b c +". Assume you are
writing a simple calculator to parse expression in post fix notation. While
both the space and the + separate tokens, you only only interested in the +
and not in the space. Indeed having the space returned as a token would
only complicate your code. In this case you would specify + as a
returnable, and space as a nonreturnable delimiter.</p>
<p>To use this class, pass an object of it anywhere a TokenizerFunction
object is required.</p>
<h3>Template Parameters</h3>
<table border="1" summary="">
<tr>
<th>Parameter</th>
<th>Description</th>
</tr>
<tr>
<td><tt>Char</tt></td>
<td>The type of the elements within a token, typically
<tt>char</tt>.</td>
</tr>
<tr>
<td>Traits</td>
<td>The traits class for Char, typically
std::char_traits&lt;Char&gt;</td>
</tr>
</table>
<h2>Model of</h2>
<p><a href="tokenizerfunction.htm">TokenizerFunction</a></p>
<p>&nbsp;</p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B, %Y" startspan -->25
December, 2006<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
<p><i>Copyright &copy; 2001 John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,230 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 6.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Boost Char Separator</title>
</head>
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE" vlink="#551A8B" alink=
"#FF0000">
<p><img src="../../../boost.png" alt="C++ Boost" width="277" height=
"86"><br></p>
<h1>char_separator&lt;Char, Traits&gt;</h1>
<p>The <tt>char_separator</tt> class breaks a sequence of characters into
tokens based on character delimiters much in the same way that
<tt>strtok()</tt> does (but without all the evils of non-reentrancy and
destruction of the input sequence).</p>
<p>The <tt>char_separator</tt> class is used in conjunction with the
<a href="token_iterator.htm"><tt>token_iterator</tt></a> or <a href=
"tokenizer.htm"><tt>tokenizer</tt></a> to perform tokenizing.</p>
<h2>Definitions</h2>
<p>The <tt>strtok()</tt> function does not include matches with the
character delimiters in the output sequence of tokens. However, sometimes
it is useful to have the delimiters show up in the output sequence,
therefore <tt>char_separator</tt> provides this as an option. We refer to
delimiters that show up as output tokens as <b><i>kept delimiters</i></b>
and delimiters that do now show up as output tokens as <b><i>dropped
delimiters</i></b>.</p>
<p>When two delimiters appear next to each other in the input sequence,
there is the question of whether to output an <b><i>empty token</i></b> or
to skip ahead. The behaviour of <tt>strtok()</tt> is to skip ahead. The
<tt>char_separator</tt> class provides both options.</p>
<h2>Examples</h2>
<p>This first examples shows how to use <tt>char_separator</tt> as a
replacement for the <tt>strtok()</tt> function. We've specified three
character delimiters, and they will not show up as output tokens. We have
not specified any kept delimiters, and by default any empty tokens will be
ignored.</p>
<blockquote>
<pre>
// char_sep_example_1.cpp
#include &lt;iostream&gt;
#include &lt;boost/tokenizer.hpp&gt;
#include &lt;string&gt;
int main()
{
std::string str = ";;Hello|world||-foo--bar;yow;baz|";
typedef boost::tokenizer&lt;boost::char_separator&lt;char&gt; &gt;
tokenizer;
boost::char_separator&lt;char&gt; sep("-;|");
tokenizer tokens(str, sep);
for (tokenizer::iterator tok_iter = tokens.begin();
tok_iter != tokens.end(); ++tok_iter)
std::cout &lt;&lt; "&lt;" &lt;&lt; *tok_iter &lt;&lt; "&gt; ";
std::cout &lt;&lt; "\n";
return EXIT_SUCCESS;
}
</pre>
</blockquote>The output is:
<blockquote>
<pre>
&lt;Hello&gt; &lt;world&gt; &lt;foo&gt; &lt;bar&gt; &lt;yow&gt; &lt;baz&gt;
</pre>
</blockquote>
<p>The next example shows tokenizing with two dropped delimiters '-' and
';' and a single kept delimiter '|'. We also specify that empty tokens
should show up in the output when two delimiters are next to each
other.</p>
<blockquote>
<pre>
// char_sep_example_2.cpp
#include &lt;iostream&gt;
#include &lt;boost/tokenizer.hpp&gt;
#include &lt;string&gt;
int main()
{
std::string str = ";;Hello|world||-foo--bar;yow;baz|";
typedef boost::tokenizer&lt;boost::char_separator&lt;char&gt; &gt;
tokenizer;
boost::char_separator&lt;char&gt; sep("-;", "|", boost::keep_empty_tokens);
tokenizer tokens(str, sep);
for (tokenizer::iterator tok_iter = tokens.begin();
tok_iter != tokens.end(); ++tok_iter)
std::cout &lt;&lt; "&lt;" &lt;&lt; *tok_iter &lt;&lt; "&gt; ";
std::cout &lt;&lt; "\n";
return EXIT_SUCCESS;
}
</pre>
</blockquote>The output is:
<blockquote>
<pre>
&lt;&gt; &lt;&gt; &lt;Hello&gt; &lt;|&gt; &lt;world&gt; &lt;|&gt; &lt;&gt; &lt;|&gt; &lt;&gt; &lt;foo&gt; &lt;&gt; &lt;bar&gt; &lt;yow&gt; &lt;baz&gt; &lt;|&gt; &lt;&gt;
</pre>
</blockquote>
<p>The final example shows tokenizing on punctuation and whitespace
characters using the default constructor of the
<tt>char_separator</tt>.</p>
<blockquote>
<pre>
// char_sep_example_3.cpp
#include &lt;iostream&gt;
#include &lt;boost/tokenizer.hpp&gt;
#include &lt;string&gt;
int main()
{
std::string str = "This is, a test";
typedef boost::tokenizer&lt;boost::char_separator&lt;char&gt; &gt; Tok;
boost::char_separator&lt;char&gt; sep; // default constructed
Tok tok(str, sep);
for(Tok::iterator tok_iter = tok.begin(); tok_iter != tok.end(); ++tok_iter)
std::cout &lt;&lt; "&lt;" &lt;&lt; *tok_iter &lt;&lt; "&gt; ";
std::cout &lt;&lt; "\n";
return EXIT_SUCCESS;
}
</pre>
</blockquote>The output is:
<blockquote>
<pre>
&lt;This&gt; &lt;is&gt; &lt;,&gt; &lt;a&gt; &lt;test&gt;
</pre>
</blockquote>
<h2>Template parameters</h2>
<table border summary="">
<tr>
<th>Parameter</th>
<th>Description</th>
<th>Default</th>
</tr>
<tr>
<td><tt>Char</tt></td>
<td>The type of elements within a token, typically <tt>char</tt>.</td>
<td>&nbsp;</td>
</tr>
<tr>
<td><tt>Traits</tt></td>
<td>The <tt>char_traits</tt> for the character type.</td>
<td><tt>char_traits&lt;char&gt;</tt></td>
</tr>
</table>
<h2>Model of</h2><a href="tokenizerfunction.htm">Tokenizer Function</a>
<h2>Members</h2>
<hr>
<pre>
explicit char_separator(const Char* dropped_delims,
const Char* kept_delims = "",
empty_token_policy empty_tokens = drop_empty_tokens)
</pre>
<p>This creates a <tt>char_separator</tt> object, which can then be used to
create a <a href="token_iterator.htm"><tt>token_iterator</tt></a> or
<a href="tokenizer.htm"><tt>tokenizer</tt></a> to perform tokenizing. The
<tt>dropped_delims</tt> and <tt>kept_delims</tt> are strings of characters
where each character is used as delimiter during tokenizing. Whenever a
delimiter is seen in the input sequence, the current token is finished, and
a new token begins. The delimiters in <tt>dropped_delims</tt> do not show
up as tokens in the output whereas the delimiters in <tt>kept_delims</tt>
do show up as tokens. If <tt>empty_tokens</tt> is
<tt>drop_empty_tokens</tt>, then empty tokens will not show up in the
output. If <tt>empty_tokens</tt> is <tt>keep_empty_tokens</tt> then empty
tokens will show up in the output.</p>
<hr>
<pre>
explicit char_separator()
</pre>
<p>The function <tt>std::isspace()</tt> is used to identify dropped
delimiters and <tt>std::ispunct()</tt> is used to identify kept delimiters.
In addition, empty tokens are dropped.</p>
<hr>
<pre>
template &lt;typename InputIterator, typename Token&gt;
bool operator()(InputIterator&amp; next, InputIterator end, Token&amp; tok)
</pre>
<p>This function is called by the <a href=
"token_iterator.htm"><tt>token_iterator</tt></a> to perform tokenizing. The
user typically does not call this function directly.</p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B, %Y" startspan -->25
December, 2006<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
<p><i>Copyright &copy; 2001-2002 Jeremy Siek and John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,231 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 6.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Boost Escaped List Separator</title>
</head>
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE" vlink="#551A8B" alink=
"#FF0000">
<h1 align="left"><img src="../../../boost.png" alt="C++ Boost" width="277"
height="86"></h1>
<h1 align="center">Escaped List Separator</h1>
<div align="left">
<pre>
escaped_list_separator&lt;Char, Traits = std::char_traits&lt;Char&gt; &gt;
</pre>
</div>
<p>The <tt>escaped_list_separator</tt> class is an implementation of the
<a href="tokenizerfunction.htm">TokenizerFunction</a>. The
escaped_list_separator parses a superset of the csv (comma separated value)
format. The examples of this formate are below. It is assumed that the
default characters for separator, quote, and escape are used.</p>
<p>Field 1,Field 2,Field 3<br>
Field 1,"Field 2, with comma",Field 3<br>
Field 1,Field 2 with \"embedded quote\",Field 3<br>
Field 1, Field 2 with \n new line,Field 3<br>
Field 1, Field 2 with embedded \\ ,Field 3</p>
<p>Fields are normally separated by commas. If you want to put a comma in a
field, you need to put quotes around it. Also 3 escape sequences are
supported</p>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Escape Sequence</strong></p>
</td>
<td>
<p align="center"><strong>Result</strong></p>
</td>
</tr>
<tr>
<td>&lt;escape&gt;&lt;quote&gt;</td>
<td>&lt;quote&gt;</td>
</tr>
<tr>
<td>&lt;escape&gt;n</td>
<td>newline</td>
</tr>
<tr>
<td>&lt;escape&gt;&lt;escape&gt;</td>
<td>&lt;escape&gt;</td>
</tr>
</table>
<p>Where &lt;quote&gt; is any character specified to be a quote
and&lt;escape&gt; is any character specified to be an escape character.</p>
<h2>Example</h2>
<pre>
// simple_example_2.cpp
#include&lt;iostream&gt;
#include&lt;boost/tokenizer.hpp&gt;
#include&lt;string&gt;
int main(){
using namespace std;
using namespace boost;
string s = "Field 1,\"putting quotes around fields, allows commas\",Field 3";
tokenizer&lt;escaped_list_separator&lt;char&gt; &gt; tok(s);
for(tokenizer&lt;escaped_list_separator&lt;char&gt; &gt;::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout &lt;&lt; *beg &lt;&lt; "\n";
}
}
</pre>
<p>&nbsp;</p>
<h2>Construction and Usage</h2>
<p>escaped_list_separator has 2 constructors. They are as follows</p>
<pre>
explicit escaped_list_separator(Char e = '\\', Char c = ',',Char q = '\"')
</pre>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Parameter</strong></p>
</td>
<td>
<p align="center"><strong>Description</strong></p>
</td>
</tr>
<tr>
<td>e</td>
<td>Specifies the character to use for escape sequences. It defaults to
the C style \ (backslash). However you can override by passing in a
different character. An example of when you might want to do this is
when you have many fields which are Windows style filenames. Instead of
escaping out each \ in the path, you can change the escape to something
else.</td>
</tr>
<tr>
<td>c</td>
<td>Specifies the character to use to separate the fields</td>
</tr>
<tr>
<td>q</td>
<td>Specifies the character to use for the quote.</td>
</tr>
</table>
<p>&nbsp;</p>
<pre>
escaped_list_separator(string_type e, string_type c, string_type q):
</pre>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Parameter</strong></p>
</td>
<td>
<p align="center"><strong>Description</strong></p>
</td>
</tr>
<tr>
<td>e</td>
<td>Any character in the string e, is considered to be an escape
character. If an empty string is given, then there are no escape
characters.</td>
</tr>
<tr>
<td>c</td>
<td>Any character in the string c, is considered to be a separator. If
an empty string is given, then there are no separator characters.</td>
</tr>
<tr>
<td>q</td>
<td>Any character in the string q, is considered to be a quote. If an
empty string is given, then there are no quote characters.</td>
</tr>
</table>
<p>&nbsp;</p>
<p>To use this class, pass an object of it anywhere in the Tokenizer
package where a TokenizerFunction is required.</p>
<p>&nbsp;</p>
<h2>Template Parameters</h2>
<table border="1" summary="">
<tr>
<th><strong>Parameter</strong></th>
<th><strong>Description</strong></th>
</tr>
<tr>
<td><tt>Char</tt></td>
<td>The type of the elements within a token, typically
<tt>char</tt>.</td>
</tr>
<tr>
<td>Traits</td>
<td>The traits class for the Char type. This is used for comparing
Char's. It defaults to std::char_traits&lt;Char&gt;</td>
</tr>
</table>
<p>&nbsp;</p>
<h2>Model of</h2>
<p><a href="tokenizerfunction.htm">TokenizerFunction</a></p>
<p>&nbsp;</p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B, %Y" startspan -->25
December, 2006<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
<p><i>Copyright &copy; 2001 John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,96 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 6.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Boost Tokenizer Overview</title>
</head>
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE" vlink="#551A8B" alink=
"#FF0000">
<p><img src="../../../boost.png" alt="C++ Boost" width="277" height=
"86"><br></p>
<h1 align="center">Table Of Contents</h1>
<p align="left">&nbsp;</p>
<h2 align="left"><a href="introduc.htm">Introduction</a></h2>
<h2 align="left">Containers and Iterators</h2>
<ul>
<li>
<h3 align="left"><a href="tokenizer.htm">tokenizer</a></h3>
</li>
<li>
<h3 align="left"><a href="token_iterator.htm">token iterator</a></h3>
</li>
</ul>
<h2><a href="tokenizerfunction.htm">TokenizerFunction Concept</a></h2>
<h2>TokenizerFunction Models</h2>
<ul>
<li>
<h3><a href="char_separator.htm">char_separator</a></h3>
</li>
<li>
<h3><a href=
"escaped_list_separator.htm">escaped_list_separator</a></h3>
</li>
<li>
<h3><a href="offset_separator.htm">offset_separator</a></h3>
</li>
<li><font color="red">Deprecated:</font> <a href=
"char_delimiters_separator.htm">char_delimiters_separator</a></li>
</ul>
<h2>&nbsp;</h2>
<h2>Acknowledgements</h2>
<p>I wish to thank the members of the boost mailing list, whose comments,
compliments, and criticisms during both the development and formal review
helped make the Tokenizer library what it is. I especially wish to thank
Aleksey Gurtovoy for the idea of using a pair of iterators to specify the
input, instead of a string. I also wish to thank Jeremy Siek for his idea
of providing a container interface for the token iterators and for
simplifying the template parameters for the TokenizerFunctions. He and
Daryle Walker also emphasized the need to separate interface and
implementation. Gary Powell sparked the idea of using the isspace and
ispunct as the defaults for char_delimiters_separator. Jeff Garland
provided ideas on how to change to order of the template parameters in
order to make tokenizer easier to declare. Thanks to Douglas Gregor who
served as review manager and provided many insights both on the boost list
and in e-mail on how to polish up the implementation and presentation of
Tokenizer. Finally, thanks to Beman Dawes who integrated the final version
into the boost distribution.</p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B, %Y" startspan -->25
December, 2006<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
<p><i>Copyright &copy; 2000 Jeremy Siek<br>
Copyright &copy; 2001 John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,120 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 6.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Introduction</title>
</head>
<body bgcolor="#FFFFFF">
<p><img src="../../../boost.png" alt="C++ Boost" width="277" height=
"86"/><br></p>
<h1 align="center">Introduction</h1>
<p align="left">The Boost Tokenizer package provides a flexible and
easy-to-use way to break a string or other character sequence into a series
of tokens. Below is a simple example that will break up a phrase into
words.</p>
<div align="left">
<pre>
// simple_example_1.cpp
#include&lt;iostream&gt;
#include&lt;boost/tokenizer.hpp&gt;
#include&lt;string&gt;
int main(){
using namespace std;
using namespace boost;
string s = "This is, a test";
tokenizer&lt;&gt; tok(s);
for(tokenizer&lt;&gt;::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout &lt;&lt; *beg &lt;&lt; "\n";
}
}
</pre>
</div>
<p align="left">You can choose how the string gets parsed by using the
TokenizerFunction. If you do not specify anything, the default
TokenizerFunction is <em>char_delimiters_separator&lt;char&gt;</em> which
defaults to breaking up a string based on space and punctuation. Here is an
example using another TokenizerFunction called
<em>escaped_list_separator</em>. This TokenizerFunction parses a superset
of comma-separated value (CSV) lines. The format looks like this:</p>
<p align="left">Field 1,"putting quotes around fields, allows commas",Field
3</p>
<p align="left">Below is an example that will break the previous line into
its three fields.</p>
<div align="left">
<pre>
// simple_example_2.cpp
#include&lt;iostream&gt;
#include&lt;boost/tokenizer.hpp&gt;
#include&lt;string&gt;
int main(){
using namespace std;
using namespace boost;
string s = "Field 1,\"putting quotes around fields, allows commas\",Field 3";
tokenizer&lt;escaped_list_separator&lt;char&gt; &gt; tok(s);
for(tokenizer&lt;escaped_list_separator&lt;char&gt; &gt;::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout &lt;&lt; *beg &lt;&lt; "\n";
}
}
</pre>
</div>
<p align="left">Finally, for some TokenizerFunctions you have to pass
something into the constructor in order to do anything interesting. An
example is the offset_separator. This class breaks a string into tokens based
on offsets. For example, when <em>12252001</em> is parsed using offsets of
2,2,4 it becomes <em>12 25 2001</em>. Below is the code used.</p>
<div align="left">
<pre>
// simple_example_3.cpp
#include&lt;iostream&gt;
#include&lt;boost/tokenizer.hpp&gt;
#include&lt;string&gt;
int main(){
using namespace std;
using namespace boost;
string s = "12252001";
int offsets[] = {2,2,4};
offset_separator f(offsets, offsets+3);
tokenizer&lt;offset_separator&gt; tok(s,f);
for(tokenizer&lt;offset_separator&gt;::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout &lt;&lt; *beg &lt;&lt; "\n";
}
}
</pre>
</div>
<p align="left">&nbsp;</p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B %Y" startspan -->9 June 2010<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
<p><i>Copyright &copy; 2001 John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,131 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 6.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Boost Offset Separator</title>
</head>
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE" vlink="#551A8B" alink=
"#FF0000">
<p><img src="../../../boost.png" alt="C++ Boost" width="277" height=
"86"><br></p>
<h1 align="center">Offset Separator</h1>
<pre>
class offset_separator
</pre>
<p>The <tt>offset_separator</tt> class is an implementation of the <a href=
"tokenizerfunction.htm">TokenizerFunction</a> concept that can be used with
the <a href="tokenizer.htm">tokenizer</a> class to break text up into
tokens. The <tt>offset_separator</tt> breaks a sequence of <tt>Char</tt>'s
into strings based on a sequence of offsets. For example, if you had the
string "12252001" and offsets (2,2,4) it would break the string into 12 25
2001. Here is an example.</p>
<h2>Example</h2>
<pre>
// simple_example_3.cpp
#include&lt;iostream&gt;
#include&lt;boost/tokenizer.hpp&gt;
#include&lt;string&gt;
int main(){
using namespace std;
using namespace boost;
string s = "12252001";
int offsets[] = {2,2,4};
offset_separator f(offsets, offsets+3);
tokenizer&lt;offset_separator&gt; tok(s,f);
for(tokenizer&lt;offset_separator&gt;::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout &lt;&lt; *beg &lt;&lt; "\n";
}
}
</pre>
<p>&nbsp;</p>
<h2>Construction and Usage</h2>
<p>The offset_separator has 1 constructor of interest. (The default
constructor is just there to make some compilers happy). The declaration is
below</p>
<pre>
template&lt;typename Iter&gt;
offset_separator(Iter begin,Iter end,bool bwrapoffsets = true, bool breturnpartiallast = true)
</pre>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Parameter</strong></p>
</td>
<td>
<p align="center"><strong>Description</strong></p>
</td>
</tr>
<tr>
<td>begin, end</td>
<td>Specify the sequence of integer offsets.</td>
</tr>
<tr>
<td>bwrapoffsets</td>
<td>Tells whether to wrap around to the beginning of the offsets when
the all the offsets have been used. For example the string
"1225200101012002" with offsets (2,2,4) with bwrapoffsets to true,
would parse to 12 25 2001 01 01 2002. With bwrapoffsets to false, it
would parse to 12 25 2001 and then stop because all the offsets have
been used.</td>
</tr>
<tr>
<td>breturnpartiallast</td>
<td>Tells whether, when the parsed sequence terminates before yielding
the number of characters in the current offset, to create a token with
what was parsed, or to ignore it. For example the string "122501" with
offsets (2,2,4) with breturnpartiallast set to true will parse to 12 25
01. With it set to false, it will parse to 12 25 and then will stop
because there are only 2 characters left in the sequence instead of the
4 that should have been there.</td>
</tr>
</table>
<p>To use this class, pass an object of it anywhere a TokenizerFunction is
required. If you default constructruct the object, it will just return
every character in the parsed sequence as a token. (ie it defaults to an
offset of 1, and bwrapoffsets is true).</p>
<p>&nbsp;</p>
<h2>Model of</h2>
<p><a href="tokenizerfunction.htm">TokenizerFunction</a></p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B, %Y" startspan -->25
December, 2006<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
<p><i>Copyright &copy; 2001 John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,171 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 6.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Boost Token Iterator</title>
</head>
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE" vlink="#551A8B" alink=
"#FF0000">
<p><img src="../../../boost.png" alt="C++ Boost" width="277" height=
"86"><br></p>
<h1 align="center">Token Iterator</h1>
<pre>
template &lt;
class TokenizerFunc = char_delimiters_separator&lt;char&gt;,
class Iterator = std::string::const_iterator,
class Type = std::string
&gt;
class token_iterator_generator
</pre>
<pre>
template&lt;class Type, class Iterator, class TokenizerFunc&gt;
typename token_iterator_generator&lt;TokenizerFunc,Iterator,Type&gt;::type
make_token_iterator(Iterator begin, Iterator end,const TokenizerFunc&amp; fun)
</pre>
<p>The token iterator serves to provide an iterator view of the tokens in a
parsed sequence.</p>
<h2>Example</h2>
<pre>
/// simple_example_5.cpp
#include&lt;iostream&gt;
#include&lt;boost/token_iterator.hpp&gt;
#include&lt;string&gt;
int main(){
using namespace std;
using namespace boost;
string s = "12252001";
int offsets[] = {2,2,4};
offset_separator f(offsets, offsets+3);
typedef token_iterator_generator&lt;offset_separator&gt;::type Iter;
Iter beg = make_token_iterator&lt;string&gt;(s.begin(),s.end(),f);
Iter end = make_token_iterator&lt;string&gt;(s.end(),s.end(),f);
// The above statement could also have been what is below
// Iter end;
for(;beg!=end;++beg){
cout &lt;&lt; *beg &lt;&lt; "\n";
}
}
</pre>
<p>&nbsp;</p>
<h3>Template Parameters</h3>
<table border="1" summary="">
<tr>
<th>Parameter</th>
<th>Description</th>
</tr>
<tr>
<td><tt>TokenizerFunc</tt></td>
<td>The TokenizerFunction used to parse the sequence.</td>
</tr>
<tr>
<td><tt>Iterator</tt></td>
<td>The type of the iterator the specifies the sequence.</td>
</tr>
<tr>
<td><tt>Type</tt></td>
<td>The type of the token, typically string.</td>
</tr>
</table>
<h2>Model of</h2>
<p>The category of Iterator, up to and including Forward Iterator. Anything
higher will get scaled down to Forward Iterator.</p>
<h2>Related Types</h2>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Type</strong></p>
</td>
<td>
<p align="center"><strong>Remarks</strong></p>
</td>
</tr>
<tr>
<td>token_iterator_generator::type</td>
<td>The type of the token iterator.</td>
</tr>
</table>
<h2>Creation</h2>
<pre>
template&lt;class Type, class Iterator, class TokenizerFunc&gt;
typename token_iterator_generator&lt;TokenizerFunc,Iterator,Type&gt;::type
make_token_iterator(Iterator begin, Iterator end,const TokenizerFunc&amp; fun)
</pre>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Parameter</strong></p>
</td>
<td>
<p align="center"><strong>Description</strong></p>
</td>
</tr>
<tr>
<td>begin</td>
<td>The beginning of the sequence to be parsed.</td>
</tr>
<tr>
<td>end</td>
<td>Past the end of the sequence to be parsed.</td>
</tr>
<tr>
<td>fun</td>
<td>A functor that is a model of TokenizerFunction</td>
</tr>
</table>
<p>&nbsp;</p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B, %Y" startspan -->25
December, 2006<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
<p><i>Copyright &copy; 2001 John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,244 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 5.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Boost Tokenizer Class</title>
</head>
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE" vlink="#551A8B" alink=
"#FF0000">
<p><img src="../../../boost.png" alt="C++ Boost" width="277" height=
"86"><br></p>
<h1 align="center">Tokenizer Class</h1>
<pre> template &lt;
class TokenizerFunc = char_delimiters_separator&lt;char&gt;,
class Iterator = std::string::const_iterator,
class Type = std::string
&gt;
class tokenizer
</pre>
<p>The tokenizer class provides a container view of a series of tokens
contained in a sequence. You set the sequence to parse and the
TokenizerFunction to use to parse the sequence either upon construction or
using the assign member function. Note: No parsing is actually done upon
construction. Parsing is done on demand as the tokens are accessed via the
iterator provided by begin.</p>
<h2>Example</h2>
<pre>// simple_example_1.cpp
#include&lt;iostream&gt;
#include&lt;boost/tokenizer.hpp&gt;
#include&lt;string&gt;
int main(){
using namespace std;
using namespace boost;
string s = "This is, a test";
tokenizer&lt;&gt; tok(s);
for(tokenizer&lt;&gt;::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout &lt;&lt; *beg &lt;&lt; "\n";
}
}
</pre>
<p>The output from simple_example_1 is:</p>
<blockquote>
<p><code>This<br>
is<br>
a<br>
test</code></p>
</blockquote>
<h3>Template Parameters</h3>
<table border="1" summary="">
<tr>
<th>Parameter</th>
<th>Description</th>
</tr>
<tr>
<td><tt>TokenizerFunc</tt></td>
<td>The TokenizerFunction used to parse the sequence.</td>
</tr>
<tr>
<td><tt>Iterator</tt></td>
<td>The type of the iterator the specifies the sequence.</td>
</tr>
<tr>
<td><tt>Type</tt></td>
<td>The type of the token, typically string.</td>
</tr>
</table>
<p>&nbsp;</p>
<h2>Related Types</h2>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Type</strong></p>
</td>
<td>
<p align="center"><strong>Remarks</strong></p>
</td>
</tr>
<tr>
<td>iterator</td>
<td>The type returned by begin and end. Note: the category of iterator
will be at most ForwardIterator. It will be InputIterator if the
Iterator template parameter is an InputIterator. For any other
category, it will be ForwardIterator.</td>
</tr>
<tr>
<td>const_iterator</td>
<td>Same type as iterator.</td>
</tr>
<tr>
<td>value_type</td>
<td>Same type as the template parameter Type</td>
</tr>
<tr>
<td>reference</td>
<td>Same type as value_type&amp;</td>
</tr>
<tr>
<td>const_reference</td>
<td>Same type as const reference</td>
</tr>
<tr>
<td>pointer</td>
<td>Same type as value_type*</td>
</tr>
<tr>
<td>const_pointer</td>
<td>Same type as const pointer</td>
</tr>
<tr>
<td>size_type</td>
<td>void</td>
</tr>
<tr>
<td>difference_type</td>
<td>void</td>
</tr>
</table>
<p>&nbsp;</p>
<h2>Construction and Member Functions</h2>
<pre>tokenizer(Iterator first, Iterator last,const TokenizerFunc&amp; f = TokenizerFunc())
template&lt;class Container&gt;
tokenizer(const Container&amp; c,const TokenizerFunc&amp; f = TokenizerFunc())
void assign(Iterator first, Iterator last)
void assign(Iterator first, Iterator last, const TokenizerFunc&amp; f)
template&lt;class Container&gt;
void assign(const Container&amp; c)
template&lt;class Container&gt;
void assign(const Container&amp; c, const TokenizerFunc&amp; f)
iterator begin() const
iterator end() const
</pre>
<table border="1" summary="">
<tr>
<td>
<p align="center"><strong>Parameter</strong></p>
</td>
<td>
<p align="center"><strong>Description</strong></p>
</td>
</tr>
<tr>
<td>c</td>
<td>A container that contains the sequence to parse. Note: c.begin()
and c.end() must be convertible to the template parameter
Iterator.</td>
</tr>
<tr>
<td>f</td>
<td>A functor that is a model of TokenizerFunction that will be used to
parse the sequence.</td>
</tr>
<tr>
<td>first</td>
<td>The iterator that represents the beginning position in the sequence
to be parsed.</td>
</tr>
<tr>
<td>last</td>
<td>The iterator that represents the past the end position in the
sequence to be parsed.</td>
</tr>
</table>
<p>&nbsp;</p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B, %Y" startspan -->16 February, 2008<!--webbot bot="Timestamp" endspan i-checksum="40414" --></p>
<p><i>Copyright &copy; 2001 John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,182 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="GENERATOR" content="Microsoft FrontPage 6.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>TokenizerFunction Concept</title>
</head>
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE" vlink="#551A8B" alink=
"#FF0000">
<p><img src="../../../boost.png" alt="C++ Boost" width="277" height="86"></p>
<h1 align="center">TokenizerFunction Concept</h1>
<p>A TokenizerFunction is a functor whose purpose is to parse a given
sequence until exactly 1 token has been found or the end is reached. It
then updates the token, and informs the caller of the location in the
sequence of the next element immediately after the last element of the
sequence that was parsed for the current token.</p>
<h2>Refinement of</h2>
<p>Assignable, CopyConstructable</p>
<h2>Notation</h2>
<table border="1" summary="">
<tr>
<td valign="top"><tt>X</tt></td>
<td valign="top">A type that is a model of TokenizerFunction</td>
</tr>
<tr>
<td valign="top"><tt>func</tt></td>
<td valign="top">Object of type <tt>X</tt></td>
</tr>
<tr>
<td valign="top"><tt>tok</tt></td>
<td valign="top">Object of Token</td>
</tr>
<tr>
<td>next</td>
<td>iterator that points to the first unparsed element of the sequence
being parsed</td>
</tr>
<tr>
<td>end</td>
<td>iterator that points to the past the end of the sequence being
parsed</td>
</tr>
</table>
<h2>Definitions</h2>
<p>A token is the result of parsing a sequence.</p>
<h2>Valid expressions</h2>
<p>In addition to the expression in Assignable and CopyConstructable the
following expressions are valid</p>
<table border="1" summary="">
<tr>
<th>Name</th>
<th>Expression</th>
<th>Return type</th>
</tr>
<tr>
<td valign="top">Functor</td>
<td valign="top"><tt>func(next, end, tok)</tt></td>
<td valign="top"><tt>bool</tt></td>
</tr>
<tr>
<td valign="top">reset</td>
<td valign="top"><tt>reset()</tt></td>
<td valign="top"><tt>void</tt></td>
</tr>
</table>
<h2>Expression semantics</h2>
<p>In addition to the expression semantics in Assignable and
CopyConstructable, TokenizerFunction has the following expression
semantcs</p>
<table border="1" summary="">
<tr>
<th>Name</th>
<th>Expression</th>
<th>Precondition</th>
<th>Semantics</th>
<th>Postcondition</th>
</tr>
<tr>
<td>operator()</td>
<td><tt>func(next, end, tok)</tt></td>
<td><tt>next</tt> and <tt>end</tt> are valid iterators to the same
sequence. next is a reference the function is free to modify. tok is
constructed.</td>
<td>The return value indicates whether a new token was found in the
sequence [next,end)</td>
<td>If the return value is true, the new token is assigned to tok. next
is always updated to the position where parsing should start on the
subsequent call.</td>
</tr>
<tr>
<td>reset</td>
<td><tt>reset()</tt></td>
<td><tt>None</tt></td>
<td>Clears out all state variables that are used by the object in
parsing the current sequence.</td>
<td>A new sequence to parse can be given.</td>
</tr>
</table>
<h2>Complexity guarantees</h2>
<p>No guarantees. Models of TokenizerFunction are free to define their own
complexity</p>
<h2>Models</h2>
<p><a href="escaped_list_separator.htm">escaped_list_separator</a></p>
<p><a href="offset_separator.htm">offset_separator</a></p>
<p><a href=
"char_delimiters_separator.htm">char_delimiters_separator</a></p>
<p>&nbsp;</p>
<hr>
<p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
"../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
height="31" width="88"></a></p>
<p>Revised
<!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B, %Y" startspan -->25
December, 2006<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
<p><i>Copyright &copy; 2001 John R. Bandela</i></p>
<p><i>Distributed under the Boost Software License, Version 1.0. (See
accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
copy at <a href=
"http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
</body>
</html>

View File

@@ -0,0 +1,15 @@
# Boost.Tokenizer Library example Jamfile
#
# Copyright (c) 2008 James E. King III
#
# Distributed under the Boost Software License, Version 1.0. (See accompany-
# ing file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
import testing ;
test-suite "tokenizer-examples"
: [ run char_sep_example_1.cpp ]
[ run char_sep_example_2.cpp ]
[ run char_sep_example_3.cpp ]
;

View File

@@ -0,0 +1,28 @@
// (c) Copyright Jeremy Siek 2002.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// Sample output:
//
// <Hello> <world> <foo> <bar> <yow> <baz>
// char_sep_example_1.cpp
#include <iostream>
#include <boost/tokenizer.hpp>
#include <string>
int main()
{
std::string str = ";;Hello|world||-foo--bar;yow;baz|";
typedef boost::tokenizer<boost::char_separator<char> >
tokenizer;
boost::char_separator<char> sep("-;|");
tokenizer tokens(str, sep);
for (tokenizer::iterator tok_iter = tokens.begin();
tok_iter != tokens.end(); ++tok_iter)
std::cout << "<" << *tok_iter << "> ";
std::cout << "\n";
return EXIT_SUCCESS;
}

View File

@@ -0,0 +1,29 @@
// (c) Copyright Jeremy Siek 2002.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// Sample output:
//
// <> <> <Hello> <|> <world> <|> <> <|> <> <foo> <> <bar> <yow> <baz> <|> <>
// char_sep_example_2.cpp
#include <iostream>
#include <boost/tokenizer.hpp>
#include <string>
int main()
{
std::string str = ";;Hello|world||-foo--bar;yow;baz|";
typedef boost::tokenizer<boost::char_separator<char> >
tokenizer;
boost::char_separator<char> sep("-;", "|", boost::keep_empty_tokens);
tokenizer tokens(str, sep);
for (tokenizer::iterator tok_iter = tokens.begin();
tok_iter != tokens.end(); ++tok_iter)
std::cout << "<" << *tok_iter << "> ";
std::cout << "\n";
return EXIT_SUCCESS;
}

View File

@@ -0,0 +1,26 @@
// (c) Copyright Jeremy Siek 2002.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// Sample output:
// <This> <is> <,> <a> <test>
// char_sep_example_3.cpp
#include <iostream>
#include <boost/tokenizer.hpp>
#include <string>
int main()
{
std::string str = "This is, a test";
typedef boost::tokenizer<boost::char_separator<char> > Tok;
boost::char_separator<char> sep; // default constructed
Tok tok(str, sep);
for(Tok::iterator tok_iter = tok.begin(); tok_iter != tok.end(); ++tok_iter)
std::cout << "<" << *tok_iter << "> ";
std::cout << "\n";
return EXIT_SUCCESS;
}

View File

@@ -0,0 +1,653 @@
// Boost token_functions.hpp ------------------------------------------------//
// Copyright John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer/ for documentation.
// Revision History:
// 01 Oct 2004 Joaquin M Lopez Munoz
// Workaround for a problem with string::assign in msvc-stlport
// 06 Apr 2004 John Bandela
// Fixed a bug involving using char_delimiter with a true input iterator
// 28 Nov 2003 Robert Zeh and John Bandela
// Converted into "fast" functions that avoid using += when
// the supplied iterator isn't an input_iterator; based on
// some work done at Archelon and a version that was checked into
// the boost CVS for a short period of time.
// 20 Feb 2002 John Maddock
// Removed using namespace std declarations and added
// workaround for BOOST_NO_STDC_NAMESPACE (the library
// can be safely mixed with regex).
// 06 Feb 2002 Jeremy Siek
// Added char_separator.
// 02 Feb 2002 Jeremy Siek
// Removed tabs and a little cleanup.
#ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
#define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
#include <vector>
#include <stdexcept>
#include <string>
#include <cctype>
#include <algorithm> // for find_if
#include <boost/config.hpp>
#include <boost/assert.hpp>
#include <boost/type_traits/is_pointer.hpp>
#include <boost/detail/workaround.hpp>
#include <boost/mpl/if.hpp>
#include <boost/throw_exception.hpp>
#if !defined(BOOST_NO_CWCTYPE)
#include <cwctype>
#endif
//
// the following must not be macros if we are to prefix them
// with std:: (they shouldn't be macros anyway...)
//
#ifdef ispunct
# undef ispunct
#endif
#ifdef iswpunct
# undef iswpunct
#endif
#ifdef isspace
# undef isspace
#endif
#ifdef iswspace
# undef iswspace
#endif
//
// fix namespace problems:
//
#ifdef BOOST_NO_STDC_NAMESPACE
namespace std{
using ::ispunct;
using ::isspace;
#if !defined(BOOST_NO_CWCTYPE)
using ::iswpunct;
using ::iswspace;
#endif
}
#endif
namespace boost{
//===========================================================================
// The escaped_list_separator class. Which is a model of TokenizerFunction
// An escaped list is a super-set of what is commonly known as a comma
// separated value (csv) list.It is separated into fields by a comma or
// other character. If the delimiting character is inside quotes, then it is
// counted as a regular character.To allow for embedded quotes in a field,
// there can be escape sequences using the \ much like C.
// The role of the comma, the quotation mark, and the escape
// character (backslash \), can be assigned to other characters.
struct escaped_list_error : public std::runtime_error{
escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
};
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
// MSVC does not like the following typename
template <class Char,
class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
class escaped_list_separator {
private:
typedef std::basic_string<Char,Traits> string_type;
struct char_eq {
Char e_;
char_eq(Char e):e_(e) { }
bool operator()(Char c) {
return Traits::eq(e_,c);
}
};
string_type escape_;
string_type c_;
string_type quote_;
bool last_;
bool is_escape(Char e) {
char_eq f(e);
return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
}
bool is_c(Char e) {
char_eq f(e);
return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
}
bool is_quote(Char e) {
char_eq f(e);
return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
}
template <typename iterator, typename Token>
void do_escape(iterator& next,iterator end,Token& tok) {
if (++next == end)
BOOST_THROW_EXCEPTION(escaped_list_error(std::string("cannot end with escape")));
if (Traits::eq(*next,'n')) {
tok+='\n';
return;
}
else if (is_quote(*next)) {
tok+=*next;
return;
}
else if (is_c(*next)) {
tok+=*next;
return;
}
else if (is_escape(*next)) {
tok+=*next;
return;
}
else
BOOST_THROW_EXCEPTION(escaped_list_error(std::string("unknown escape sequence")));
}
public:
explicit escaped_list_separator(Char e = '\\',
Char c = ',',Char q = '\"')
: escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
escaped_list_separator(string_type e, string_type c, string_type q)
: escape_(e), c_(c), quote_(q), last_(false) { }
void reset() {last_=false;}
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next,InputIterator end,Token& tok) {
bool bInQuote = false;
tok = Token();
if (next == end) {
if (last_) {
last_ = false;
return true;
}
else
return false;
}
last_ = false;
for (;next != end;++next) {
if (is_escape(*next)) {
do_escape(next,end,tok);
}
else if (is_c(*next)) {
if (!bInQuote) {
// If we are not in quote, then we are done
++next;
// The last character was a c, that means there is
// 1 more blank field
last_ = true;
return true;
}
else tok+=*next;
}
else if (is_quote(*next)) {
bInQuote=!bInQuote;
}
else {
tok += *next;
}
}
return true;
}
};
//===========================================================================
// The classes here are used by offset_separator and char_separator to implement
// faster assigning of tokens using assign instead of +=
namespace tokenizer_detail {
//===========================================================================
// Tokenizer was broken for wide character separators, at least on Windows, since
// CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
// if higher values are passed in. The traits extension class should take care of this.
// Assuming that the conditional will always get optimized out in the function
// implementations, argument types are not a problem since both forms of character classifiers
// expect an int.
#if !defined(BOOST_NO_CWCTYPE)
template<typename traits, int N>
struct traits_extension_details : public traits {
typedef typename traits::char_type char_type;
static bool isspace(char_type c)
{
return std::iswspace(c) != 0;
}
static bool ispunct(char_type c)
{
return std::iswpunct(c) != 0;
}
};
template<typename traits>
struct traits_extension_details<traits, 1> : public traits {
typedef typename traits::char_type char_type;
static bool isspace(char_type c)
{
return std::isspace(c) != 0;
}
static bool ispunct(char_type c)
{
return std::ispunct(c) != 0;
}
};
#endif
// In case there is no cwctype header, we implement the checks manually.
// We make use of the fact that the tested categories should fit in ASCII.
template<typename traits>
struct traits_extension : public traits {
typedef typename traits::char_type char_type;
static bool isspace(char_type c)
{
#if !defined(BOOST_NO_CWCTYPE)
return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
#else
return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
#endif
}
static bool ispunct(char_type c)
{
#if !defined(BOOST_NO_CWCTYPE)
return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
#else
return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
#endif
}
};
// The assign_or_plus_equal struct contains functions that implement
// assign, +=, and clearing based on the iterator type. The
// generic case does nothing for plus_equal and clearing, while
// passing through the call for assign.
//
// When an input iterator is being used, the situation is reversed.
// The assign method does nothing, plus_equal invokes operator +=,
// and the clearing method sets the supplied token to the default
// token constructor's result.
//
template<class IteratorTag>
struct assign_or_plus_equal {
template<class Iterator, class Token>
static void assign(Iterator b, Iterator e, Token &t) {
t.assign(b, e);
}
template<class Token, class Value>
static void plus_equal(Token &, const Value &) { }
// If we are doing an assign, there is no need for the
// the clear.
//
template<class Token>
static void clear(Token &) { }
};
template <>
struct assign_or_plus_equal<std::input_iterator_tag> {
template<class Iterator, class Token>
static void assign(Iterator , Iterator , Token &) { }
template<class Token, class Value>
static void plus_equal(Token &t, const Value &v) {
t += v;
}
template<class Token>
static void clear(Token &t) {
t = Token();
}
};
template<class Iterator>
struct pointer_iterator_category{
typedef std::random_access_iterator_tag type;
};
template<class Iterator>
struct class_iterator_category{
typedef typename Iterator::iterator_category type;
};
// This portably gets the iterator_tag without partial template specialization
template<class Iterator>
struct get_iterator_category{
typedef typename mpl::if_<is_pointer<Iterator>,
pointer_iterator_category<Iterator>,
class_iterator_category<Iterator>
>::type cat;
typedef typename cat::type iterator_category;
};
} // namespace tokenizer_detail
//===========================================================================
// The offset_separator class, which is a model of TokenizerFunction.
// Offset breaks a string into tokens based on a range of offsets
class offset_separator {
private:
std::vector<int> offsets_;
unsigned int current_offset_;
bool wrap_offsets_;
bool return_partial_last_;
public:
template <typename Iter>
offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
bool return_partial_last = true)
: offsets_(begin,end), current_offset_(0),
wrap_offsets_(wrap_offsets),
return_partial_last_(return_partial_last) { }
offset_separator()
: offsets_(1,1), current_offset_(),
wrap_offsets_(true), return_partial_last_(true) { }
void reset() {
current_offset_ = 0;
}
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next, InputIterator end, Token& tok)
{
typedef tokenizer_detail::assign_or_plus_equal<
BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
InputIterator
>::iterator_category
> assigner;
BOOST_ASSERT(!offsets_.empty());
assigner::clear(tok);
InputIterator start(next);
if (next == end)
return false;
if (current_offset_ == offsets_.size())
{
if (wrap_offsets_)
current_offset_=0;
else
return false;
}
int c = offsets_[current_offset_];
int i = 0;
for (; i < c; ++i) {
if (next == end)break;
assigner::plus_equal(tok,*next++);
}
assigner::assign(start,next,tok);
if (!return_partial_last_)
if (i < (c-1) )
return false;
++current_offset_;
return true;
}
};
//===========================================================================
// The char_separator class breaks a sequence of characters into
// tokens based on the character delimiters (very much like bad old
// strtok). A delimiter character can either be kept or dropped. A
// kept delimiter shows up as an output token, whereas a dropped
// delimiter does not.
// This class replaces the char_delimiters_separator class. The
// constructor for the char_delimiters_separator class was too
// confusing and needed to be deprecated. However, because of the
// default arguments to the constructor, adding the new constructor
// would cause ambiguity, so instead I deprecated the whole class.
// The implementation of the class was also simplified considerably.
enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
template <typename Char,
typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
class char_separator
{
typedef tokenizer_detail::traits_extension<Tr> Traits;
typedef std::basic_string<Char,Tr> string_type;
public:
explicit
char_separator(const Char* dropped_delims,
const Char* kept_delims = 0,
empty_token_policy empty_tokens = drop_empty_tokens)
: m_dropped_delims(dropped_delims),
m_use_ispunct(false),
m_use_isspace(false),
m_empty_tokens(empty_tokens),
m_output_done(false)
{
// Borland workaround
if (kept_delims)
m_kept_delims = kept_delims;
}
// use ispunct() for kept delimiters and isspace for dropped.
explicit
char_separator()
: m_use_ispunct(true),
m_use_isspace(true),
m_empty_tokens(drop_empty_tokens),
m_output_done(false) { }
void reset() { }
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next, InputIterator end, Token& tok)
{
typedef tokenizer_detail::assign_or_plus_equal<
BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
InputIterator
>::iterator_category
> assigner;
assigner::clear(tok);
// skip past all dropped_delims
if (m_empty_tokens == drop_empty_tokens)
for (; next != end && is_dropped(*next); ++next)
{ }
InputIterator start(next);
if (m_empty_tokens == drop_empty_tokens) {
if (next == end)
return false;
// if we are on a kept_delims move past it and stop
if (is_kept(*next)) {
assigner::plus_equal(tok,*next);
++next;
} else
// append all the non delim characters
for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
assigner::plus_equal(tok,*next);
}
else { // m_empty_tokens == keep_empty_tokens
// Handle empty token at the end
if (next == end)
{
if (m_output_done == false)
{
m_output_done = true;
assigner::assign(start,next,tok);
return true;
}
else
return false;
}
if (is_kept(*next)) {
if (m_output_done == false)
m_output_done = true;
else {
assigner::plus_equal(tok,*next);
++next;
m_output_done = false;
}
}
else if (m_output_done == false && is_dropped(*next)) {
m_output_done = true;
}
else {
if (is_dropped(*next))
start=++next;
for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
assigner::plus_equal(tok,*next);
m_output_done = true;
}
}
assigner::assign(start,next,tok);
return true;
}
private:
string_type m_kept_delims;
string_type m_dropped_delims;
bool m_use_ispunct;
bool m_use_isspace;
empty_token_policy m_empty_tokens;
bool m_output_done;
bool is_kept(Char E) const
{
if (m_kept_delims.length())
return m_kept_delims.find(E) != string_type::npos;
else if (m_use_ispunct) {
return Traits::ispunct(E) != 0;
} else
return false;
}
bool is_dropped(Char E) const
{
if (m_dropped_delims.length())
return m_dropped_delims.find(E) != string_type::npos;
else if (m_use_isspace) {
return Traits::isspace(E) != 0;
} else
return false;
}
};
//===========================================================================
// The following class is DEPRECATED, use class char_separators instead.
//
// The char_delimiters_separator class, which is a model of
// TokenizerFunction. char_delimiters_separator breaks a string
// into tokens based on character delimiters. There are 2 types of
// delimiters. returnable delimiters can be returned as
// tokens. These are often punctuation. nonreturnable delimiters
// cannot be returned as tokens. These are often whitespace
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
template <class Char,
class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
class char_delimiters_separator {
private:
typedef tokenizer_detail::traits_extension<Tr> Traits;
typedef std::basic_string<Char,Tr> string_type;
string_type returnable_;
string_type nonreturnable_;
bool return_delims_;
bool no_ispunct_;
bool no_isspace_;
bool is_ret(Char E)const
{
if (returnable_.length())
return returnable_.find(E) != string_type::npos;
else{
if (no_ispunct_) {return false;}
else{
int r = Traits::ispunct(E);
return r != 0;
}
}
}
bool is_nonret(Char E)const
{
if (nonreturnable_.length())
return nonreturnable_.find(E) != string_type::npos;
else{
if (no_isspace_) {return false;}
else{
int r = Traits::isspace(E);
return r != 0;
}
}
}
public:
explicit char_delimiters_separator(bool return_delims = false,
const Char* returnable = 0,
const Char* nonreturnable = 0)
: returnable_(returnable ? returnable : string_type().c_str()),
nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
return_delims_(return_delims), no_ispunct_(returnable!=0),
no_isspace_(nonreturnable!=0) { }
void reset() { }
public:
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next, InputIterator end,Token& tok) {
tok = Token();
// skip past all nonreturnable delims
// skip past the returnable only if we are not returning delims
for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
&& !return_delims_ ) );++next) { }
if (next == end) {
return false;
}
// if we are to return delims and we are one a returnable one
// move past it and stop
if (is_ret(*next) && return_delims_) {
tok+=*next;
++next;
}
else
// append all the non delim characters
for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)
tok+=*next;
return true;
}
};
} //namespace boost
#endif

View File

@@ -0,0 +1,131 @@
// Boost token_iterator.hpp -------------------------------------------------//
// Copyright John R. Bandela 2001
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer for documentation.
// Revision History:
// 16 Jul 2003 John Bandela
// Allowed conversions from convertible base iterators
// 03 Jul 2003 John Bandela
// Converted to new iterator adapter
#ifndef BOOST_TOKENIZER_POLICY_JRB070303_HPP_
#define BOOST_TOKENIZER_POLICY_JRB070303_HPP_
#include <boost/assert.hpp>
#include <boost/iterator/iterator_adaptor.hpp>
#include <boost/iterator/minimum_category.hpp>
#include <boost/token_functions.hpp>
#include <utility>
namespace boost
{
template <class TokenizerFunc, class Iterator, class Type>
class token_iterator
: public iterator_facade<
token_iterator<TokenizerFunc, Iterator, Type>
, Type
, typename iterators::minimum_category<
forward_traversal_tag
, typename iterator_traversal<Iterator>::type
>::type
, const Type&
>
{
#ifdef __DCC__
friend class boost::iterator_core_access;
#else
friend class iterator_core_access;
#endif
TokenizerFunc f_;
Iterator begin_;
Iterator end_;
bool valid_;
Type tok_;
void increment(){
BOOST_ASSERT(valid_);
valid_ = f_(begin_,end_,tok_);
}
const Type& dereference() const {
BOOST_ASSERT(valid_);
return tok_;
}
template<class Other>
bool equal(const Other& a) const{
return (a.valid_ && valid_)
?( (a.begin_==begin_) && (a.end_ == end_) )
:(a.valid_==valid_);
}
void initialize(){
if(valid_) return;
f_.reset();
valid_ = (begin_ != end_)?
f_(begin_,end_,tok_):false;
}
public:
token_iterator():begin_(),end_(),valid_(false),tok_() { }
token_iterator(TokenizerFunc f, Iterator begin, Iterator e = Iterator())
: f_(f),begin_(begin),end_(e),valid_(false),tok_(){ initialize(); }
token_iterator(Iterator begin, Iterator e = Iterator())
: f_(),begin_(begin),end_(e),valid_(false),tok_() {initialize();}
template<class OtherIter>
token_iterator(
token_iterator<TokenizerFunc, OtherIter,Type> const& t
, typename enable_if_convertible<OtherIter, Iterator>::type* = 0)
: f_(t.tokenizer_function()),begin_(t.base())
,end_(t.end()),valid_(!t.at_end()),tok_(t.current_token()) {}
Iterator base()const{return begin_;}
Iterator end()const{return end_;}
TokenizerFunc tokenizer_function()const{return f_;}
Type current_token()const{return tok_;}
bool at_end()const{return !valid_;}
};
template <
class TokenizerFunc = char_delimiters_separator<char>,
class Iterator = std::string::const_iterator,
class Type = std::string
>
class token_iterator_generator {
private:
public:
typedef token_iterator<TokenizerFunc,Iterator,Type> type;
};
// Type has to be first because it needs to be explicitly specified
// because there is no way the function can deduce it.
template<class Type, class Iterator, class TokenizerFunc>
typename token_iterator_generator<TokenizerFunc,Iterator,Type>::type
make_token_iterator(Iterator begin, Iterator end,const TokenizerFunc& fun){
typedef typename
token_iterator_generator<TokenizerFunc,Iterator,Type>::type ret_type;
return ret_type(fun,begin,end);
}
} // namespace boost
#endif

View File

@@ -0,0 +1,98 @@
// Boost tokenizer.hpp -----------------------------------------------------//
// (c) Copyright Jeremy Siek and John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer for documenation
// Revision History:
// 03 Jul 2003 John Bandela
// Converted to new iterator adapter
// 02 Feb 2002 Jeremy Siek
// Removed tabs and a little cleanup.
#ifndef BOOST_TOKENIZER_JRB070303_HPP_
#define BOOST_TOKENIZER_JRB070303_HPP_
#include <boost/token_iterator.hpp>
namespace boost {
//===========================================================================
// A container-view of a tokenized "sequence"
template <
typename TokenizerFunc = char_delimiters_separator<char>,
typename Iterator = std::string::const_iterator,
typename Type = std::string
>
class tokenizer {
private:
typedef token_iterator_generator<TokenizerFunc,Iterator,Type> TGen;
// It seems that MSVC does not like the unqualified use of iterator,
// Thus we use iter internally when it is used unqualified and
// the users of this class will always qualify iterator.
typedef typename TGen::type iter;
public:
typedef iter iterator;
typedef iter const_iterator;
typedef Type value_type;
typedef value_type& reference;
typedef const value_type& const_reference;
typedef value_type* pointer;
typedef const pointer const_pointer;
typedef void size_type;
typedef void difference_type;
tokenizer(Iterator first, Iterator last,
const TokenizerFunc& f = TokenizerFunc())
: first_(first), last_(last), f_(f) { }
template <typename Container>
tokenizer(const Container& c)
: first_(c.begin()), last_(c.end()), f_() { }
template <typename Container>
tokenizer(const Container& c,const TokenizerFunc& f)
: first_(c.begin()), last_(c.end()), f_(f) { }
void assign(Iterator first, Iterator last){
first_ = first;
last_ = last;
}
void assign(Iterator first, Iterator last, const TokenizerFunc& f){
assign(first,last);
f_ = f;
}
template <typename Container>
void assign(const Container& c){
assign(c.begin(),c.end());
}
template <typename Container>
void assign(const Container& c, const TokenizerFunc& f){
assign(c.begin(),c.end(),f);
}
iter begin() const { return iter(f_,first_,last_); }
iter end() const { return iter(f_,last_,last_); }
private:
Iterator first_;
Iterator last_;
TokenizerFunc f_;
};
} // namespace boost
#endif

View File

@@ -0,0 +1,13 @@
<html>
<head>
<meta http-equiv="refresh" content="0; URL=doc/index.html">
</head>
<body>
Automatic redirection failed, please go to
<a href="doc/index.html">doc/index.html</a>.&nbsp;<hr>
<p><EFBFBD> Copyright Beman Dawes, 2001</p>
<p>Distributed under the Boost Software License, Version 1.0. (See accompanying
file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or copy
at <a href="http://www.boost.org/LICENSE_1_0.txt">www.boost.org/LICENSE_1_0.txt</a>)</p>
</body>
</html>

View File

@@ -0,0 +1,16 @@
{
"key": "tokenizer",
"name": "Tokenizer",
"authors": [
"John Bandela"
],
"description": "Break of a string or other character sequence into a series of tokens.",
"category": [
"Iterators",
"String"
],
"maintainers": [
"John R. Bandela <jbandela -at- ufl.edu>"
],
"cxxstd": "03"
}

View File

@@ -0,0 +1,12 @@
#~ Copyright Rene Rivera 2008
#~ Distributed under the Boost Software License, Version 1.0.
#~ (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
import testing ;
run examples.cpp ;
run simple_example_1.cpp ;
run simple_example_2.cpp ;
run simple_example_3.cpp ;
run simple_example_4.cpp ;
run simple_example_5.cpp ;

View File

@@ -0,0 +1,149 @@
// Boost tokenizer examples -------------------------------------------------//
// (c) Copyright John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org for updates, documentation, and revision history.
#include <iostream>
#include <iterator>
#include <string>
#include <algorithm>
#include <boost/tokenizer.hpp>
#include <boost/array.hpp>
#include <boost/core/lightweight_test.hpp>
int main()
{
using namespace boost;
// Use tokenizer
{
const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|";
std::string answer[] = { "Hello", "world", "foo", "bar", "yow", "baz" };
typedef tokenizer<char_separator<char> > Tok;
char_separator<char> sep("-;|");
Tok t(test_string, sep);
BOOST_TEST(std::equal(t.begin(),t.end(),answer));
}
{
const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|";
std::string answer[] = { "", "", "Hello", "|", "world", "|", "", "|", "",
"foo", "", "bar", "yow", "baz", "|", "" };
typedef tokenizer<char_separator<char> > Tok;
char_separator<char> sep("-;", "|", boost::keep_empty_tokens);
Tok t(test_string, sep);
BOOST_TEST(std::equal(t.begin(), t.end(), answer));
}
{
const std::string test_string = "This,,is, a.test..";
std::string answer[] = {"This","is","a","test"};
typedef tokenizer<> Tok;
Tok t(test_string);
BOOST_TEST(std::equal(t.begin(),t.end(),answer));
}
{
const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
std::string answer[] = {"Field 1","embedded,comma","quote \""," escape \\"};
typedef tokenizer<escaped_list_separator<char> > Tok;
Tok t(test_string);
BOOST_TEST(std::equal(t.begin(),t.end(),answer));
}
{
const std::string test_string = ",1,;2\\\";3\\;,4,5^\\,\'6,7\';";
std::string answer[] = {"","1","","2\"","3;","4","5\\","6,7",""};
typedef tokenizer<escaped_list_separator<char> > Tok;
escaped_list_separator<char> sep("\\^",",;","\"\'");
Tok t(test_string,sep);
BOOST_TEST(std::equal(t.begin(),t.end(),answer));
}
{
const std::string test_string = "12252001";
std::string answer[] = {"12","25","2001"};
typedef tokenizer<offset_separator > Tok;
boost::array<int,3> offsets = {{2,2,4}};
offset_separator func(offsets.begin(),offsets.end());
Tok t(test_string,func);
BOOST_TEST(std::equal(t.begin(),t.end(),answer));
}
// Use token_iterator_generator
{
const std::string test_string = "This,,is, a.test..";
std::string answer[] = {"This","is","a","test"};
typedef token_iterator_generator<char_delimiters_separator<char> >::type Iter;
Iter begin = make_token_iterator<std::string>(test_string.begin(),
test_string.end(),char_delimiters_separator<char>());
Iter end;
BOOST_TEST(std::equal(begin,end,answer));
}
{
const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
std::string answer[] = {"Field 1","embedded,comma","quote \""," escape \\"};
typedef token_iterator_generator<escaped_list_separator<char> >::type Iter;
Iter begin = make_token_iterator<std::string>(test_string.begin(),
test_string.end(),escaped_list_separator<char>());
Iter begin_c(begin);
Iter end;
BOOST_TEST(std::equal(begin,end,answer));
while(begin_c != end)
{
BOOST_TEST(begin_c.at_end() == 0);
++begin_c;
}
BOOST_TEST(begin_c.at_end());
}
{
const std::string test_string = "12252001";
std::string answer[] = {"12","25","2001"};
typedef token_iterator_generator<offset_separator>::type Iter;
boost::array<int,3> offsets = {{2,2,4}};
offset_separator func(offsets.begin(),offsets.end());
Iter begin = make_token_iterator<std::string>(test_string.begin(),
test_string.end(),func);
Iter end= make_token_iterator<std::string>(test_string.end(),
test_string.end(),func);
BOOST_TEST(std::equal(begin,end,answer));
}
// Test copying
{
const std::string test_string = "abcdef";
token_iterator_generator<offset_separator>::type beg, end, other;
boost::array<int,3> ar = {{1,2,3}};
offset_separator f(ar.begin(),ar.end());
beg = make_token_iterator<std::string>(test_string.begin(),test_string.end(),f);
++beg;
other = beg;
++other;
BOOST_TEST(*beg=="bc");
BOOST_TEST(*other=="def");
other = make_token_iterator<std::string>(test_string.begin(),
test_string.end(),f);
BOOST_TEST(*other=="a");
}
// Test non-default constructed char_delimiters_separator
{
const std::string test_string = "how,are you, doing";
std::string answer[] = {"how",",","are you",","," doing"};
tokenizer<> t(test_string,char_delimiters_separator<char>(true,",",""));
BOOST_TEST(std::equal(t.begin(),t.end(),answer));
}
return boost::report_errors();
}

View File

@@ -0,0 +1,25 @@
// (c) Copyright John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer for documenation
// simple_example_1.cpp
#include<iostream>
#include<boost/tokenizer.hpp>
#include<string>
int main(){
using namespace std;
using namespace boost;
string s = "This is, a test";
tokenizer<> tok(s);
for(tokenizer<>::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout << *beg << "\n";
}
return 0;
}

View File

@@ -0,0 +1,24 @@
// (c) Copyright John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer for documenation
// simple_example_2.cpp
#include<iostream>
#include<boost/tokenizer.hpp>
#include<string>
int main(){
using namespace std;
using namespace boost;
string s = "Field 1,\"putting quotes around fields, allows commas\",Field 3";
tokenizer<escaped_list_separator<char> > tok(s);
for(tokenizer<escaped_list_separator<char> >::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout << *beg << "\n";
}
return 0;
}

View File

@@ -0,0 +1,25 @@
// (c) Copyright John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer for documenation
// simple_example_3.cpp
#include<iostream>
#include<boost/tokenizer.hpp>
#include<string>
int main(){
using namespace std;
using namespace boost;
string s = "12252001";
int offsets[] = {2,2,4};
offset_separator f(offsets, offsets+3);
tokenizer<offset_separator> tok(s,f);
for(tokenizer<offset_separator>::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout << *beg << "\n";
}
return 0;
}

View File

@@ -0,0 +1,24 @@
// (c) Copyright John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer for documenation
// simple_example_4.cpp
#include<iostream>
#include<boost/tokenizer.hpp>
#include<string>
int main(){
using namespace std;
using namespace boost;
string s = "This is, a test";
tokenizer<char_delimiters_separator<char> > tok(s);
for(tokenizer<char_delimiters_separator<char> >::iterator beg=tok.begin(); beg!=tok.end();++beg){
cout << *beg << "\n";
}
return 0;
}

View File

@@ -0,0 +1,34 @@
// (c) Copyright John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer for documenation
/// simple_example_5.cpp
#include<iostream>
#include<boost/token_iterator.hpp>
#include<string>
#ifdef BOOST_BORLANDC
// compiler bug fix:
template class boost::token_iterator_generator<boost::offset_separator>::type;
#endif
int main(){
using namespace std;
using namespace boost;
string s = "12252001";
int offsets[] = {2,2,4};
offset_separator f(offsets, offsets+3);
typedef token_iterator_generator<offset_separator>::type Iter;
Iter beg = make_token_iterator<string>(s.begin(),s.end(),f);
Iter end = make_token_iterator<string>(s.end(),s.end(),f);
// The above statement could also have been what is below
// Iter end;
for(;beg!=end;++beg){
cout << *beg << "\n";
}
return 0;
}

View File

@@ -0,0 +1,7 @@
-- Downloading https://github.com/boostorg/tokenizer/archive/boost-1.79.0.tar.gz -> boostorg-tokenizer-boost-1.79.0.tar.gz...
-- Extracting source D:/a/1/s/externals/vcpkg/downloads/boostorg-tokenizer-boost-1.79.0.tar.gz
-- Using source at D:/a/1/s/externals/vcpkg/buildtrees/boost-tokenizer/src/ost-1.79.0-3b3fcb0fc7.clean
-- Copying headers
-- Copying headers done
-- Installing: D:/a/1/s/externals/vcpkg/packages/boost-tokenizer_x64-windows/share/boost-tokenizer/usage
-- Installing: D:/a/1/s/externals/vcpkg/packages/boost-tokenizer_x64-windows/share/boost-tokenizer/copyright

View File

@@ -0,0 +1,19 @@
boost-assert a50eed453b8be6c8932fb3d5f8feaf194a2ebeaed7982db4e36e3ba17f3ec107
boost-config 797535e8975ed7cf5bbe11d9f7fe26caa5da8fe819888564758d82a21109fade
boost-core 498aea0b6b68bcfe1ec683e76c2f0d32477dfe9ba958f518980ff806b6faba90
boost-iterator 198407720ab08cc171b7d0c2a7640ae035c7aee5a21a0e1d3c185c4b8c495f30
boost-mpl 89695bf75ab1fa2b706b2a0c2ef28c6726c3abd61ff51fb0966e3270153a6cd9
boost-throw-exception 4bb4a0182f0f071c3c1ffd9cbd82fbc3bc7db4a35346f930bbe504e7e3a94e0a
boost-type-traits 74f62124585467fbb6c4fa16015164d11e1a079d6bdb70ec1c3fe7cf65b9a594
boost-vcpkg-helpers c81c7b003df356a1a120a7c0c2f5a2ac95f3c33b006a2a5b4c02dcf0c9f3deaa
cmake 3.23.2
features core
portfile.cmake 80a864bab9c30c2c13231e1c4bdae3cf74f03055e6d5693d3c3f8183fc362355
ports.cmake 366c60b768113102408b32ac1d7c7b48ef7d30a477af2a220ecc222d9ffa3166
post_build_checks 2
powershell 7.2.5
triplet x64-windows
triplet_abi 4556164a2cd3dd6f4742101eabb46def7e71b6e5856faa88e5d005aac12a803c-c0600b35e024ce0485ed253ef5419f3686f7257cfb58cb6a24febcb600fc4b4c-27ebd443f77a6c449168adfa6ce8def60cf46e88
vcpkg.json 46b4f7b9b3566a927242460a4f482ae534cc4c960fddab351a00cd0798fc3f92
vcpkg_from_git 0aab20e34e84d52ba4763f009e539bfa8f418c41c918c8cf700156f1a8551a10
vcpkg_from_github b743742296a114ea1b18ae99672e02f142c4eb2bef7f57d36c038bedbfb0502f