A simple tokenizer for PDF files and PDF content streams. More...

#include <PdfTokenizer.h>

Inheritance diagram for PoDoFo::PdfTokenizer:

Classes
struct	ParsingOptions

Public Member Functions
	PdfTokenizer (std::shared_ptr< charbuff > buffer)

	PdfTokenizer (const PdfTokenizerOptions &options)

	PdfTokenizer (std::shared_ptr< charbuff > buffer, const PdfTokenizerOptions &options)

bool	TryReadNextToken (InputStreamDevice &device, std::string_view &token)
	Reads the next token from the current file position ignoring all comments.

bool	TryReadNextToken (InputStreamDevice &device, std::string_view &token, PdfTokenType &tokenType)

bool	TryPeekNextToken (InputStreamDevice &device, std::string_view &token)
	Try peek the next token from the current file position ignoring all comments, without actually consuming it.

bool	TryPeekNextToken (InputStreamDevice &device, std::string_view &token, PdfTokenType &tokenType)

int64_t	ReadNextNumber (InputStreamDevice &device)
	Read the next number from the current file position ignoring all comments.

bool	TryReadNextNumber (InputStreamDevice &device, int64_t &value)

void	ReadNextVariant (InputStreamDevice &device, PdfVariant &variant, const PdfStatefulEncrypt *encrypt={ })
	Read the next variant from the current file position ignoring all comments.

bool	TryReadNextVariant (InputStreamDevice &device, PdfVariant &variant, const PdfStatefulEncrypt *encrypt={ })

void	Reset ()

void	SetParameters (const PdfTokenizerParams &params)

const PdfTokenizerParams &	GetParameters () const

Static Public Attributes
static constexpr unsigned	BufferSize = 4096

static constexpr size_t	MaxStringLength = 64 * 1024 * 1024

Protected Types
enum class	PdfLiteralDataType : uint8_t { Unknown = 0 , Bool , Number , Real , String , HexString , Name , Array , Dictionary , Null , Reference }

Protected Member Functions
void	ReadNextVariant (InputStreamDevice &device, const std::string_view &token, PdfTokenType tokenType, PdfVariant &variant, const PdfStatefulEncrypt *encrypt)
	Read the next variant from the current file position ignoring all comments.

bool	TryReadNextVariant (InputStreamDevice &device, const std::string_view &token, PdfTokenType tokenType, PdfVariant &variant, const PdfStatefulEncrypt *encrypt)

void	EnqueueToken (const std::string_view &token, PdfTokenType type)
	Add a token to the queue of tokens.

bool	ReadDictionary (InputStreamDevice &device, PdfVariant &variant, const PdfStatefulEncrypt *encrypt, ParsingOptions opts)
	Read a dictionary from the input device and store it into a variant.

bool	ReadArray (InputStreamDevice &device, PdfVariant &variant, const PdfStatefulEncrypt *encrypt, ParsingOptions opts)
	Read an array from the input device and store it into a variant.

bool	ReadString (InputStreamDevice &device, PdfVariant &variant, const PdfStatefulEncrypt *encrypt, ParsingOptions opts)
	Read a string from the input device and store it into a variant.

bool	ReadHexString (InputStreamDevice &device, PdfVariant &variant, const PdfStatefulEncrypt *encrypt, ParsingOptions opts)
	Read a hex string from the input device and store it into a variant.

bool	ReadName (InputStreamDevice &device, PdfVariant &variant, ParsingOptions opts)
	Read a name from the input device and store it into a variant.

PdfLiteralDataType	DetermineDataType (InputStreamDevice &device, const std::string_view &token, PdfTokenType tokenType, PdfVariant &variant, ParsingOptions opts)
	Determine the possible datatype of a token.

Friends
class	PdfParser

class	PdfPostScriptTokenizer

Detailed Description

A simple tokenizer for PDF files and PDF content streams.

Member Function Documentation

◆ DetermineDataType()

PdfTokenizer::PdfLiteralDataType PdfTokenizer::DetermineDataType	(	InputStreamDevice &	device,
		const std::string_view &	token,
		PdfTokenType	tokenType,
		PdfVariant &	variant,
		ParsingOptions	opts
	)

protected

Determine the possible datatype of a token.

Numbers, reals, bools or nullptr values are parsed directly by this function and saved to a variant.

Parameters

opts	per-call flags controlling throw-vs-log-and-return-false semantics

Returns: the expected datatype, or PdfLiteralDataType::Unknown when ThrowOnError is unset and the token could not be classified

◆ EnqueueToken()

void PdfTokenizer::EnqueueToken	(	const std::string_view &	token,
		PdfTokenType	type
	)

protected

Add a token to the queue of tokens.

tryReadNextToken() will return all enqueued tokens first before reading new tokens from the input device.

Parameters

token	string of the token
type	type of the token

See also: tryReadNextToken

◆ ReadArray()

bool PdfTokenizer::ReadArray	(	InputStreamDevice &	device,
		PdfVariant &	variant,
		const PdfStatefulEncrypt *	encrypt,
		ParsingOptions	opts
	)

protected

Read an array from the input device and store it into a variant.

Parameters

variant	store the array into this variable
encrypt	an encryption object which is used to decrypt strings during parsing
opts	per-call flags controlling throw-vs-log-and-return-false semantics

Returns: true on success, false only when ThrowOnError is unset and parsing failed

◆ ReadDictionary()

bool PdfTokenizer::ReadDictionary	(	InputStreamDevice &	device,
		PdfVariant &	variant,
		const PdfStatefulEncrypt *	encrypt,
		ParsingOptions	opts
	)

protected

Read a dictionary from the input device and store it into a variant.

Parameters

variant	store the dictionary into this variable
encrypt	an encryption object which is used to decrypt strings during parsing
opts	per-call flags controlling throw-vs-log-and-return-false semantics

Returns: true on success, false only when ThrowOnError is unset and parsing failed

◆ ReadHexString()

bool PdfTokenizer::ReadHexString	(	InputStreamDevice &	device,
		PdfVariant &	variant,
		const PdfStatefulEncrypt *	encrypt,
		ParsingOptions	opts
	)

protected

Read a hex string from the input device and store it into a variant.

Parameters

variant	store the hex string into this variable
encrypt	an encryption object which is used to decrypt strings during parsing
opts	per-call flags controlling throw-vs-log-and-return-false semantics

Returns: true on success, false only when ThrowOnError is unset and parsing failed

◆ ReadName()

bool PdfTokenizer::ReadName	(	InputStreamDevice &	device,
		PdfVariant &	variant,
		ParsingOptions	opts
	)

protected

Read a name from the input device and store it into a variant.

Throws UnexpectedEOF if there is nothing to read.

Parameters

variant	store the name into this variable
device	the input device to read from
opts	per-call flags controlling throw-vs-log-and-return-false semantics

Returns: true on success, false only when ThrowOnError is unset and parsing failed

◆ ReadNextNumber()

int64_t PdfTokenizer::ReadNextNumber ( InputStreamDevice & device )

Read the next number from the current file position ignoring all comments.

Raises NoNumber exception if the next token is no number, and UnexpectedEOF if no token could be read. No token is consumed if NoNumber is thrown.

Returns: a number read from the input device.

◆ ReadNextVariant() [1/2]

void PoDoFo::PdfTokenizer::ReadNextVariant	(	InputStreamDevice &	device,
		const std::string_view &	token,
		PdfTokenType	tokenType,
		PdfVariant &	variant,
		const PdfStatefulEncrypt *	encrypt
	)

protected

Read the next variant from the current file position ignoring all comments.

Raises an exception if there is no variant left in the file.

Parameters

token	a token that has already been read
tokenType	type of the passed token
variant	write the read variant to this value
encrypt	an encryption object which is used to decrypt strings during parsing

◆ ReadNextVariant() [2/2]

void PdfTokenizer::ReadNextVariant	(	InputStreamDevice &	device,
		PdfVariant &	variant,
		const PdfStatefulEncrypt *	encrypt = `{ }`
	)

Read the next variant from the current file position ignoring all comments.

Raises an UnexpectedEOF exception if there is no variant left in the file.

Parameters

variant	write the read variant to this value
encrypt	an encryption object which is used to decrypt strings during parsing

◆ ReadString()

bool PdfTokenizer::ReadString	(	InputStreamDevice &	device,
		PdfVariant &	variant,
		const PdfStatefulEncrypt *	encrypt,
		ParsingOptions	opts
	)

protected

Read a string from the input device and store it into a variant.

Parameters

variant	store the string into this variable
encrypt	an encryption object which is used to decrypt strings during parsing
opts	per-call flags controlling throw-vs-log-and-return-false semantics

Returns: true on success, false only when ThrowOnError is unset and parsing failed

◆ TryPeekNextToken()

bool PoDoFo::PdfTokenizer::TryPeekNextToken	(	InputStreamDevice &	device,
		std::string_view &	token
	)

Try peek the next token from the current file position ignoring all comments, without actually consuming it.

Returns: false if EOF

◆ TryReadNextToken()

bool PoDoFo::PdfTokenizer::TryReadNextToken	(	InputStreamDevice &	device,
		std::string_view &	token
	)

Reads the next token from the current file position ignoring all comments.

Parameters

[out]	token	On true return, set to a pointer to the read token (a nullptr-terminated C string). The pointer is to memory owned by PdfTokenizer and must NOT be freed. The contents are invalidated on the next call to tryReadNextToken(..) and by the destruction of the PdfTokenizer. Undefined on false return.
[out]	tokenType	On true return, if not nullptr the type of the read token will be stored into this parameter. Undefined on false return.

Returns: True if a token was read, false if there are no more tokens to read.

The documentation for this class was generated from the following files:

src/podofo/main/PdfTokenizer.h
src/podofo/main/PdfTokenizer.cpp

Classes

Public Member Functions

Static Public Attributes

Protected Types

Protected Member Functions

Friends

Detailed Description

Member Function Documentation

◆ DetermineDataType()

◆ EnqueueToken()

◆ ReadArray()

◆ ReadDictionary()

◆ ReadHexString()

◆ ReadName()

◆ ReadNextNumber()

◆ ReadNextVariant() [1/2]

◆ ReadNextVariant() [2/2]

◆ ReadString()

◆ TryPeekNextToken()

◆ TryReadNextToken()