PdfTokenizer.h

// SPDX-FileCopyrightText: 2006 Dominik Seichter <domseichter@web.de>
// SPDX-FileCopyrightText: 2020 Francesco Pretto <ceztko@gmail.com>
// SPDX-License-Identifier: LGPL-2.0-or-later OR MPL-2.0
 
#ifndef PDF_TOKENIZER_H
#define PDF_TOKENIZER_H
 
#include "PdfDeclarations.h"
#include <podofo/auxiliary/InputDevice.h>
#include "PdfStatefulEncrypt.h"
 
#include <deque>
 
namespace PoDoFo {
 
class PdfVariant;
 
enum class PdfPostScriptLanguageLevel : uint8_t
{
    L1 = 1,
    L2 = 2,
};
 
enum class PdfTokenizerFlags : uint32_t
{
    None = 0,
    StrictParsing = 1,
    SkipReferences = 2,     
};
 
struct PODOFO_API PdfTokenizerParams final
{
    PdfPostScriptLanguageLevel LanguageLevel = PdfPostScriptLanguageLevel::L2;
    PdfTokenizerFlags Flags = PdfTokenizerFlags::None;
};
 
struct PODOFO_API PdfTokenizerOptions final
{
    PdfPostScriptLanguageLevel LanguageLevel = PdfPostScriptLanguageLevel::L2;
    bool ReadReferences = true;
};
 
class PODOFO_API PdfTokenizer
{
    friend class PdfParser;
    friend class PdfPostScriptTokenizer;
    PODOFO_PRIVATE_FRIEND(class PdfParserObject);
 
public:
    static constexpr unsigned BufferSize = 4096;
    static constexpr size_t MaxStringLength = 64 * 1024 * 1024; // 64 MiB
 
public:
    PdfTokenizer();
    PdfTokenizer(std::shared_ptr<charbuff> buffer);
 
    [[deprecated("Use the overloads with PdfTokenizerParams instead")]]
    PdfTokenizer(const PdfTokenizerOptions& options);
 
    [[deprecated("Use the overloads with PdfTokenizerParams instead")]]
    PdfTokenizer(std::shared_ptr<charbuff> buffer, const PdfTokenizerOptions& options);
 
    bool TryReadNextToken(InputStreamDevice& device, std::string_view& token);
    bool TryReadNextToken(InputStreamDevice& device, std::string_view& token, PdfTokenType& tokenType);
 
    bool TryPeekNextToken(InputStreamDevice& device, std::string_view& token);
    bool TryPeekNextToken(InputStreamDevice& device, std::string_view& token, PdfTokenType& tokenType);
 
    int64_t ReadNextNumber(InputStreamDevice& device);
    bool TryReadNextNumber(InputStreamDevice& device, int64_t& value);
 
    void ReadNextVariant(InputStreamDevice& device, PdfVariant& variant, const PdfStatefulEncrypt* encrypt = { });
    bool TryReadNextVariant(InputStreamDevice& device, PdfVariant& variant, const PdfStatefulEncrypt* encrypt = { });
 
    void Reset();
 
    void SetParameters(const PdfTokenizerParams& params) { m_Params = params; }
 
    const PdfTokenizerParams& GetParameters() const { return m_Params; }
 
protected:
    // This enum differs from regular PdfDataType in the sense
    // it enumerates only data types that can be determined literally
    // by the tokenization and specify better if the strings literals
    // are regular or hex strings
    enum class PdfLiteralDataType : uint8_t
    {
        Unknown = 0,
        Bool,
        Number,
        Real,
        String,
        HexString,
        Name,
        Array,
        Dictionary,
        Null,
        Reference,
    };
 
    struct ParsingOptions final
    {
        bool ThrowOnError : 1;
        bool Strict : 1;
    };
 
protected:
    void ReadNextVariant(InputStreamDevice& device, const std::string_view& token, PdfTokenType tokenType, PdfVariant& variant, const PdfStatefulEncrypt* encrypt);
    bool TryReadNextVariant(InputStreamDevice& device, const std::string_view& token, PdfTokenType tokenType, PdfVariant& variant, const PdfStatefulEncrypt* encrypt);
 
    void EnqueueToken(const std::string_view& token, PdfTokenType type);
 
    bool ReadDictionary(InputStreamDevice& device, PdfVariant& variant, const PdfStatefulEncrypt* encrypt, ParsingOptions opts);
 
    bool ReadArray(InputStreamDevice& device, PdfVariant& variant, const PdfStatefulEncrypt* encrypt, ParsingOptions opts);
 
    bool ReadString(InputStreamDevice& device, PdfVariant& variant, const PdfStatefulEncrypt* encrypt, ParsingOptions opts);
 
    bool ReadHexString(InputStreamDevice& device, PdfVariant& variant, const PdfStatefulEncrypt* encrypt, ParsingOptions opts);
 
    bool ReadName(InputStreamDevice& device, PdfVariant& variant, ParsingOptions opts);
 
    PdfLiteralDataType DetermineDataType(InputStreamDevice& device, const std::string_view& token, PdfTokenType tokenType, PdfVariant& variant, ParsingOptions opts);
 
private:
    PdfTokenizer(std::in_place_t, std::shared_ptr<charbuff>&& buffer);
    bool readDataType(InputStreamDevice& device, PdfLiteralDataType dataType, PdfVariant& variant, const PdfStatefulEncrypt* encrypt, ParsingOptions opts);
    bool drainContainer(InputStreamDevice& device, PdfTokenType endToken);
 
    bool handleContainerTruncation(ParsingOptions opts, PdfErrorCode code, std::string_view msg);
 
    bool handleContainerInnerError(InputStreamDevice& device, PdfTokenType endDelim, ParsingOptions opts,
        PdfErrorCode code, std::string_view msg);
 
private:
    using TokenizerPair = std::pair<std::string, PdfTokenType>;
    using TokenizerQueue = std::deque<TokenizerPair>;
 
private:
    std::shared_ptr<charbuff> m_buffer;
    PdfTokenizerParams m_Params;
    TokenizerQueue m_tokenQueue;
    charbuff m_charBuffer;
};
 
};
 
ENABLE_BITMASK_OPERATORS(PoDoFo::PdfTokenizerFlags);
 
#endif // PDF_TOKENIZER_H