urlparser.h

Go to the documentation of this file.

 
#ifndef URLPARSER_H
#define URLPARSER_H
 
// Single source of truth for the project version. Read by:
//   - cmake/DynamicVersion.cmake (sets PROJECT_VERSION for the CMake build)
//   - pyproject.toml's [tool.scikit-build.metadata.version] regex provider
//   - version.py (bump/tag helper)
#define URLPARSER_VERSION_MAJOR 2
#define URLPARSER_VERSION_MINOR 1
#define URLPARSER_VERSION_PATCH 0
 
#define URLPARSER_VERSION_ENCODE(maj,min,pat) (((maj)*10000)+((min)*100)+(pat))
#define URLPARSER_VERSION \
    URLPARSER_VERSION_ENCODE(URLPARSER_VERSION_MAJOR,URLPARSER_VERSION_MINOR,URLPARSER_VERSION_PATCH)
 
#define URLPARSER_VERSION_HEX \
    ((URLPARSER_VERSION_MAJOR << 16) | (URLPARSER_VERSION_MINOR << 8) | URLPARSER_VERSION_PATCH)
 
/* Internal helpers for URLPARSER_VERSION_STRING */
#define _URLPARSER_VERSION_XSTR(a,b,c) #a"."#b"."#c
#define _URLPARSER_VERSION_STR(a,b,c)  _URLPARSER_VERSION_XSTR(a,b,c)
 
#define URLPARSER_VERSION_STRING \
    _URLPARSER_VERSION_STR(URLPARSER_VERSION_MAJOR,URLPARSER_VERSION_MINOR,URLPARSER_VERSION_PATCH)
 
// Single source of truth for the Public Suffix List URL. Read by:
//   - cmake/PublicSuffixList.cmake (downloads the .dat file this URL
//     points to, to embed into the compiled extension at build time)
//   - urlparser::psl::source_url() below (src/urlparser.cpp), so
//     psl.url / psl.update() in Python point at the same place the
//     build itself downloaded from - one URL, not two to keep in sync.
#define URLPARSER_PUBLIC_SUFFIX_LIST_URL "https://publicsuffix.org/list/public_suffix_list.dat"
 
#include <array>
#include <cstdint>
#include <iostream>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <variant>
#include <vector>
 
namespace urlparser {
constexpr bool DEFAULT_IGNORE_WWW = false;
 
class version {
   public:
    static unsigned int major() noexcept { return URLPARSER_VERSION_MAJOR; }
    static unsigned int minor() noexcept { return URLPARSER_VERSION_MINOR; }
    static unsigned int patch() noexcept { return URLPARSER_VERSION_PATCH; }
    static unsigned int hex() noexcept { return URLPARSER_VERSION_HEX; }
    static std::string_view string() noexcept { return URLPARSER_VERSION_STRING; }
 
   private:
    version();
};
 
using query_params = std::vector<std::string>;
 
class hostname {
   public:
    static hostname from_url(std::string_view url,
                        const bool ignore_www = DEFAULT_IGNORE_WWW);
 
    static hostname from_url(const char* url,
                        const bool ignore_www = DEFAULT_IGNORE_WWW);
 
    static hostname from_url(const std::string& url,
                        const bool ignore_www = DEFAULT_IGNORE_WWW);
 
    static hostname from_url(std::string&& url,
                        const bool ignore_www = DEFAULT_IGNORE_WWW);
 
    static std::string_view remove_www(const std::string_view& host) noexcept;
 
   public:
    hostname(std::string host, const bool ignore_www = DEFAULT_IGNORE_WWW);
 
    hostname() noexcept = default;
 
    bool operator==(const hostname& other) const noexcept;
 
    bool operator==(const std::string& other) const noexcept;
 
    const std::string& suffix() const noexcept;
    const std::string& domain() const noexcept;
    std::string domain_name() const noexcept;
    const std::string& subdomain() const noexcept;
    const std::string& full_domain() const noexcept;
    const std::string& str() const noexcept;
    std::string normalized_ascii() const;
 
   private:
    // Runs the PSL lookup + domain/subdomain/suffix split exactly once, the
    // first time any of those fields is actually requested.
    void ensure_parsed() const noexcept;
 
    std::string host_;
    bool ignore_www_ = false;
    mutable bool parsed_ = false;
    mutable std::string domain_;
    mutable std::string subdomain_;
    mutable std::string suffix_;
    mutable std::string fulldomain_;
};
 
namespace detail {
// Wraps ankerl::unordered_dense::map<std::string, size_t> (vendored in
// src/ankerl/ - see src/ankerl/README.md). Forward-
// declared here and defined only in urlparser.cpp so that including this
// public header doesn't require the vendored third-party header too -
// psl::levels_ is a private implementation detail, and callers of the
// public API never need to know what's actually inside it.
class suffix_table;
}  // namespace detail
 
class psl {
   public:
    static psl& instance() noexcept;
 
    std::string_view source_url() const noexcept;
 
    std::string_view version() const noexcept { return version_; }
 
    bool is_loaded() const noexcept;
 
    void load_from_path(const std::string& filepath);
 
    void load_from_string(const std::string& filestr);
 
    bool is_suffix(std::string_view text) const noexcept;
 
    std::string suffix_of(const std::string& hostname_text) const;
 
   private:
    // levels_ is a std::unique_ptr<detail::suffix_table> (an incomplete
    // type here), so the compiler-generated destructor/move members
    // would need that type complete to know how to delete/move it -
    // declared here, defined in urlparser.cpp where suffix_table is
    // fully visible. Both the singleton instance (instance()'s static
    // local) and the self-move-assignments in load_from_path()/
    // load_from_string() are psl's own member functions, so these stay
    // private along with the constructors - nothing outside the class
    // should ever construct, move, or destroy a psl directly. Copy is
    // deleted outright for the same reason (a unique_ptr member already
    // makes an implicit copy ill-formed; an explicit delete fails at the
    // call site with a clear reason instead of a unique_ptr error deep
    // in a template instantiation).
    ~psl();
    psl(const psl&) = delete;
    psl& operator=(const psl&) = delete;
    psl(psl&&) noexcept;
    psl& operator=(psl&&) noexcept;
 
    psl() noexcept;
    explicit psl(std::istream& stream);
 
    size_t segment_count(const std::string& text) const;
    void add_rule(std::string& rule, int level_adjust, size_t trim);
 
    std::unique_ptr<detail::suffix_table> levels_;
    std::string version_;
};
 
class ipv4 {
   public:
    ipv4() noexcept = default;
 
    ipv4(std::string_view text);
 
    static bool is_valid(std::string_view text) noexcept;
 
    static ipv4 from_uint32(uint32_t address) noexcept;
 
    static ipv4 from_url(std::string_view url);
 
    std::string str() const;
 
    uint32_t to_uint32() const noexcept { return value_; }
 
    std::array<uint8_t, 4> bytes() const noexcept;
 
    ipv4& operator++() noexcept {
        ++value_;
        return *this;
    }
    ipv4 operator++(int) noexcept {
        ipv4 tmp = *this;
        ++value_;
        return tmp;
    }
    ipv4& operator--() noexcept {
        --value_;
        return *this;
    }
    ipv4 operator--(int) noexcept {
        ipv4 tmp = *this;
        --value_;
        return tmp;
    }
    ipv4& operator+=(int64_t delta) noexcept {
        value_ += static_cast<uint32_t>(delta);
        return *this;
    }
    ipv4& operator-=(int64_t delta) noexcept {
        value_ -= static_cast<uint32_t>(delta);
        return *this;
    }
 
    friend ipv4 operator+(ipv4 lhs, int64_t delta) noexcept { return lhs += delta; }
    friend ipv4 operator-(ipv4 lhs, int64_t delta) noexcept { return lhs -= delta; }
    friend int64_t operator-(const ipv4& a, const ipv4& b) noexcept {
        return static_cast<int64_t>(a.value_) - static_cast<int64_t>(b.value_);
    }
 
    friend bool operator==(const ipv4& a, const ipv4& b) noexcept { return a.value_ == b.value_; }
    friend bool operator!=(const ipv4& a, const ipv4& b) noexcept { return a.value_ != b.value_; }
    friend bool operator<(const ipv4& a, const ipv4& b) noexcept { return a.value_ < b.value_; }
    friend bool operator<=(const ipv4& a, const ipv4& b) noexcept { return a.value_ <= b.value_; }
    friend bool operator>(const ipv4& a, const ipv4& b) noexcept { return a.value_ > b.value_; }
    friend bool operator>=(const ipv4& a, const ipv4& b) noexcept { return a.value_ >= b.value_; }
 
   private:
    explicit ipv4(uint32_t v) noexcept : value_(v) {}
    uint32_t value_ = 0;
};
 
class ipv6 {
   public:
    ipv6() noexcept = default;
 
    ipv6(std::string_view text);
 
    static bool is_valid(std::string_view text) noexcept;
 
    static ipv6 from_uint64_pair(uint64_t high, uint64_t low) noexcept;
 
    static ipv6 from_url(std::string_view url);
 
    std::string str() const;
 
    std::array<uint8_t, 16> bytes() const noexcept;
 
    uint64_t high64() const noexcept { return hi_; }
    uint64_t low64() const noexcept { return lo_; }
 
    ipv6& operator++() noexcept {
        add_bits(1);
        return *this;
    }
    ipv6 operator++(int) noexcept {
        ipv6 tmp = *this;
        ++(*this);
        return tmp;
    }
    ipv6& operator--() noexcept {
        add_bits(~uint64_t{0});  // two's-complement bit pattern of -1
        return *this;
    }
    ipv6 operator--(int) noexcept {
        ipv6 tmp = *this;
        --(*this);
        return tmp;
    }
    ipv6& operator+=(int64_t delta) noexcept {
        add_bits(static_cast<uint64_t>(delta));
        return *this;
    }
    ipv6& operator-=(int64_t delta) noexcept {
        // -delta's bit pattern, computed in unsigned arithmetic so this is
        // well-defined even when delta == INT64_MIN (no positive int64_t
        // counterpart, so negating it as a signed value would overflow).
        add_bits(~static_cast<uint64_t>(delta) + 1);
        return *this;
    }
 
    friend ipv6 operator+(ipv6 lhs, int64_t delta) noexcept { return lhs += delta; }
    friend ipv6 operator-(ipv6 lhs, int64_t delta) noexcept { return lhs -= delta; }
 
    friend bool operator==(const ipv6& a, const ipv6& b) noexcept {
        return a.hi_ == b.hi_ && a.lo_ == b.lo_;
    }
    friend bool operator!=(const ipv6& a, const ipv6& b) noexcept { return !(a == b); }
    friend bool operator<(const ipv6& a, const ipv6& b) noexcept {
        return a.hi_ != b.hi_ ? a.hi_ < b.hi_ : a.lo_ < b.lo_;
    }
    friend bool operator<=(const ipv6& a, const ipv6& b) noexcept { return !(b < a); }
    friend bool operator>(const ipv6& a, const ipv6& b) noexcept { return b < a; }
    friend bool operator>=(const ipv6& a, const ipv6& b) noexcept { return !(a < b); }
 
   private:
    ipv6(uint64_t hi, uint64_t lo) noexcept : hi_(hi), lo_(lo) {}
 
    // Adds the 128-bit sign-extension of the two's-complement 64-bit
    // pattern `low64_bits` to (hi_, lo_), wrapping mod 2^128.
    void add_bits(uint64_t low64_bits) noexcept;
 
    uint64_t hi_ = 0;
    uint64_t lo_ = 0;
};
 
class host {
   public:
    using variant_type = std::variant<hostname, ipv4, ipv6>;
 
    host() noexcept = default;
 
    static host wrap(hostname h) noexcept {
        host result;
        result.value_ = std::move(h);
        return result;
    }
    static host wrap(ipv4 v) noexcept {
        host result;
        result.value_ = v;
        return result;
    }
    static host wrap(ipv6 v) noexcept {
        host result;
        result.value_ = v;
        return result;
    }
 
    host(std::string_view host_text, bool ignore_www = DEFAULT_IGNORE_WWW) noexcept;
 
    host(const char* host_text, bool ignore_www = DEFAULT_IGNORE_WWW) noexcept;
 
    host(std::string&& host_text, bool ignore_www = DEFAULT_IGNORE_WWW) noexcept;
 
    static host from_url(std::string_view url, bool ignore_www = DEFAULT_IGNORE_WWW) noexcept;
 
    static host from_url(const char* url, bool ignore_www = DEFAULT_IGNORE_WWW) noexcept;
 
    static host from_url(std::string&& url, bool ignore_www = DEFAULT_IGNORE_WWW) noexcept;
 
    bool is_hostname() const noexcept { return std::holds_alternative<hostname>(value_); }
    bool is_ipv4() const noexcept { return std::holds_alternative<ipv4>(value_); }
    bool is_ipv6() const noexcept { return std::holds_alternative<ipv6>(value_); }
    bool is_ip() const noexcept { return !is_hostname(); }
 
    const hostname& get_hostname() const { return std::get<hostname>(value_); }
    const ipv4& get_ipv4() const { return std::get<ipv4>(value_); }
    const ipv6& get_ipv6() const { return std::get<ipv6>(value_); }
 
    const hostname* try_hostname() const noexcept { return std::get_if<hostname>(&value_); }
    const ipv4* try_ipv4() const noexcept { return std::get_if<ipv4>(&value_); }
    const ipv6* try_ipv6() const noexcept { return std::get_if<ipv6>(&value_); }
 
    std::string str() const;
 
    const variant_type& variant() const noexcept { return value_; }
 
    friend bool operator==(const host& a, const host& b) noexcept { return a.value_ == b.value_; }
    friend bool operator!=(const host& a, const host& b) noexcept { return !(a == b); }
 
   private:
    variant_type value_;
};
 
class url {
   public:
    static bool is_psl_loaded() noexcept;
 
    static std::string extract_host(std::string_view url) noexcept;
 
    static std::string extract_host(const char* url) noexcept;
 
    static std::string extract_host(std::string&& url) noexcept;
 
   public:
    url(std::string_view url, const bool ignore_www = DEFAULT_IGNORE_WWW);
 
    url() noexcept = default;
 
    bool operator==(const url& other) const noexcept;
 
    std::string str() const noexcept;
    std::string_view protocol() const noexcept { return field(scheme_); }
    std::string_view query() const noexcept { return field(query_); }
    std::string_view fragment() const noexcept { return field(fragment_); }
    std::string_view userinfo() const noexcept { return field(userinfo_); }
    std::string_view path() const noexcept { return field(path_); }
    const std::string& abspath() const noexcept;
    std::string normalized() const;
    std::string_view host_text() const noexcept { return field(host_); }
    int port() const noexcept { return port_; }
    query_params params() const noexcept;
    using search_params = std::unordered_map<std::string, std::vector<std::string>>;
    search_params get_search_params() const;
    const urlparser::host& host() const noexcept;
 
   private:
    // An (offset, length) pair into storage_ - not a raw pointer/view, so
    // that copying a url (which deep-copies storage_) never needs to
    // "rebase" anything: the offsets stay correct as-is.
    struct Span {
        uint32_t pos = 0;
        uint32_t len = 0;
    };
    std::string_view field(Span s) const noexcept {
        return std::string_view(storage_).substr(s.pos, s.len);
    }
 
    const urlparser::host& ensure_host() const noexcept;
 
    std::string storage_;
    Span scheme_, userinfo_, host_, path_, params_, query_, fragment_;
    int port_ = 0;
    bool has_params_ = false;
    bool has_query_ = false;
    bool ignore_www_ = false;
    mutable std::optional<urlparser::host> host_cache_;
    mutable std::optional<std::string> abspath_cache_;
};
 
class scp_url {
   public:
    static bool is_scp_like(std::string_view input) noexcept;
 
    static std::string normalize(std::string_view input);
 
    explicit scp_url(std::string_view input);
 
    const url& as_url() const noexcept { return url_; }
    operator const url&() const noexcept { return url_; }
 
    bool was_scp_like() const noexcept { return was_scp_like_; }
 
   private:
    url url_;
    bool was_scp_like_;
};
 
using git_url = scp_url;
 
std::string build_search_params(const std::vector<std::pair<std::string, std::string>>& pairs);
 
std::string build_search_params(const url::search_params& params);
 
inline std::string normalize(std::string_view input) { return url(input).normalized(); }
 
}  // namespace urlparser
 
std::ostream& operator<<(std::ostream& os, const urlparser::query_params& dt);
 
std::ostream& operator<<(std::ostream& os, const urlparser::url& dt);
 
std::ostream& operator<<(std::ostream& os, const urlparser::hostname& dt);
 
std::ostream& operator<<(std::ostream& os, const urlparser::ipv4& dt);
 
std::ostream& operator<<(std::ostream& os, const urlparser::ipv6& dt);
 
std::ostream& operator<<(std::ostream& os, const urlparser::host& dt);
 
// ============================================================================
// IDNA (Internationalized Domain Names): Unicode <-> ASCII/Punycode.
// ============================================================================
namespace urlparser {
 
namespace idna {
 
std::string to_ascii(std::string_view utf8_domain);
 
}  // namespace idna
}  // namespace urlparser
 
// ============================================================================
// Percent-encoding ("quote") / percent-decoding ("unquote").
// ============================================================================
namespace urlparser {
 
namespace percent_codec {
 
using character_set = std::array<uint64_t, 4>;
 
constexpr bool contains(const character_set& set, unsigned char c) noexcept {
    return (set[c >> 6] >> (c & 63)) & 1ULL;
}
 
character_set unreserved_set();
 
character_set path_safe_set();
 
std::string decode(std::string_view input);
 
std::string encode(std::string_view input,
                    const character_set& safe = unreserved_set());
 
}  // namespace percent_codec
}  // namespace urlparser
 
// ============================================================================
// URL reference resolution ("join"): RFC 3986 §5.
// ============================================================================
namespace urlparser {
 
std::string resolve(std::string_view base, std::string_view ref);
 
}  // namespace urlparser
 
#endif  // URLPARSER_H