master

分支 (217)

标签 (1721)

管理

管理

master

gg/add-phi-3-support

gg/flash-attn

test-bench

hp/quantize/imatrix-metadata

update_flake_lock_action

gg/llama3-support

hp/server/avoid-infinite-loop

gg/disable-sgemm

hp/tmp/kv-cache-defrag

sl/hash-improvements

gg/imatrix-remove-assert

compilade/refactor-kv-cache

gg/authors

mlx-challenge

ceb/bert-tokenizer-fixes

sycl-refactor

sync

gg/flash-attn-a

compilade/fix-command-r

b2715

b2714

b2712

b2710

b2709

b2708

b2707

b2702

b2700

b2699

b2698

b2697

b2696

b2694

b2692

b2691

b2690

b2687

b2686

b2684

llama.cpp
/
unicode-data.h

#pragma once

#include <cstdint>
#include <map>
#include <utility>
#include <vector>

extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_digit;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_letter;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_whitespace;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_mark;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control;
extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;
extern const std::map<char32_t, char32_t> unicode_map_lowercase;