forked from mlc-ai/tokenizers-cpp
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathrwkv_world_tokenizer.h
More file actions
33 lines (30 loc) · 1.03 KB
/
Copy pathrwkv_world_tokenizer.h
File metadata and controls
33 lines (30 loc) · 1.03 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
/*!
* Copyright (c) 2023 by Contributors daquexian
* \file rwkv_world_tokenizer.h
* \brief Implementation of llm chat.
*/
#ifndef RWKV_WORLD_TOKENIZER_H_
#define RWKV_WORLD_TOKENIZER_H_
#include <exception>
#include <sstream>
#include <stdexcept>
#include <string>
#define STRINGIFY(...) STRINGIFY_(__VA_ARGS__)
#define STRINGIFY_(...) #__VA_ARGS__
#define RV_CHECK(...) \
for (bool _rv_check_status = (__VA_ARGS__); !_rv_check_status;) \
throw FRException() << ("Check \"" STRINGIFY(__VA_ARGS__) "\" failed at " + \
std::to_string(__LINE__) + " in " __FILE__ "\n > Error msg: ")
struct FRException : public std::runtime_error {
FRException() : std::runtime_error("") {}
const char* what() const noexcept override { return msg.c_str(); }
template <typename T>
FRException& operator<<(const T& s) {
std::stringstream ss;
ss << s;
msg += ss.str();
return *this;
}
std::string msg;
};
#endif // RWKV_WORLD_TOKENIZER_H_