1 Star 0 Fork 0

橙子/proto

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
token.go 4.68 KB
一键复制 编辑 原始数据 按行查看 历史
// Copyright (c) 2017 Ernest Micklei
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package proto
import (
"strings"
)
// token represents a lexical token.
type token int
const (
// Special tokens
tILLEGAL token = iota
tEOF
tWS
// Literals
tIDENT
// Misc characters
tSEMICOLON // ;
tCOLON // :
tEQUALS // =
tQUOTE // "
tSINGLEQUOTE // '
tLEFTPAREN // (
tRIGHTPAREN // )
tLEFTCURLY // {
tRIGHTCURLY // }
tLEFTSQUARE // [
tRIGHTSQUARE // ]
tCOMMENT // /
tLESS // <
tGREATER // >
tCOMMA // ,
tDOT // .
// Keywords
keywordsStart
tSYNTAX
tSERVICE
tRPC
tRETURNS
tMESSAGE
tIMPORT
tPACKAGE
tOPTION
tREPEATED
tWEAK
tPUBLIC
// special fields
tONEOF
tMAP
tRESERVED
tENUM
tSTREAM
// BEGIN proto2
tOPTIONAL
tGROUP
tEXTENSIONS
tEXTEND
tREQUIRED
// END proto2
keywordsEnd
)
// typeTokens exists for future validation
const typeTokens = "double float int32 int64 uint32 uint64 sint32 sint64 fixed32 sfixed32 sfixed64 bool string bytes"
// isKeyword returns if tok is in the keywords range
func isKeyword(tok token) bool {
return keywordsStart < tok && tok < keywordsEnd
}
// isWhitespace checks for space,tab and newline
func isWhitespace(r rune) bool {
return r == ' ' || r == '\t' || r == '\n'
}
// isDigit returns true if the rune is a digit.
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
// isString checks if the literal is quoted (single or double).
func isString(lit string) bool {
if lit == "'" {
return false
}
return (strings.HasPrefix(lit, "\"") &&
strings.HasSuffix(lit, "\"")) ||
(strings.HasPrefix(lit, "'") &&
strings.HasSuffix(lit, "'"))
}
func isComment(lit string) bool {
return strings.HasPrefix(lit, "//") || strings.HasPrefix(lit, "/*")
}
const doubleQuoteRune = rune('"')
// unQuote removes one matching leading and trailing single or double quote.
//
// https://github.com/emicklei/proto/issues/103
// cannot use strconv.Unquote as this unescapes quotes.
func unQuote(lit string) (string, rune) {
if len(lit) < 2 {
return lit, doubleQuoteRune
}
chars := []rune(lit)
first, last := chars[0], chars[len(chars)-1]
if first != last {
return lit, doubleQuoteRune
}
if s := string(chars[0]); s == "\"" || s == stringWithSingleQuote {
return string(chars[1 : len(chars)-1]), chars[0]
}
return lit, doubleQuoteRune
}
func asToken(literal string) token {
switch literal {
// delimiters
case ";":
return tSEMICOLON
case ":":
return tCOLON
case "=":
return tEQUALS
case "\"":
return tQUOTE
case "'":
return tSINGLEQUOTE
case "(":
return tLEFTPAREN
case ")":
return tRIGHTPAREN
case "{":
return tLEFTCURLY
case "}":
return tRIGHTCURLY
case "[":
return tLEFTSQUARE
case "]":
return tRIGHTSQUARE
case "<":
return tLESS
case ">":
return tGREATER
case ",":
return tCOMMA
case ".":
return tDOT
// words
case "syntax":
return tSYNTAX
case "service":
return tSERVICE
case "rpc":
return tRPC
case "returns":
return tRETURNS
case "option":
return tOPTION
case "message":
return tMESSAGE
case "import":
return tIMPORT
case "package":
return tPACKAGE
case "oneof":
return tONEOF
// special fields
case "map":
return tMAP
case "reserved":
return tRESERVED
case "enum":
return tENUM
case "repeated":
return tREPEATED
case "weak":
return tWEAK
case "public":
return tPUBLIC
case "stream":
return tSTREAM
// proto2
case "optional":
return tOPTIONAL
case "group":
return tGROUP
case "extensions":
return tEXTENSIONS
case "extend":
return tEXTEND
case "required":
return tREQUIRED
default:
// special cases
if isComment(literal) {
return tCOMMENT
}
return tIDENT
}
}
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
1
https://gitee.com/xuchengzhi/proto-emicklei.git
[email protected]:xuchengzhi/proto-emicklei.git
xuchengzhi
proto-emicklei
proto
master

搜索帮助