наработки

LICENSE

@@ -0,0 +1,13 @@
+           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                   Version 2, December 2004
+
+Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+Everyone is permitted to copy and distribute verbatim or modified
+copies of this license document, and changing it is allowed as long
+as the name is changed.
+
+           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+  TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. You just DO WHAT THE FUCK YOU WANT TO.

err.go

@@ -0,0 +1,88 @@
+package tier
+
+import (
+	"encoding/xml"
+	"fmt"
+	"github.com/kpmy/ypk/halt"
+)
+
+type Marker interface {
+	Mark(...interface{})
+	FutureMark() Marker
+	Log(...interface{})
+}
+
+type Error struct {
+	XMLName xml.Name
+	From    string `xml:"from,attr"`
+	Pos     int    `xml:"pos,attr"`
+	Line    int    `xml:"line,attr"`
+	Column  int    `xml:"column,attr"`
+	Message string `xml:",chardata"`
+}
+
+func (e *Error) String() string {
+	data, _ := xml.Marshal(e)
+	return string(data)
+}
+
+func Err(sender string, pos, line, col int, msg ...interface{}) *Error {
+	err := &Error{From: sender, Pos: pos, Line: line, Column: col, Message: fmt.Sprint(msg...)}
+	err.XMLName.Local = "error"
+	return err
+}
+
+type mark struct {
+	rd        int
+	line, col int
+	marker    Marker
+}
+
+func (m *mark) Mark(msg ...interface{}) {
+	m.marker.(*mk).m = m
+	m.marker.Mark(msg...)
+}
+
+func (m *mark) FutureMark() Marker { halt.As(100); return nil }
+
+func (m *mark) Log(msg ...interface{}) { m.marker.Log(msg...) }
+
+type mk struct {
+	sc  Scanner
+	m   *mark
+	log func(...interface{})
+}
+
+func (m *mk) Mark(msg ...interface{}) {
+	rd := m.sc.Count()
+	str, pos := m.sc.Pos()
+	if len(msg) == 0 {
+		m.m = &mark{rd: rd, line: str, col: pos}
+	} else if m.m != nil {
+		rd, str, pos = m.m.rd, m.m.line, m.m.col
+		m.m = nil
+	}
+	if m.m == nil {
+		panic(Err("parser", rd, str, pos, msg...))
+	}
+}
+
+func (m *mk) FutureMark() Marker {
+	rd := m.sc.Count()
+	str, pos := m.sc.Pos()
+	ret := &mark{marker: m, rd: rd, line: str, col: pos}
+	return ret
+}
+
+func (m *mk) Log(msg ...interface{}) {
+	if m.log != nil {
+		m.log(msg...)
+	}
+}
+
+func NewMarker(s Scanner, log func(...interface{})) Marker {
+	ret := &mk{}
+	ret.sc = s
+	ret.log = log
+	return ret
+}

run.go

@@ -0,0 +1,112 @@
+package tier
+
+import (
+	"fmt"
+	"github.com/kpmy/ypk/assert"
+)
+
+type Runner interface {
+	Next() Symbol
+	//expect is the most powerful step forward runner, breaks the compilation if unexpected sym found
+	Expect(SymCode, interface{}, ...SymCode)
+	//await runs for the sym through skip list, but may not find the sym
+	Await(SymCode, ...SymCode) bool
+	//pass runs through skip list
+	Pass(...SymCode)
+	//run runs to the first sym through any other sym
+	Run(SymCode)
+	//Is current symbol?
+	Is(SymCode) bool
+}
+
+type rn struct {
+	sc    Scanner
+	done  bool
+	debug bool
+	sym   Symbol
+
+	marker Marker
+}
+
+func (r *rn) Next() Symbol {
+	r.done = true
+	r.sym = r.sc.Get()
+	if r.debug {
+		r.marker.Log("`" + fmt.Sprint(r.sym) + "`")
+	}
+	return r.sym
+}
+
+//expect is the most powerful step forward runner, breaks the compilation if unexpected sym found
+func (r *rn) Expect(sym SymCode, msg interface{}, skip ...SymCode) {
+	assert.For(r.done, 20)
+	if !r.Await(sym, skip...) {
+		r.marker.Mark(msg)
+	}
+	r.done = false
+}
+
+//await runs for the sym through skip list, but may not find the sym
+func (r *rn) Await(sym SymCode, skip ...SymCode) bool {
+	assert.For(r.done, 20)
+	skipped := func() (ret bool) {
+		for _, v := range skip {
+			if v == r.sym.Code {
+				ret = true
+			}
+		}
+		return
+	}
+
+	for sym != r.sym.Code && skipped() && r.sc.Error() == nil {
+		r.Next()
+	}
+	r.done = r.sym.Code != sym
+	return r.sym.Code == sym
+}
+
+//pass runs through skip list
+func (r *rn) Pass(skip ...SymCode) {
+	skipped := func() (ret bool) {
+		for _, v := range skip {
+			if v == r.sym.Code {
+				ret = true
+			}
+		}
+		return
+	}
+	for skipped() && r.sc.Error() == nil {
+		r.Next()
+	}
+}
+
+//run runs to the first sym through any other sym
+func (r *rn) Run(sym SymCode) {
+	if r.sym.Code != sym {
+		for r.sc.Error() == nil && r.Next().Code != sym {
+			if r.sc.Error() != nil {
+				r.marker.Mark(sym, " not found")
+				break
+			}
+		}
+	}
+}
+
+func (r *rn) Is(sym SymCode) bool {
+	return r.sym.Code == sym
+}
+
+func NewRunner(s Scanner, m Marker) Runner {
+	ret := &rn{}
+	ret.sc = s
+	ret.marker = m
+	ret.Next()
+	return ret
+}
+
+func Debug(r Runner) Runner {
+	if rr, ok := r.(*rn); ok {
+		rr.debug = !rr.debug
+	}
+	return r
+}

sc.go

@@ -0,0 +1,366 @@
+package tier
+
+import (
+	"fmt"
+	"github.com/kpmy/ypk/assert"
+	"io"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+type SymCode int
+
+const (
+	None SymCode = iota
+	Ident
+	Number
+	String
+)
+
+type Symbol struct {
+	Code  SymCode
+	Value string
+
+	StringOpts struct {
+		Apos bool
+	}
+
+	NumberOpts struct {
+		Modifier string
+		Period   bool
+	}
+}
+
+type Opts struct {
+	IdentMap      map[string]SymCode
+	IdentStarts   string
+	IdentContains string
+
+	NumContains  string
+	NumModifiers string
+
+	SpaceMap map[string]SymCode
+
+	CombinedMap map[string]SymCode
+
+	CommentTriplet [3]rune
+}
+
+type Scanner interface {
+	Get() Symbol
+	Error() error
+
+	Count() int
+	Pos() (int, int)
+}
+
+func Token2(r rune) string {
+	return string([]rune{r})
+}
+
+func Token(r rune) string {
+	if unicode.IsSpace(r) || int(r) <= int(' ') {
+		return strconv.Itoa(int(r)) + "U"
+	} else {
+		return string([]rune{r})
+	}
+}
+
+func (s Symbol) String() string {
+	return fmt.Sprint("sym: `", s.Code, "` ", s.Value)
+}
+
+func (sym SymCode) String() (s string) {
+	switch sym {
+	case None:
+		s = "none"
+	case Ident:
+		s = "ident"
+	case Number:
+		s = "num"
+	case String:
+		s = "string"
+	default:
+		s = strconv.Itoa(int(sym))
+	}
+	return
+}
+
+type sc struct {
+	rd  io.RuneReader
+	err error
+
+	ch  rune
+	pos int
+
+	opts Opts
+
+	lines struct {
+		count int
+		last  int
+		crlf  bool
+		lens  map[int]func() (int, int)
+	}
+}
+
+func (s *sc) Count() int { return s.pos }
+
+func (s *sc) Pos() (int, int) { return s.lines.count, s.lines.last }
+
+func (s *sc) Get() (sym Symbol) {
+	for stop := s.err != nil; !stop; {
+		sym = s.get()
+		stop = sym.Code != 0 || s.err != nil
+	}
+	return
+}
+
+func (s *sc) Error() error {
+	return s.err
+}
+
+func (s *sc) mark(msg ...interface{}) {
+	//log.Println("at pos ", s.pos, " ", fmt.Sprintln(msg...))
+	l, c := s.Pos()
+	panic(Err("scanner", s.Count(), l, c, msg...))
+}
+
+func (o Opts) isIdentLetter(r rune) bool {
+	return o.isIdentFirstLetter(r) || unicode.IsDigit(r) || strings.ContainsRune(o.IdentContains, r)
+}
+
+func (o Opts) isIdentFirstLetter(r rune) bool {
+	return unicode.IsLetter(r) || strings.ContainsRune(o.IdentStarts, r)
+}
+
+func (o Opts) validate() {
+	assert.For(o.CommentTriplet[0] != o.CommentTriplet[1], 20)
+	assert.For(o.CommentTriplet[1] != o.CommentTriplet[2], 20)
+}
+
+func (s *sc) ident() (sym Symbol) {
+	assert.For(s.opts.isIdentFirstLetter(s.ch), 20, "letter must be first")
+	buf := make([]rune, 0)
+	for s.err == nil && s.opts.isIdentLetter(s.ch) {
+		buf = append(buf, s.ch)
+		s.next()
+	}
+	if s.err == nil || s.err == io.EOF {
+		sym.Value = string(buf)
+		if code, ok := s.opts.IdentMap[sym.Value]; ok {
+			sym.Code = code
+		} else {
+			sym.Code = Ident
+		}
+	} else {
+		s.mark("error while reading ident ", s.err)
+	}
+	return
+}
+
+const dec = "0123456789"
+
+//first char always 0..9
+func (s *sc) num() (sym Symbol) {
+	assert.For(unicode.IsDigit(s.ch), 20, "digit expected")
+	var buf []rune
+	var mbuf []rune
+	hasDot := false
+
+	for {
+		buf = append(buf, s.ch)
+		s.next()
+		if s.ch == '.' {
+			if !hasDot {
+				hasDot = true
+			} else if hasDot {
+				s.mark("dot unexpected")
+			}
+		}
+		if s.err != nil || !(s.ch == '.' || strings.ContainsRune(dec, s.ch) || strings.ContainsRune(s.opts.NumContains, s.ch)) {
+			break
+		}
+	}
+	if strings.ContainsRune(s.opts.NumModifiers, s.ch) {
+		mbuf = append(mbuf, s.ch)
+		s.next()
+	}
+	if strings.ContainsAny(string(buf), s.opts.NumContains) && len(mbuf) == 0 {
+		s.mark("modifier expected")
+	}
+	if s.err == nil {
+		sym.Code = Number
+		sym.Value = string(buf)
+		sym.NumberOpts.Modifier = string(mbuf)
+		sym.NumberOpts.Period = hasDot
+	} else {
+		s.mark("error reading number")
+	}
+	return
+}
+
+func (s *sc) str() string {
+	assert.For(s.ch == '"' || s.ch == '\'' || s.ch == '`', 20, "quote expected")
+	var buf []rune
+	ending := s.ch
+	s.next()
+	for ; s.err == nil && s.ch != ending; s.next() {
+		buf = append(buf, s.ch)
+	}
+	if s.err == nil {
+		s.next()
+	} else {
+		s.mark("string expected")
+	}
+	return string(buf)
+}
+
+func (s *sc) next() rune {
+	read := 0
+	s.ch, read, s.err = s.rd.ReadRune()
+	if s.err == nil {
+		s.pos += read
+	}
+	if s.ch == '\r' || s.ch == '\n' {
+		s.line()
+	} else {
+		s.lines.last++
+	}
+	//log.Println(Token(s.ch), s.err)
+	return s.ch
+}
+
+func (s *sc) line() {
+	if s.ch == '\r' {
+		s.lines.crlf = true
+	}
+	if (s.lines.crlf && s.ch == '\r') || (!s.lines.crlf && s.ch == '\n') {
+		s.lines.lens[s.lines.count] = func() (int, int) {
+			return s.lines.count, s.pos
+		}
+		s.lines.count++
+		s.lines.last = 1
+	} else if s.lines.crlf && s.ch == '\n' {
+		s.lines.last--
+	}
+}
+
+func (s *sc) comment() {
+	assert.For(s.ch == '*', 20, "expected ", s.opts.CommentTriplet[1], "got ", Token(s.ch))
+	for {
+		for s.err == nil && s.ch != s.opts.CommentTriplet[1] {
+			if s.ch == s.opts.CommentTriplet[0] {
+				if s.next() == s.opts.CommentTriplet[1] {
+					s.comment()
+				}
+			} else {
+				s.next()
+			}
+		}
+		for s.err == nil && s.ch == s.opts.CommentTriplet[1] {
+			s.next()
+		}
+		if s.err != nil || s.ch == s.opts.CommentTriplet[2] {
+			break
+		}
+	}
+	if s.err == nil {
+		s.next()
+	} else {
+		s.mark("unclosed comment")
+	}
+}
+
+func (s *sc) filter(r ...rune) SymCode {
+	var run func(keys map[string]SymCode, r ...rune) SymCode
+
+	run = func(keys map[string]SymCode, r ...rune) (ret SymCode) {
+		key := string(r)
+		ret = keys[key]
+		continues := make(map[string]SymCode)
+		for k, v := range keys {
+			if key != k && strings.HasPrefix(k, key) {
+				continues[k] = v
+			}
+		}
+		//log.Println(ret, continues)
+		if len(continues) > 0 {
+			nr := []rune(key)
+			nr = append(nr, s.next())
+			if x := run(continues, nr...); x != None {
+				ret = x
+			}
+		}
+		return
+	}
+
+	return run(s.opts.CombinedMap, r...)
+}
+
+func (s *sc) get() (sym Symbol) {
+	switch s.ch {
+	case s.opts.CommentTriplet[0]:
+		if s.next() == s.opts.CommentTriplet[1] {
+			s.comment()
+		} else if symCode := s.filter(s.opts.CommentTriplet[0], s.ch); symCode != None {
+			sym.Code = symCode
+		} else {
+			sym.Code = s.filter(s.opts.CommentTriplet[0])
+		}
+	case '"', '\'', '`':
+		sym.StringOpts.Apos = (s.ch == '\'' || s.ch == '`')
+		sym.Value = s.str()
+		sym.Code = String
+	default:
+		switch {
+		case s.opts.isIdentFirstLetter(s.ch):
+			sym = s.ident()
+		case unicode.IsSpace(s.ch):
+			sym.Value = Token2(s.ch)
+			sym.Code, _ = s.opts.SpaceMap[sym.Value]
+			s.next()
+		case unicode.IsDigit(s.ch):
+			sym = s.num()
+		default:
+			if symCode := s.filter(s.ch); symCode != None {
+				sym.Code = symCode
+				s.next()
+			} else {
+				s.mark("unhandled ", "`", Token(s.ch), "`")
+				s.next()
+			}
+		}
+	}
+	return
+}
+
+func NewScanner(rd io.RuneReader, opts ...Opts) Scanner {
+	ret := &sc{}
+	ret.rd = rd
+	ret.lines.lens = make(map[int]func() (int, int))
+	ret.lines.count++
+	if len(opts) > 0 {
+		ret.opts = opts[0]
+	} else {
+		ret.opts = defaultOpts
+	}
+	ret.opts.validate()
+	ret.next()
+	return ret
+}
+
+var defaultOpts Opts
+
+func init() {
+	defaultOpts.IdentMap = make(map[string]SymCode)
+
+	defaultOpts.SpaceMap = make(map[string]SymCode)
+
+	defaultOpts.NumContains = "ABCDEF"
+	defaultOpts.NumModifiers = "U"
+
+	defaultOpts.CombinedMap = make(map[string]SymCode)
+
+	defaultOpts.CommentTriplet = [3]rune{'(', '*', ')'}
+}

tier_test.go

@@ -0,0 +1,78 @@
+package tier
+
+import (
+	"bufio"
+	"bytes"
+	"testing"
+)
+
+func TestScanner(t *testing.T) {
+	const testString = `
+		BEGIN
+
+		f asdf asdf xx x23 (* dfa3asd *) 33FH 3FU 234U 3.3  : := :== > < 0.12314 003141 -efef23 asdfd asf "dfsdfa sdf asdf " 'df' df'd' ;;      ;
+	`
+	defaultOpts := Opts{}
+	defaultOpts.IdentMap = make(map[string]SymCode)
+	defaultOpts.IdentMap["BEGIN"] = 100
+
+	defaultOpts.SpaceMap = make(map[string]SymCode)
+	defaultOpts.SpaceMap[" "] = 101
+	defaultOpts.SpaceMap["\n"] = 102
+
+	defaultOpts.NumContains = "ABCDEF"
+	defaultOpts.NumModifiers = "UH"
+
+	defaultOpts.CombinedMap = make(map[string]SymCode)
+	defaultOpts.CombinedMap[":"] = 200
+	defaultOpts.CombinedMap[":="] = 201
+	defaultOpts.CombinedMap[":=="] = 203
+	defaultOpts.CombinedMap[">"] = 204
+	defaultOpts.CombinedMap["<"] = 205
+	defaultOpts.CombinedMap["-"] = 206
+	defaultOpts.CombinedMap[";"] = 207
+
+	defaultOpts.CommentTriplet = [3]rune{'(', '*', ')'}
+
+	sc := NewScanner(bufio.NewReader(bytes.NewBufferString(testString)), defaultOpts)
+	for sc.Error() == nil {
+		t.Log(sc.Get())
+	}
+}
+
+func TestRunner(t *testing.T) {
+	const testString = `
+		BEGIN
+
+		f asdf asdf xx x23 (* dfa3asd *) 33FH 3FU 234U 3.3  : := :== > < 0.12314 003141 -efef23 asdfd asf "dfsdfa sdf asdf " 'df' df'd' ;;      ;
+	`
+	defaultOpts := Opts{}
+	defaultOpts.IdentMap = make(map[string]SymCode)
+	defaultOpts.IdentMap["BEGIN"] = 100
+
+	defaultOpts.SpaceMap = make(map[string]SymCode)
+	defaultOpts.SpaceMap[" "] = 101
+	defaultOpts.SpaceMap["\n"] = 102
+
+	defaultOpts.NumContains = "ABCDEF"
+	defaultOpts.NumModifiers = "UH"
+
+	defaultOpts.CombinedMap = make(map[string]SymCode)
+	defaultOpts.CombinedMap[":"] = 200
+	defaultOpts.CombinedMap[":="] = 201
+	defaultOpts.CombinedMap[":=="] = 203
+	defaultOpts.CombinedMap[">"] = 204
+	defaultOpts.CombinedMap["<"] = 205
+	defaultOpts.CombinedMap["-"] = 206
+	defaultOpts.CombinedMap[";"] = 207
+
+	defaultOpts.CommentTriplet = [3]rune{'(', '*', ')'}
+
+	sc := NewScanner(bufio.NewReader(bytes.NewBufferString(testString)), defaultOpts)
+	run := NewRunner(sc, NewMarker(sc, func(msg ...interface{}) {
+		t.Log(msg...)
+	}))
+	Debug(run)
+	run.Expect(100, "begin expected", 102)
+	run.Run(207)
+}