package main import ( "bufio" "fmt" "os" ) type lexer struct { insertConcat bool } func (l *lexer) splitTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { n := 0 Whitespace: for ; n < len(data); n++ { switch data[n] { case ' ', '\t': l.insertConcat = false case '#': i := n + 1 for ; i < len(data) && data[i] != '\n'; i++ { } if i < len(data) { return i + 1, []byte(";"), nil } if atEOF { return i, nil, nil } return n, nil, nil case '\\': if n+1 < len(data) && data[n+1] == '\n' { n++ l.insertConcat = false continue } fallthrough default: break Whitespace } } data = data[n:] if len(data) == 0 { return n, nil, nil } switch data[0] { case '$': if l.insertConcat { l.insertConcat = false return n, []byte{}, nil } i := 1 for ; i < len(data); i++ { if b := data[i]; !(b == '_' || '0' <= b && b <= '9' || 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z') { break } } if i == len(data) && !atEOF { return n, nil, nil } if i < len(data) && data[i] == '(' { i++ } else { l.insertConcat = true } if i == 1 { return 0, nil, fmt.Errorf(`invalid token "$"`) } return n + i, data[:i], nil case '<', '>': if len(data) == 1 && !atEOF { return n, nil, nil } l.insertConcat = false if len(data) > 1 && data[1] == '(' { return n + 2, data[:2], nil } return n + 1, data[:1], nil case '\'': if l.insertConcat { l.insertConcat = false return n, []byte{}, nil } i := 1 for ; i < len(data); i++ { if data[i] == '\'' { if i+1 == len(data) && !atEOF { return n, nil, nil } if i+1 == len(data) || data[i+1] != '\'' { break } } } if i == len(data) { if atEOF { return 0, nil, fmt.Errorf("unterminated string") } return n, nil, nil } i++ l.insertConcat = true return n + i, data[:i], nil case '\n': l.insertConcat = false return n + 1, []byte(";"), nil case '&': if len(data) == 1 && !atEOF { return n, nil, nil } l.insertConcat = false if len(data) > 1 && data[1] == '&' { return n + 2, data[:2], nil } return n + 1, data[:1], nil case '|': if len(data) == 1 && !atEOF { return n, nil, nil } l.insertConcat = false if len(data) > 1 && data[1] == '|' { return n + 2, data[:2], nil } return n + 1, data[:1], nil case ';': l.insertConcat = false return n + 1, data[:1], nil case ')': l.insertConcat = true return n + 1, data[:1], nil case '*': if l.insertConcat { l.insertConcat = false return n, []byte{}, nil } l.insertConcat = true return n + 1, data[:1], nil default: if l.insertConcat { l.insertConcat = false return n, []byte{}, nil } i := 1 Word: for ; i < len(data); i++ { switch data[i] { case '\\': if i+1 == len(data) && !atEOF { return n, nil, nil } if i+1 < len(data) && data[i+1] == '\n' { break Word } case '&', '|', '*', ')', ';', '$', '<', '>', '\'', '#', ' ', '\t', '\n': break Word } } if i == len(data) && !atEOF { return n, nil, nil } l.insertConcat = true return n + i, data[:i], nil } } func splitTokens() bufio.SplitFunc { return new(lexer).splitTokens } func main() { scanner := bufio.NewScanner(os.Stdin) scanner.Split(splitTokens()) var tokens []string for scanner.Scan() { tokens = append(tokens, scanner.Text()) } if err := scanner.Err(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } fmt.Printf("%q\n", tokens) }