// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // The testing package implements a simple regular expression library. // It is a reduced version of the regular expression package suitable // for use in tests; it avoids many dependencies. // // The syntax of the regular expressions accepted is: // // regexp: // concatenation { '|' concatenation } // concatenation: // { closure } // closure: // term [ '*' | '+' | '?' ] // term: // '^' // '$' // '.' // character // '[' [ '^' ] character-ranges ']' // '(' regexp ')' // package testing import ( "utf8"; ) var debug = false // Error codes returned by failures to parse an expression. var ( ErrInternal = "internal error"; ErrUnmatchedLpar = "unmatched ''"; ErrUnmatchedRpar = "unmatched ''"; ErrUnmatchedLbkt = "unmatched '['"; ErrUnmatchedRbkt = "unmatched ']'"; ErrBadRange = "bad range in character class"; ErrExtraneousBackslash = "extraneous backslash"; ErrBadClosure = "repeated closure **, ++, etc."; ErrBareClosure = "closure applies to nothing"; ErrBadBackslash = "illegal backslash escape"; ) // An instruction executed by the NFA type instr interface { kind() int; // the type of this instruction: _CHAR, _ANY, etc. next() instr; // the instruction to execute after this one setNext(i instr); index() int; setIndex(i int); print(); } // Fields and methods common to all instructions type common struct { _next instr; _index int; } func (c *common) next() instr { return c._next } func (c *common) setNext(i instr) { c._next = i } func (c *common) index() int { return c._index } func (c *common) setIndex(i int) { c._index = i } // The representation of a compiled regular expression. // The public interface is entirely through methods. type Regexp struct { expr string; // the original expression inst []instr; start instr; nbra int; // number of brackets in expression, for subexpressions } const ( _START = // beginning of program iota; _END; // end of program: success _BOT; // '^' beginning of text _EOT; // '$' end of text _CHAR; // 'a' regular character _CHARCLASS; // [a-z] character class _ANY; // '.' any character including newline _NOTNL; // [^\n] special case: any character but newline _BRA; // '(' parenthesized expression _EBRA; // ')'; end of '(' parenthesized expression _ALT; // '|' alternation _NOP; // do nothing; makes it easy to link without patching ) // --- START start of program type _Start struct { common; } func (start *_Start) kind() int { return _START } func (start *_Start) print() { print("start") } // --- END end of program type _End struct { common; } func (end *_End) kind() int { return _END } func (end *_End) print() { print("end") } // --- BOT beginning of text type _Bot struct { common; } func (bot *_Bot) kind() int { return _BOT } func (bot *_Bot) print() { print("bot") } // --- EOT end of text type _Eot struct { common; } func (eot *_Eot) kind() int { return _EOT } func (eot *_Eot) print() { print("eot") } // --- CHAR a regular character type _Char struct { common; char int; } func (char *_Char) kind() int { return _CHAR } func (char *_Char) print() { print("char ", string(char.char)) } func newChar(char int) *_Char { c := new(_Char); c.char = char; return c; } // --- CHARCLASS [a-z] type _CharClass struct { common; char int; negate bool; // is character class negated? ([^a-z]) // stored pairwise: [a-z] is (a,z); x is (x,x): ranges []int; } func (cclass *_CharClass) kind() int { return _CHARCLASS } func (cclass *_CharClass) print() { print("charclass"); if cclass.negate { print(" (negated)") } for i := 0; i < len(cclass.ranges); i += 2 { l := cclass.ranges[i]; r := cclass.ranges[i+1]; if l == r { print(" [", string(l), "]") } else { print(" [", string(l), "-", string(r), "]") } } } func (cclass *_CharClass) addRange(a, b int) { // range is a through b inclusive n := len(cclass.ranges); if n >= cap(cclass.ranges) { nr := make([]int, n, 2*n); for i, j := range nr { nr[i] = j } cclass.ranges = nr; } cclass.ranges = cclass.ranges[0 : n+2]; cclass.ranges[n] = a; n++; cclass.ranges[n] = b; n++; } func (cclass *_CharClass) matches(c int) bool { for i := 0; i < len(cclass.ranges); i = i + 2 { min := cclass.ranges[i]; max := cclass.ranges[i+1]; if min <= c && c <= max { return !cclass.negate } } return cclass.negate; } func newCharClass() *_CharClass { c := new(_CharClass); c.ranges = make([]int, 0, 20); return c; } // --- ANY any character type _Any struct { common; } func (any *_Any) kind() int { return _ANY } func (any *_Any) print() { print("any") } // --- NOTNL any character but newline type _NotNl struct { common; } func (notnl *_NotNl) kind() int { return _NOTNL } func (notnl *_NotNl) print() { print("notnl") } // --- BRA parenthesized expression type _Bra struct { common; n int; // subexpression number } func (bra *_Bra) kind() int { return _BRA } func (bra *_Bra) print() { print("bra", bra.n) } // --- EBRA end of parenthesized expression type _Ebra struct { common; n int; // subexpression number } func (ebra *_Ebra) kind() int { return _EBRA } func (ebra *_Ebra) print() { print("ebra ", ebra.n) } // --- ALT alternation type _Alt struct { common; left instr; // other branch } func (alt *_Alt) kind() int { return _ALT } func (alt *_Alt) print() { print("alt(", alt.left.index(), ")") } // --- NOP no operation type _Nop struct { common; } func (nop *_Nop) kind() int { return _NOP } func (nop *_Nop) print() { print("nop") } func (re *Regexp) add(i instr) instr { n := len(re.inst); i.setIndex(len(re.inst)); if n >= cap(re.inst) { ni := make([]instr, n, 2*n); for i, j := range re.inst { ni[i] = j } re.inst = ni; } re.inst = re.inst[0 : n+1]; re.inst[n] = i; return i; } type parser struct { re *Regexp; error string; nlpar int; // number of unclosed lpars pos int; ch int; } const endOfFile = -1 func (p *parser) c() int { return p.ch } func (p *parser) nextc() int { if p.pos >= len(p.re.expr) { p.ch = endOfFile } else { c, w := utf8.DecodeRuneInString(p.re.expr[p.pos:]); p.ch = c; p.pos += w; } return p.ch; } func newParser(re *Regexp) *parser { p := new(parser); p.re = re; p.nextc(); // load p.ch return p; } func special(c int) bool { s := `\.+*?()|[]^$`; for i := 0; i < len(s); i++ { if c == int(s[i]) { return true } } return false; } func specialcclass(c int) bool { s := `\-[]`; for i := 0; i < len(s); i++ { if c == int(s[i]) { return true } } return false; } func (p *parser) charClass() instr { cc := newCharClass(); if p.c() == '^' { cc.negate = true; p.nextc(); } left := -1; for { switch c := p.c(); c { case ']', endOfFile: if left >= 0 { p.error = ErrBadRange; return nil; } // Is it [^\n]? if cc.negate && len(cc.ranges) == 2 && cc.ranges[0] == '\n' && cc.ranges[1] == '\n' { nl := new(_NotNl); p.re.add(nl); return nl; } p.re.add(cc); return cc; case '-': // do this before backslash processing p.error = ErrBadRange; return nil; case '\\': c = p.nextc(); switch { case c == endOfFile: p.error = ErrExtraneousBackslash; return nil; case c == 'n': c = '\n' case specialcclass(c): // c is as delivered default: p.error = ErrBadBackslash; return nil; } fallthrough; default: p.nextc(); switch { case left < 0: // first of pair if p.c() == '-' { // range p.nextc(); left = c; } else { // single char cc.addRange(c, c) } case left <= c: // second of pair cc.addRange(left, c); left = -1; default: p.error = ErrBadRange; return nil; } } } return nil; } func (p *parser) term() (start, end instr) { // term() is the leaf of the recursion, so it's sufficient to pick off the // error state here for early exit. // The other functions (closure(), concatenation() etc.) assume // it's safe to recur to here. if p.error != "" { return } switch c := p.c(); c { case '|', endOfFile: return nil, nil case '*', '+': p.error = ErrBareClosure; return; case ')': if p.nlpar == 0 { p.error = ErrUnmatchedRpar; return; } return nil, nil; case ']': p.error = ErrUnmatchedRbkt; return; case '^': p.nextc(); start = p.re.add(new(_Bot)); return start, start; case '$': p.nextc(); start = p.re.add(new(_Eot)); return start, start; case '.': p.nextc(); start = p.re.add(new(_Any)); return start, start; case '[': p.nextc(); start = p.charClass(); if p.error != "" { return } if p.c() != ']' { p.error = ErrUnmatchedLbkt; return; } p.nextc(); return start, start; case '(': p.nextc(); p.nlpar++; p.re.nbra++; // increment first so first subexpr is \1 nbra := p.re.nbra; start, end = p.regexp(); if p.c() != ')' { p.error = ErrUnmatchedLpar; return; } p.nlpar--; p.nextc(); bra := new(_Bra); p.re.add(bra); ebra := new(_Ebra); p.re.add(ebra); bra.n = nbra; ebra.n = nbra; if start == nil { if end == nil { p.error = ErrInternal; return; } start = ebra; } else { end.setNext(ebra) } bra.setNext(start); return bra, ebra; case '\\': c = p.nextc(); switch { case c == endOfFile: p.error = ErrExtraneousBackslash; return; case c == 'n': c = '\n' case special(c): // c is as delivered default: p.error = ErrBadBackslash; return; } fallthrough; default: p.nextc(); start = newChar(c); p.re.add(start); return start, start; } panic("unreachable"); } func (p *parser) closure() (start, end instr) { start, end = p.term(); if start == nil || p.error != "" { return } switch p.c() { case '*': // (start,end)*: alt := new(_Alt); p.re.add(alt); end.setNext(alt); // after end, do alt alt.left = start; // alternate brach: return to start start = alt; // alt becomes new (start, end) end = alt; case '+': // (start,end)+: alt := new(_Alt); p.re.add(alt); end.setNext(alt); // after end, do alt alt.left = start; // alternate brach: return to start end = alt; // start is unchanged; end is alt case '?': // (start,end)?: alt := new(_Alt); p.re.add(alt); nop := new(_Nop); p.re.add(nop); alt.left = start; // alternate branch is start alt.setNext(nop); // follow on to nop end.setNext(nop); // after end, go to nop start = alt; // start is now alt end = nop; // end is nop pointed to by both branches default: return } switch p.nextc() { case '*', '+', '?': p.error = ErrBadClosure } return; } func (p *parser) concatenation() (start, end instr) { for { nstart, nend := p.closure(); if p.error != "" { return } switch { case nstart == nil: // end of this concatenation if start == nil { // this is the empty string nop := p.re.add(new(_Nop)); return nop, nop; } return; case start == nil: // this is first element of concatenation start, end = nstart, nend default: end.setNext(nstart); end = nend; } } panic("unreachable"); } func (p *parser) regexp() (start, end instr) { start, end = p.concatenation(); if p.error != "" { return } for { switch p.c() { default: return case '|': p.nextc(); nstart, nend := p.concatenation(); if p.error != "" { return } alt := new(_Alt); p.re.add(alt); alt.left = start; alt.setNext(nstart); nop := new(_Nop); p.re.add(nop); end.setNext(nop); nend.setNext(nop); start, end = alt, nop; } } panic("unreachable"); } func unNop(i instr) instr { for i.kind() == _NOP { i = i.next() } return i; } func (re *Regexp) eliminateNops() { for i := 0; i < len(re.inst); i++ { inst := re.inst[i]; if inst.kind() == _END { continue } inst.setNext(unNop(inst.next())); if inst.kind() == _ALT { alt := inst.(*_Alt); alt.left = unNop(alt.left); } } } func (re *Regexp) doParse() string { p := newParser(re); start := new(_Start); re.add(start); s, e := p.regexp(); if p.error != "" { return p.error } start.setNext(s); re.start = start; e.setNext(re.add(new(_End))); re.eliminateNops(); return p.error; } // CompileRegexp parses a regular expression and returns, if successful, a Regexp // object that can be used to match against text. func CompileRegexp(str string) (regexp *Regexp, error string) { regexp = new(Regexp); regexp.expr = str; regexp.inst = make([]instr, 0, 20); error = regexp.doParse(); return; } // MustCompileRegexp is like CompileRegexp but panics if the expression cannot be parsed. // It simplifies safe initialization of global variables holding compiled regular // expressions. func MustCompile(str string) *Regexp { regexp, error := CompileRegexp(str); if error != "" { panicln(`regexp: compiling "`, str, `": `, error) } return regexp; } type state struct { inst instr; // next instruction to execute match []int; // pairs of bracketing submatches. 0th is start,end } // Append new state to to-do list. Leftmost-longest wins so avoid // adding a state that's already active. func addState(s []state, inst instr, match []int) []state { index := inst.index(); l := len(s); pos := match[0]; // TODO: Once the state is a vector and we can do insert, have inputs always // go in order correctly and this "earlier" test is never necessary, for i := 0; i < l; i++ { if s[i].inst.index() == index && // same instruction s[i].match[0] < pos { // earlier match already going; lefmost wins return s } } if l == cap(s) { s1 := make([]state, 2*l)[0:l]; for i := 0; i < l; i++ { s1[i] = s[i] } s = s1; } s = s[0 : l+1]; s[l].inst = inst; s[l].match = match; return s; } // Accepts either string or bytes - the logic is identical either way. // If bytes == nil, scan str. func (re *Regexp) doExecute(str string, bytes []byte, pos int) []int { var s [2][]state; // TODO: use a vector when state values (not ptrs) can be vector elements s[0] = make([]state, 10)[0:0]; s[1] = make([]state, 10)[0:0]; in, out := 0, 1; var final state; found := false; end := len(str); if bytes != nil { end = len(bytes) } for pos <= end { if !found { // prime the pump if we haven't seen a match yet match := make([]int, 2*(re.nbra+1)); for i := 0; i < len(match); i++ { match[i] = -1 // no match seen; catches cases like "a(b)?c" on "ac" } match[0] = pos; s[out] = addState(s[out], re.start.next(), match); } in, out = out, in; // old out state is new in state s[out] = s[out][0:0]; // clear out state if len(s[in]) == 0 { // machine has completed break } charwidth := 1; c := endOfFile; if pos < end { if bytes == nil { c, charwidth = utf8.DecodeRuneInString(str[pos:end]) } else { c, charwidth = utf8.DecodeRune(bytes[pos:end]) } } for i := 0; i < len(s[in]); i++ { st := s[in][i]; switch s[in][i].inst.kind() { case _BOT: if pos == 0 { s[in] = addState(s[in], st.inst.next(), st.match) } case _EOT: if pos == end { s[in] = addState(s[in], st.inst.next(), st.match) } case _CHAR: if c == st.inst.(*_Char).char { s[out] = addState(s[out], st.inst.next(), st.match) } case _CHARCLASS: if st.inst.(*_CharClass).matches(c) { s[out] = addState(s[out], st.inst.next(), st.match) } case _ANY: if c != endOfFile { s[out] = addState(s[out], st.inst.next(), st.match) } case _NOTNL: if c != endOfFile && c != '\n' { s[out] = addState(s[out], st.inst.next(), st.match) } case _BRA: n := st.inst.(*_Bra).n; st.match[2*n] = pos; s[in] = addState(s[in], st.inst.next(), st.match); case _EBRA: n := st.inst.(*_Ebra).n; st.match[2*n+1] = pos; s[in] = addState(s[in], st.inst.next(), st.match); case _ALT: s[in] = addState(s[in], st.inst.(*_Alt).left, st.match); // give other branch a copy of this match vector s1 := make([]int, 2*(re.nbra+1)); for i := 0; i < len(s1); i++ { s1[i] = st.match[i] } s[in] = addState(s[in], st.inst.next(), s1); case _END: // choose leftmost longest if !found || // first st.match[0] < final.match[0] || // leftmost (st.match[0] == final.match[0] && pos > final.match[1]) { // longest final = st; final.match[1] = pos; } found = true; default: st.inst.print(); panic("unknown instruction in execute"); } } pos += charwidth; } return final.match; } // ExecuteString matches the Regexp against the string s. // The return value is an array of integers, in pairs, identifying the positions of // substrings matched by the expression. // s[a[0]:a[1]] is the substring matched by the entire expression. // s[a[2*i]:a[2*i+1]] for i > 0 is the substring matched by the ith parenthesized subexpression. // A negative value means the subexpression did not match any element of the string. // An empty array means "no match". func (re *Regexp) ExecuteString(s string) (a []int) { return re.doExecute(s, nil, 0) } // Execute matches the Regexp against the byte slice b. // The return value is an array of integers, in pairs, identifying the positions of // subslices matched by the expression. // b[a[0]:a[1]] is the subslice matched by the entire expression. // b[a[2*i]:a[2*i+1]] for i > 0 is the subslice matched by the ith parenthesized subexpression. // A negative value means the subexpression did not match any element of the slice. // An empty array means "no match". func (re *Regexp) Execute(b []byte) (a []int) { return re.doExecute("", b, 0) } // MatchString returns whether the Regexp matches the string s. // The return value is a boolean: true for match, false for no match. func (re *Regexp) MatchString(s string) bool { return len(re.doExecute(s, nil, 0)) > 0 } // Match returns whether the Regexp matches the byte slice b. // The return value is a boolean: true for match, false for no match. func (re *Regexp) Match(b []byte) bool { return len(re.doExecute("", b, 0)) > 0 } // MatchStrings matches the Regexp against the string s. // The return value is an array of strings matched by the expression. // a[0] is the substring matched by the entire expression. // a[i] for i > 0 is the substring matched by the ith parenthesized subexpression. // An empty array means ``no match''. func (re *Regexp) MatchStrings(s string) (a []string) { r := re.doExecute(s, nil, 0); if r == nil { return nil } a = make([]string, len(r)/2); for i := 0; i < len(r); i += 2 { if r[i] != -1 { // -1 means no match for this subexpression a[i/2] = s[r[i]:r[i+1]] } } return; } // MatchSlices matches the Regexp against the byte slice b. // The return value is an array of subslices matched by the expression. // a[0] is the subslice matched by the entire expression. // a[i] for i > 0 is the subslice matched by the ith parenthesized subexpression. // An empty array means ``no match''. func (re *Regexp) MatchSlices(b []byte) (a [][]byte) { r := re.doExecute("", b, 0); if r == nil { return nil } a = make([][]byte, len(r)/2); for i := 0; i < len(r); i += 2 { if r[i] != -1 { // -1 means no match for this subexpression a[i/2] = b[r[i]:r[i+1]] } } return; } // MatchString checks whether a textual regular expression // matches a string. More complicated queries need // to use Compile and the full Regexp interface. func MatchString(pattern string, s string) (matched bool, error string) { re, err := CompileRegexp(pattern); if err != "" { return false, err } return re.MatchString(s), ""; } // Match checks whether a textual regular expression // matches a byte slice. More complicated queries need // to use Compile and the full Regexp interface. func Match(pattern string, b []byte) (matched bool, error string) { re, err := CompileRegexp(pattern); if err != "" { return false, err } return re.Match(b), ""; }