github-code/leanote
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vendor

Browse Source
This commit is contained in:
lealife committed 2017-06-22 13:18:16 +08:00
1 parent 2654b684df
commit b140cd538f
549 files changed
+185885 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vendor/code.google.com/p/cascadia/LICENSE
Generated Vendored Executable
+24
View File
@@ -0,0 +1,24 @@
Copyright (c) 2011 Andy Balholm. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/code.google.com/p/cascadia/Makefile
Generated Vendored
+12
View File
@@ -0,0 +1,12 @@
include $(GOROOT)/src/Make.inc
TARG=cascadia
GOFILES= \
parser.go \
selector.go \
include $(GOROOT)/src/Make.pkg
format:
gofmt -w ${GOFILES} *_test.go
vendor/code.google.com/p/cascadia/parser.go
Generated Vendored
+812
View File
@@ -0,0 +1,812 @@
// The cascadia package is an implementation of CSS selectors.
package cascadia
import (
"code.google.com/p/go.net/html"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
// a parser for CSS selectors
type parser struct {
s string // the source text
i int // the current position
}
// parseEscape parses a backslash escape.
func (p *parser) parseEscape() (result string, err error) {
if len(p.s) < p.i+2 || p.s[p.i] != '\\' {
return "", errors.New("invalid escape sequence")
}
start := p.i + 1
c := p.s[start]
switch {
case c == '\r' || c == '\n' || c == '\f':
return "", errors.New("escaped line ending outside string")
case hexDigit(c):
// unicode escape (hex)
var i int
for i = start; i < p.i+6 && i < len(p.s) && hexDigit(p.s[i]); i++ {
// empty
}
v, _ := strconv.ParseUint(p.s[start:i], 16, 21)
if len(p.s) > i {
switch p.s[i] {
case '\r':
i++
if len(p.s) > i && p.s[i] == '\n' {
i++
}
case ' ', '\t', '\n', '\f':
i++
}
}
p.i = i
return string(rune(v)), nil
}
// Return the literal character after the backslash.
result = p.s[start : start+1]
p.i += 2
return result, nil
}
func hexDigit(c byte) bool {
return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F'
}
// nameStart returns whether c can be the first character of an identifier
// (not counting an initial hyphen, or an escape sequence).
func nameStart(c byte) bool {
return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_' || c > 127
}
// nameChar returns whether c can be a character within an identifier
// (not counting an escape sequence).
func nameChar(c byte) bool {
return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_' || c > 127 ||
c == '-' || '0' <= c && c <= '9'
}
// parseIdentifier parses an identifier.
func (p *parser) parseIdentifier() (result string, err error) {
startingDash := false
if len(p.s) > p.i && p.s[p.i] == '-' {
startingDash = true
p.i++
}
if len(p.s) <= p.i {
return "", errors.New("expected identifier, found EOF instead")
}
if c := p.s[p.i]; !(nameStart(c) || c == '\\') {
return "", fmt.Errorf("expected identifier, found %c instead", c)
}
result, err = p.parseName()
if startingDash && err == nil {
result = "-" + result
}
return
}
// parseName parses a name (which is like an identifier, but doesn't have
// extra restrictions on the first character).
func (p *parser) parseName() (result string, err error) {
i := p.i
loop:
for i < len(p.s) {
c := p.s[i]
switch {
case nameChar(c):
start := i
for i < len(p.s) && nameChar(p.s[i]) {
i++
}
result += p.s[start:i]
case c == '\\':
p.i = i
val, err := p.parseEscape()
if err != nil {
return "", err
}
i = p.i
result += val
default:
break loop
}
}
if result == "" {
return "", errors.New("expected name, found EOF instead")
}
p.i = i
return result, nil
}
// parseString parses a single- or double-quoted string.
func (p *parser) parseString() (result string, err error) {
i := p.i
if len(p.s) < i+2 {
return "", errors.New("expected string, found EOF instead")
}
quote := p.s[i]
i++
loop:
for i < len(p.s) {
switch p.s[i] {
case '\\':
if len(p.s) > i+1 {
switch c := p.s[i+1]; c {
case '\r':
if len(p.s) > i+2 && p.s[i+2] == '\n' {
i += 3
continue loop
}
fallthrough
case '\n', '\f':
i += 2
continue loop
}
}
p.i = i
val, err := p.parseEscape()
if err != nil {
return "", err
}
i = p.i
result += val
case quote:
break loop
case '\r', '\n', '\f':
return "", errors.New("unexpected end of line in string")
default:
start := i
for i < len(p.s) {
if c := p.s[i]; c == quote || c == '\\' || c == '\r' || c == '\n' || c == '\f' {
break
}
i++
}
result += p.s[start:i]
}
}
if i >= len(p.s) {
return "", errors.New("EOF in string")
}
// Consume the final quote.
i++
p.i = i
return result, nil
}
// parseRegex parses a regular expression; the end is defined by encountering an
// unmatched closing ')' or ']' which is not consumed
func (p *parser) parseRegex() (rx *regexp.Regexp, err error) {
i := p.i
if len(p.s) < i+2 {
return nil, errors.New("expected regular expression, found EOF instead")
}
// number of open parens or brackets;
// when it becomes negative, finished parsing regex
open := 0
loop:
for i < len(p.s) {
switch p.s[i] {
case '(', '[':
open++
case ')', ']':
open--
if open < 0 {
break loop
}
}
i++
}
if i >= len(p.s) {
return nil, errors.New("EOF in regular expression")
}
rx, err = regexp.Compile(p.s[p.i:i])
p.i = i
return rx, err
}
// skipWhitespace consumes whitespace characters and comments.
// It returns true if there was actually anything to skip.
func (p *parser) skipWhitespace() bool {
i := p.i
for i < len(p.s) {
switch p.s[i] {
case ' ', '\t', '\r', '\n', '\f':
i++
continue
case '/':
if strings.HasPrefix(p.s[i:], "/*") {
end := strings.Index(p.s[i+len("/*"):], "*/")
if end != -1 {
i += end + len("/**/")
continue
}
}
}
break
}
if i > p.i {
p.i = i
return true
}
return false
}
// consumeParenthesis consumes an opening parenthesis and any following
// whitespace. It returns true if there was actually a parenthesis to skip.
func (p *parser) consumeParenthesis() bool {
if p.i < len(p.s) && p.s[p.i] == '(' {
p.i++
p.skipWhitespace()
return true
}
return false
}
// consumeClosingParenthesis consumes a closing parenthesis and any preceding
// whitespace. It returns true if there was actually a parenthesis to skip.
func (p *parser) consumeClosingParenthesis() bool {
i := p.i
p.skipWhitespace()
if p.i < len(p.s) && p.s[p.i] == ')' {
p.i++
return true
}
p.i = i
return false
}
// parseTypeSelector parses a type selector (one that matches by tag name).
func (p *parser) parseTypeSelector() (result Selector, err error) {
tag, err := p.parseIdentifier()
if err != nil {
return nil, err
}
return typeSelector(tag), nil
}
// parseIDSelector parses a selector that matches by id attribute.
func (p *parser) parseIDSelector() (Selector, error) {
if p.i >= len(p.s) {
return nil, fmt.Errorf("expected id selector (#id), found EOF instead")
}
if p.s[p.i] != '#' {
return nil, fmt.Errorf("expected id selector (#id), found '%c' instead", p.s[p.i])
}
p.i++
id, err := p.parseName()
if err != nil {
return nil, err
}
return attributeEqualsSelector("id", id), nil
}
// parseClassSelector parses a selector that matches by class attribute.
func (p *parser) parseClassSelector() (Selector, error) {
if p.i >= len(p.s) {
return nil, fmt.Errorf("expected class selector (.class), found EOF instead")
}
if p.s[p.i] != '.' {
return nil, fmt.Errorf("expected class selector (.class), found '%c' instead", p.s[p.i])
}
p.i++
class, err := p.parseIdentifier()
if err != nil {
return nil, err
}
return attributeIncludesSelector("class", class), nil
}
// parseAttributeSelector parses a selector that matches by attribute value.
func (p *parser) parseAttributeSelector() (Selector, error) {
if p.i >= len(p.s) {
return nil, fmt.Errorf("expected attribute selector ([attribute]), found EOF instead")
}
if p.s[p.i] != '[' {
return nil, fmt.Errorf("expected attribute selector ([attribute]), found '%c' instead", p.s[p.i])
}
p.i++
p.skipWhitespace()
key, err := p.parseIdentifier()
if err != nil {
return nil, err
}
p.skipWhitespace()
if p.i >= len(p.s) {
return nil, errors.New("unexpected EOF in attribute selector")
}
if p.s[p.i] == ']' {
p.i++
return attributeExistsSelector(key), nil
}
if p.i+2 >= len(p.s) {
return nil, errors.New("unexpected EOF in attribute selector")
}
op := p.s[p.i : p.i+2]
if op[0] == '=' {
op = "="
} else if op[1] != '=' {
return nil, fmt.Errorf(`expected equality operator, found "%s" instead`, op)
}
p.i += len(op)
p.skipWhitespace()
if p.i >= len(p.s) {
return nil, errors.New("unexpected EOF in attribute selector")
}
var val string
var rx *regexp.Regexp
if op == "#=" {
rx, err = p.parseRegex()
} else {
switch p.s[p.i] {
case '\'', '"':
val, err = p.parseString()
default:
val, err = p.parseIdentifier()
}
}
if err != nil {
return nil, err
}
p.skipWhitespace()
if p.i >= len(p.s) {
return nil, errors.New("unexpected EOF in attribute selector")
}
if p.s[p.i] != ']' {
return nil, fmt.Errorf("expected ']', found '%c' instead", p.s[p.i])
}
p.i++
switch op {
case "=":
return attributeEqualsSelector(key, val), nil
case "~=":
return attributeIncludesSelector(key, val), nil
case "|=":
return attributeDashmatchSelector(key, val), nil
case "^=":
return attributePrefixSelector(key, val), nil
case "$=":
return attributeSuffixSelector(key, val), nil
case "*=":
return attributeSubstringSelector(key, val), nil
case "#=":
return attributeRegexSelector(key, rx), nil
}
return nil, fmt.Errorf("attribute operator %q is not supported", op)
}
var expectedParenthesis = errors.New("expected '(' but didn't find it")
var expectedClosingParenthesis = errors.New("expected ')' but didn't find it")
// parsePseudoclassSelector parses a pseudoclass selector like :not(p).
func (p *parser) parsePseudoclassSelector() (Selector, error) {
if p.i >= len(p.s) {
return nil, fmt.Errorf("expected pseudoclass selector (:pseudoclass), found EOF instead")
}
if p.s[p.i] != ':' {
return nil, fmt.Errorf("expected attribute selector (:pseudoclass), found '%c' instead", p.s[p.i])
}
p.i++
name, err := p.parseIdentifier()
if err != nil {
return nil, err
}
name = toLowerASCII(name)
switch name {
case "not", "has", "haschild":
if !p.consumeParenthesis() {
return nil, expectedParenthesis
}
sel, err := p.parseSelectorGroup()
if err != nil {
return nil, err
}
if !p.consumeClosingParenthesis() {
return nil, expectedClosingParenthesis
}
switch name {
case "not":
return negatedSelector(sel), nil
case "has":
return hasDescendantSelector(sel), nil
case "haschild":
return hasChildSelector(sel), nil
}
case "contains", "containsown":
if !p.consumeParenthesis() {
return nil, expectedParenthesis
}
var val string
switch p.s[p.i] {
case '\'', '"':
val, err = p.parseString()
default:
val, err = p.parseIdentifier()
}
if err != nil {
return nil, err
}
val = strings.ToLower(val)
p.skipWhitespace()
if p.i >= len(p.s) {
return nil, errors.New("unexpected EOF in pseudo selector")
}
if !p.consumeClosingParenthesis() {
return nil, expectedClosingParenthesis
}
switch name {
case "contains":
return textSubstrSelector(val), nil
case "containsown":
return ownTextSubstrSelector(val), nil
}
case "matches", "matchesown":
if !p.consumeParenthesis() {
return nil, expectedParenthesis
}
rx, err := p.parseRegex()
if err != nil {
return nil, err
}
if p.i >= len(p.s) {
return nil, errors.New("unexpected EOF in pseudo selector")
}
if !p.consumeClosingParenthesis() {
return nil, expectedClosingParenthesis
}
switch name {
case "matches":
return textRegexSelector(rx), nil
case "matchesown":
return ownTextRegexSelector(rx), nil
}
case "nth-child", "nth-last-child", "nth-of-type", "nth-last-of-type":
if !p.consumeParenthesis() {
return nil, expectedParenthesis
}
a, b, err := p.parseNth()
if err != nil {
return nil, err
}
if !p.consumeClosingParenthesis() {
return nil, expectedClosingParenthesis
}
return nthChildSelector(a, b,
name == "nth-last-child" || name == "nth-last-of-type",
name == "nth-of-type" || name == "nth-last-of-type"),
nil
case "first-child":
return nthChildSelector(0, 1, false, false), nil
case "last-child":
return nthChildSelector(0, 1, true, false), nil
case "first-of-type":
return nthChildSelector(0, 1, false, true), nil
case "last-of-type":
return nthChildSelector(0, 1, true, true), nil
case "only-child":
return onlyChildSelector(false), nil
case "only-of-type":
return onlyChildSelector(true), nil
case "empty":
return emptyElementSelector, nil
}
return nil, fmt.Errorf("unknown pseudoclass :%s", name)
}
// parseInteger parses a decimal integer.
func (p *parser) parseInteger() (int, error) {
i := p.i
start := i
for i < len(p.s) && '0' <= p.s[i] && p.s[i] <= '9' {
i++
}
if i == start {
return 0, errors.New("expected integer, but didn't find it.")
}
p.i = i
val, err := strconv.Atoi(p.s[start:i])
if err != nil {
return 0, err
}
return val, nil
}
// parseNth parses the argument for :nth-child (normally of the form an+b).
func (p *parser) parseNth() (a, b int, err error) {
// initial state
if p.i >= len(p.s) {
goto eof
}
switch p.s[p.i] {
case '-':
p.i++
goto negativeA
case '+':
p.i++
goto positiveA
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
goto positiveA
case 'n', 'N':
a = 1
p.i++
goto readN
case 'o', 'O', 'e', 'E':
id, err := p.parseName()
if err != nil {
return 0, 0, err
}
id = toLowerASCII(id)
if id == "odd" {
return 2, 1, nil
}
if id == "even" {
return 2, 0, nil
}
return 0, 0, fmt.Errorf("expected 'odd' or 'even', but found '%s' instead", id)
default:
goto invalid
}
positiveA:
if p.i >= len(p.s) {
goto eof
}
switch p.s[p.i] {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
a, err = p.parseInteger()
if err != nil {
return 0, 0, err
}
goto readA
case 'n', 'N':
a = 1
p.i++
goto readN
default:
goto invalid
}
negativeA:
if p.i >= len(p.s) {
goto eof
}
switch p.s[p.i] {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
a, err = p.parseInteger()
if err != nil {
return 0, 0, err
}
a = -a
goto readA
case 'n', 'N':
a = -1
p.i++
goto readN
default:
goto invalid
}
readA:
if p.i >= len(p.s) {
goto eof
}
switch p.s[p.i] {
case 'n', 'N':
p.i++
goto readN
default:
// The number we read as a is actually b.
return 0, a, nil
}
readN:
p.skipWhitespace()
if p.i >= len(p.s) {
goto eof
}
switch p.s[p.i] {
case '+':
p.i++
p.skipWhitespace()
b, err = p.parseInteger()
if err != nil {
return 0, 0, err
}
return a, b, nil
case '-':
p.i++
p.skipWhitespace()
b, err = p.parseInteger()
if err != nil {
return 0, 0, err
}
return a, -b, nil
default:
return a, 0, nil
}
eof:
return 0, 0, errors.New("unexpected EOF while attempting to parse expression of form an+b")
invalid:
return 0, 0, errors.New("unexpected character while attempting to parse expression of form an+b")
}
// parseSimpleSelectorSequence parses a selector sequence that applies to
// a single element.
func (p *parser) parseSimpleSelectorSequence() (Selector, error) {
var result Selector
if p.i >= len(p.s) {
return nil, errors.New("expected selector, found EOF instead")
}
switch p.s[p.i] {
case '*':
// It's the universal selector. Just skip over it, since it doesn't affect the meaning.
p.i++
case '#', '.', '[', ':':
// There's no type selector. Wait to process the other till the main loop.
default:
r, err := p.parseTypeSelector()
if err != nil {
return nil, err
}
result = r
}
loop:
for p.i < len(p.s) {
var ns Selector
var err error
switch p.s[p.i] {
case '#':
ns, err = p.parseIDSelector()
case '.':
ns, err = p.parseClassSelector()
case '[':
ns, err = p.parseAttributeSelector()
case ':':
ns, err = p.parsePseudoclassSelector()
default:
break loop
}
if err != nil {
return nil, err
}
if result == nil {
result = ns
} else {
result = intersectionSelector(result, ns)
}
}
if result == nil {
result = func(n *html.Node) bool {
return true
}
}
return result, nil
}
// parseSelector parses a selector that may include combinators.
func (p *parser) parseSelector() (result Selector, err error) {
p.skipWhitespace()
result, err = p.parseSimpleSelectorSequence()
if err != nil {
return
}
for {
var combinator byte
if p.skipWhitespace() {
combinator = ' '
}
if p.i >= len(p.s) {
return
}
switch p.s[p.i] {
case '+', '>', '~':
combinator = p.s[p.i]
p.i++
p.skipWhitespace()
case ',', ')':
// These characters can't begin a selector, but they can legally occur after one.
return
}
if combinator == 0 {
return
}
c, err := p.parseSimpleSelectorSequence()
if err != nil {
return nil, err
}
switch combinator {
case ' ':
result = descendantSelector(result, c)
case '>':
result = childSelector(result, c)
case '+':
result = siblingSelector(result, c, true)
case '~':
result = siblingSelector(result, c, false)
}
}
panic("unreachable")
}
// parseSelectorGroup parses a group of selectors, separated by commas.
func (p *parser) parseSelectorGroup() (result Selector, err error) {
result, err = p.parseSelector()
if err != nil {
return
}
for p.i < len(p.s) {
if p.s[p.i] != ',' {
return result, nil
}
p.i++
c, err := p.parseSelector()
if err != nil {
return nil, err
}
result = unionSelector(result, c)
}
return
}
vendor/code.google.com/p/cascadia/selector.go
Generated Vendored
+487
View File
@@ -0,0 +1,487 @@
package cascadia
import (
"bytes"
"code.google.com/p/go.net/html"
"fmt"
"regexp"
"strings"
)
// the Selector type, and functions for creating them
// A Selector is a function which tells whether a node matches or not.
type Selector func(*html.Node) bool
// hasChildMatch returns whether n has any child that matches a.
func hasChildMatch(n *html.Node, a Selector) bool {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if a(c) {
return true
}
}
return false
}
// hasDescendantMatch performs a depth-first search of n's descendants,
// testing whether any of them match a. It returns true as soon as a match is
// found, or false if no match is found.
func hasDescendantMatch(n *html.Node, a Selector) bool {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if a(c) || (c.Type == html.ElementNode && hasDescendantMatch(c, a)) {
return true
}
}
return false
}
// Compile parses a selector and returns, if successful, a Selector object
// that can be used to match against html.Node objects.
func Compile(sel string) (Selector, error) {
p := &parser{s: sel}
compiled, err := p.parseSelectorGroup()
if err != nil {
return nil, err
}
if p.i < len(sel) {
return nil, fmt.Errorf("parsing %q: %d bytes left over", sel, len(sel)-p.i)
}
return compiled, nil
}
// MustCompile is like Compile, but panics instead of returning an error.
func MustCompile(sel string) Selector {
compiled, err := Compile(sel)
if err != nil {
panic(err)
}
return compiled
}
// MatchAll returns a slice of the nodes that match the selector,
// from n and its children.
func (s Selector) MatchAll(n *html.Node) []*html.Node {
return s.matchAllInto(n, nil)
}
func (s Selector) matchAllInto(n *html.Node, storage []*html.Node) []*html.Node {
if s(n) {
storage = append(storage, n)
}
for child := n.FirstChild; child != nil; child = child.NextSibling {
storage = s.matchAllInto(child, storage)
}
return storage
}
// Match returns true if the node matches the selector.
func (s Selector) Match(n *html.Node) bool {
return s(n)
}
// Filter returns the nodes in nodes that match the selector.
func (s Selector) Filter(nodes []*html.Node) (result []*html.Node) {
for _, n := range nodes {
if s(n) {
result = append(result, n)
}
}
return result
}
// typeSelector returns a Selector that matches elements with a given tag name.
func typeSelector(tag string) Selector {
tag = toLowerASCII(tag)
return func(n *html.Node) bool {
return n.Type == html.ElementNode && n.Data == tag
}
}
// toLowerASCII returns s with all ASCII capital letters lowercased.
func toLowerASCII(s string) string {
var b []byte
for i := 0; i < len(s); i++ {
if c := s[i]; 'A' <= c && c <= 'Z' {
if b == nil {
b = make([]byte, len(s))
copy(b, s)
}
b[i] = s[i] + ('a' - 'A')
}
}
if b == nil {
return s
}
return string(b)
}
// attributeSelector returns a Selector that matches elements
// where the attribute named key satisifes the function f.
func attributeSelector(key string, f func(string) bool) Selector {
key = toLowerASCII(key)
return func(n *html.Node) bool {
if n.Type != html.ElementNode {
return false
}
for _, a := range n.Attr {
if a.Key == key && f(a.Val) {
return true
}
}
return false
}
}
// attributeExistsSelector returns a Selector that matches elements that have
// an attribute named key.
func attributeExistsSelector(key string) Selector {
return attributeSelector(key, func(string) bool { return true })
}
// attributeEqualsSelector returns a Selector that matches elements where
// the attribute named key has the value val.
func attributeEqualsSelector(key, val string) Selector {
return attributeSelector(key,
func(s string) bool {
return s == val
})
}
// attributeIncludesSelector returns a Selector that matches elements where
// the attribute named key is a whitespace-separated list that includes val.
func attributeIncludesSelector(key, val string) Selector {
return attributeSelector(key,
func(s string) bool {
for s != "" {
i := strings.IndexAny(s, " \t\r\n\f")
if i == -1 {
return s == val
}
if s[:i] == val {
return true
}
s = s[i+1:]
}
return false
})
}
// attributeDashmatchSelector returns a Selector that matches elements where
// the attribute named key equals val or starts with val plus a hyphen.
func attributeDashmatchSelector(key, val string) Selector {
return attributeSelector(key,
func(s string) bool {
if s == val {
return true
}
if len(s) <= len(val) {
return false
}
if s[:len(val)] == val && s[len(val)] == '-' {
return true
}
return false
})
}
// attributePrefixSelector returns a Selector that matches elements where
// the attribute named key starts with val.
func attributePrefixSelector(key, val string) Selector {
return attributeSelector(key,
func(s string) bool {
return strings.HasPrefix(s, val)
})
}
// attributeSuffixSelector returns a Selector that matches elements where
// the attribute named key ends with val.
func attributeSuffixSelector(key, val string) Selector {
return attributeSelector(key,
func(s string) bool {
return strings.HasSuffix(s, val)
})
}
// attributeSubstringSelector returns a Selector that matches nodes where
// the attribute named key contains val.
func attributeSubstringSelector(key, val string) Selector {
return attributeSelector(key,
func(s string) bool {
return strings.Contains(s, val)
})
}
// attributeRegexSelector returns a Selector that matches nodes where
// the attribute named key matches the regular expression rx
func attributeRegexSelector(key string, rx *regexp.Regexp) Selector {
return attributeSelector(key,
func(s string) bool {
return rx.MatchString(s)
})
}
// intersectionSelector returns a selector that matches nodes that match
// both a and b.
func intersectionSelector(a, b Selector) Selector {
return func(n *html.Node) bool {
return a(n) && b(n)
}
}
// unionSelector returns a selector that matches elements that match
// either a or b.
func unionSelector(a, b Selector) Selector {
return func(n *html.Node) bool {
return a(n) || b(n)
}
}
// negatedSelector returns a selector that matches elements that do not match a.
func negatedSelector(a Selector) Selector {
return func(n *html.Node) bool {
if n.Type != html.ElementNode {
return false
}
return !a(n)
}
}
// writeNodeText writes the text contained in n and its descendants to b.
func writeNodeText(n *html.Node, b *bytes.Buffer) {
switch n.Type {
case html.TextNode:
b.WriteString(n.Data)
case html.ElementNode:
for c := n.FirstChild; c != nil; c = c.NextSibling {
writeNodeText(c, b)
}
}
}
// nodeText returns the text contained in n and its descendants.
func nodeText(n *html.Node) string {
var b bytes.Buffer
writeNodeText(n, &b)
return b.String()
}
// nodeOwnText returns the contents of the text nodes that are direct
// children of n.
func nodeOwnText(n *html.Node) string {
var b bytes.Buffer
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.TextNode {
b.WriteString(c.Data)
}
}
return b.String()
}
// textSubstrSelector returns a selector that matches nodes that
// contain the given text.
func textSubstrSelector(val string) Selector {
return func(n *html.Node) bool {
text := strings.ToLower(nodeText(n))
return strings.Contains(text, val)
}
}
// ownTextSubstrSelector returns a selector that matches nodes that
// directly contain the given text
func ownTextSubstrSelector(val string) Selector {
return func(n *html.Node) bool {
text := strings.ToLower(nodeOwnText(n))
return strings.Contains(text, val)
}
}
// textRegexSelector returns a selector that matches nodes whose text matches
// the specified regular expression
func textRegexSelector(rx *regexp.Regexp) Selector {
return func(n *html.Node) bool {
return rx.MatchString(nodeText(n))
}
}
// ownTextRegexSelector returns a selector that matches nodes whose text
// directly matches the specified regular expression
func ownTextRegexSelector(rx *regexp.Regexp) Selector {
return func(n *html.Node) bool {
return rx.MatchString(nodeOwnText(n))
}
}
// hasChildSelector returns a selector that matches elements
// with a child that matches a.
func hasChildSelector(a Selector) Selector {
return func(n *html.Node) bool {
if n.Type != html.ElementNode {
return false
}
return hasChildMatch(n, a)
}
}
// hasDescendantSelector returns a selector that matches elements
// with any descendant that matches a.
func hasDescendantSelector(a Selector) Selector {
return func(n *html.Node) bool {
if n.Type != html.ElementNode {
return false
}
return hasDescendantMatch(n, a)
}
}
// nthChildSelector returns a selector that implements :nth-child(an+b).
// If last is true, implements :nth-last-child instead.
// If ofType is true, implements :nth-of-type instead.
func nthChildSelector(a, b int, last, ofType bool) Selector {
return func(n *html.Node) bool {
if n.Type != html.ElementNode {
return false
}
parent := n.Parent
if parent == nil {
return false
}
i := -1
count := 0
for c := parent.FirstChild; c != nil; c = c.NextSibling {
if (c.Type != html.ElementNode) || (ofType && c.Data != n.Data) {
continue
}
count++
if c == n {
i = count
if !last {
break
}
}
}
if i == -1 {
// This shouldn't happen, since n should always be one of its parent's children.
return false
}
if last {
i = count - i + 1
}
i -= b
if a == 0 {
return i == 0
}
return i%a == 0 && i/a >= 0
}
}
// onlyChildSelector returns a selector that implements :only-child.
// If ofType is true, it implements :only-of-type instead.
func onlyChildSelector(ofType bool) Selector {
return func(n *html.Node) bool {
if n.Type != html.ElementNode {
return false
}
parent := n.Parent
if parent == nil {
return false
}
count := 0
for c := parent.FirstChild; c != nil; c = c.NextSibling {
if (c.Type != html.ElementNode) || (ofType && c.Data != n.Data) {
continue
}
count++
if count > 1 {
return false
}
}
return count == 1
}
}
// emptyElementSelector is a Selector that matches empty elements.
func emptyElementSelector(n *html.Node) bool {
if n.Type != html.ElementNode {
return false
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
switch c.Type {
case html.ElementNode, html.TextNode:
return false
}
}
return true
}
// descendantSelector returns a Selector that matches an element if
// it matches d and has an ancestor that matches a.
func descendantSelector(a, d Selector) Selector {
return func(n *html.Node) bool {
if !d(n) {
return false
}
for p := n.Parent; p != nil; p = p.Parent {
if a(p) {
return true
}
}
return false
}
}
// childSelector returns a Selector that matches an element if
// it matches d and its parent matches a.
func childSelector(a, d Selector) Selector {
return func(n *html.Node) bool {
return d(n) && n.Parent != nil && a(n.Parent)
}
}
// siblingSelector returns a Selector that matches an element
// if it matches s2 and in is preceded by an element that matches s1.
// If adjacent is true, the sibling must be immediately before the element.
func siblingSelector(s1, s2 Selector, adjacent bool) Selector {
return func(n *html.Node) bool {
if !s2(n) {
return false
}
if adjacent {
for n = n.PrevSibling; n != nil; n = n.PrevSibling {
if n.Type == html.TextNode || n.Type == html.CommentNode {
continue
}
return s1(n)
}
return false
}
// Walk backwards looking for element that matches s1
for c := n.PrevSibling; c != nil; c = c.PrevSibling {
if s1(c) {
return true
}
}
return false
}
}
vendor/code.google.com/p/go.net/LICENSE
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/code.google.com/p/go.net/PATENTS
Generated Vendored
+22
View File
@@ -0,0 +1,22 @@
Additional IP Rights Grant (Patents)
"This implementation" means the copyrightable works distributed by
Google as part of the Go project.
Google hereby grants to You a perpetual, worldwide, non-exclusive,
no-charge, royalty-free, irrevocable (except as stated in this section)
patent license to make, have made, use, offer to sell, sell, import,
transfer and otherwise run, modify and propagate the contents of this
implementation of Go, where such license applies only to those patent
claims, both currently owned or controlled by Google and acquired in
the future, licensable by Google that are necessarily infringed by this
implementation of Go. This grant does not include claims that would be
infringed only as a consequence of further modification of this
implementation. If you or your agent or exclusive licensee institute or
order or agree to the institution of patent litigation against any
entity (including a cross-claim or counterclaim in a lawsuit) alleging
that this implementation of Go or any code incorporated within this
implementation of Go constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any patent
rights granted to you under this License for this implementation of Go
shall terminate as of the date such litigation is filed.
vendor/code.google.com/p/go.net/html/atom/atom.go
Generated Vendored
+78
View File
@@ -0,0 +1,78 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package atom provides integer codes (also known as atoms) for a fixed set of
// frequently occurring HTML strings: tag names and attribute keys such as "p"
// and "id".
//
// Sharing an atom's name between all elements with the same tag can result in
// fewer string allocations when tokenizing and parsing HTML. Integer
// comparisons are also generally faster than string comparisons.
//
// The value of an atom's particular code is not guaranteed to stay the same
// between versions of this package. Neither is any ordering guaranteed:
// whether atom.H1 < atom.H2 may also change. The codes are not guaranteed to
// be dense. The only guarantees are that e.g. looking up "div" will yield
// atom.Div, calling atom.Div.String will return "div", and atom.Div != 0.
package atom
// Atom is an integer code for a string. The zero value maps to "".
type Atom uint32
// String returns the atom's name.
func (a Atom) String() string {
start := uint32(a >> 8)
n := uint32(a & 0xff)
if start+n > uint32(len(atomText)) {
return ""
}
return atomText[start : start+n]
}
func (a Atom) string() string {
return atomText[a>>8 : a>>8+a&0xff]
}
// fnv computes the FNV hash with an arbitrary starting value h.
func fnv(h uint32, s []byte) uint32 {
for i := range s {
h ^= uint32(s[i])
h *= 16777619
}
return h
}
func match(s string, t []byte) bool {
for i, c := range t {
if s[i] != c {
return false
}
}
return true
}
// Lookup returns the atom whose name is s. It returns zero if there is no
// such atom. The lookup is case sensitive.
func Lookup(s []byte) Atom {
if len(s) == 0 || len(s) > maxAtomLen {
return 0
}
h := fnv(hash0, s)
if a := table[h&uint32(len(table)-1)]; int(a&0xff) == len(s) && match(a.string(), s) {
return a
}
if a := table[(h>>16)&uint32(len(table)-1)]; int(a&0xff) == len(s) && match(a.string(), s) {
return a
}
return 0
}
// String returns a string whose contents are equal to s. In that sense, it is
// equivalent to string(s) but may be more efficient.
func String(s []byte) string {
if a := Lookup(s); a != 0 {
return a.String()
}
return string(s)
}
vendor/code.google.com/p/go.net/html/atom/gen.go
Generated Vendored
+636
View File
@@ -0,0 +1,636 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
package main
// This program generates table.go and table_test.go.
// Invoke as
//
// go run gen.go |gofmt >table.go
// go run gen.go -test |gofmt >table_test.go
import (
"flag"
"fmt"
"math/rand"
"os"
"sort"
"strings"
)
// identifier converts s to a Go exported identifier.
// It converts "div" to "Div" and "accept-charset" to "AcceptCharset".
func identifier(s string) string {
b := make([]byte, 0, len(s))
cap := true
for _, c := range s {
if c == '-' {
cap = true
continue
}
if cap && 'a' <= c && c <= 'z' {
c -= 'a' - 'A'
}
cap = false
b = append(b, byte(c))
}
return string(b)
}
var test = flag.Bool("test", false, "generate table_test.go")
func main() {
flag.Parse()
var all []string
all = append(all, elements...)
all = append(all, attributes...)
all = append(all, eventHandlers...)
all = append(all, extra...)
sort.Strings(all)
if *test {
fmt.Printf("// generated by go run gen.go -test; DO NOT EDIT\n\n")
fmt.Printf("package atom\n\n")
fmt.Printf("var testAtomList = []string{\n")
for _, s := range all {
fmt.Printf("\t%q,\n", s)
}
fmt.Printf("}\n")
return
}
// uniq - lists have dups
// compute max len too
maxLen := 0
w := 0
for _, s := range all {
if w == 0 || all[w-1] != s {
if maxLen < len(s) {
maxLen = len(s)
}
all[w] = s
w++
}
}
all = all[:w]
// Find hash that minimizes table size.
var best *table
for i := 0; i < 1000000; i++ {
if best != nil && 1<<(best.k-1) < len(all) {
break
}
h := rand.Uint32()
for k := uint(0); k <= 16; k++ {
if best != nil && k >= best.k {
break
}
var t table
if t.init(h, k, all) {
best = &t
break
}
}
}
if best == nil {
fmt.Fprintf(os.Stderr, "failed to construct string table\n")
os.Exit(1)
}
// Lay out strings, using overlaps when possible.
layout := append([]string{}, all...)
// Remove strings that are substrings of other strings
for changed := true; changed; {
changed = false
for i, s := range layout {
if s == "" {
continue
}
for j, t := range layout {
if i != j && t != "" && strings.Contains(s, t) {
changed = true
layout[j] = ""
}
}
}
}
// Join strings where one suffix matches another prefix.
for {
// Find best i, j, k such that layout[i][len-k:] == layout[j][:k],
// maximizing overlap length k.
besti := -1
bestj := -1
bestk := 0
for i, s := range layout {
if s == "" {
continue
}
for j, t := range layout {
if i == j {
continue
}
for k := bestk + 1; k <= len(s) && k <= len(t); k++ {
if s[len(s)-k:] == t[:k] {
besti = i
bestj = j
bestk = k
}
}
}
}
if bestk > 0 {
layout[besti] += layout[bestj][bestk:]
layout[bestj] = ""
continue
}
break
}
text := strings.Join(layout, "")
atom := map[string]uint32{}
for _, s := range all {
off := strings.Index(text, s)
if off < 0 {
panic("lost string " + s)
}
atom[s] = uint32(off<<8 | len(s))
}
// Generate the Go code.
fmt.Printf("// generated by go run gen.go; DO NOT EDIT\n\n")
fmt.Printf("package atom\n\nconst (\n")
for _, s := range all {
fmt.Printf("\t%s Atom = %#x\n", identifier(s), atom[s])
}
fmt.Printf(")\n\n")
fmt.Printf("const hash0 = %#x\n\n", best.h0)
fmt.Printf("const maxAtomLen = %d\n\n", maxLen)
fmt.Printf("var table = [1<<%d]Atom{\n", best.k)
for i, s := range best.tab {
if s == "" {
continue
}
fmt.Printf("\t%#x: %#x, // %s\n", i, atom[s], s)
}
fmt.Printf("}\n")
datasize := (1 << best.k) * 4
fmt.Printf("const atomText =\n")
textsize := len(text)
for len(text) > 60 {
fmt.Printf("\t%q +\n", text[:60])
text = text[60:]
}
fmt.Printf("\t%q\n\n", text)
fmt.Fprintf(os.Stderr, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize)
}
type byLen []string
func (x byLen) Less(i, j int) bool { return len(x[i]) > len(x[j]) }
func (x byLen) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x byLen) Len() int { return len(x) }
// fnv computes the FNV hash with an arbitrary starting value h.
func fnv(h uint32, s string) uint32 {
for i := 0; i < len(s); i++ {
h ^= uint32(s[i])
h *= 16777619
}
return h
}
// A table represents an attempt at constructing the lookup table.
// The lookup table uses cuckoo hashing, meaning that each string
// can be found in one of two positions.
type table struct {
h0 uint32
k uint
mask uint32
tab []string
}
// hash returns the two hashes for s.
func (t *table) hash(s string) (h1, h2 uint32) {
h := fnv(t.h0, s)
h1 = h & t.mask
h2 = (h >> 16) & t.mask
return
}
// init initializes the table with the given parameters.
// h0 is the initial hash value,
// k is the number of bits of hash value to use, and
// x is the list of strings to store in the table.
// init returns false if the table cannot be constructed.
func (t *table) init(h0 uint32, k uint, x []string) bool {
t.h0 = h0
t.k = k
t.tab = make([]string, 1<<k)
t.mask = 1<<k - 1
for _, s := range x {
if !t.insert(s) {
return false
}
}
return true
}
// insert inserts s in the table.
func (t *table) insert(s string) bool {
h1, h2 := t.hash(s)
if t.tab[h1] == "" {
t.tab[h1] = s
return true
}
if t.tab[h2] == "" {
t.tab[h2] = s
return true
}
if t.push(h1, 0) {
t.tab[h1] = s
return true
}
if t.push(h2, 0) {
t.tab[h2] = s
return true
}
return false
}
// push attempts to push aside the entry in slot i.
func (t *table) push(i uint32, depth int) bool {
if depth > len(t.tab) {
return false
}
s := t.tab[i]
h1, h2 := t.hash(s)
j := h1 + h2 - i
if t.tab[j] != "" && !t.push(j, depth+1) {
return false
}
t.tab[j] = s
return true
}
// The lists of element names and attribute keys were taken from
// http://www.whatwg.org/specs/web-apps/current-work/multipage/section-index.html
// as of the "HTML Living Standard - Last Updated 30 May 2012" version.
var elements = []string{
"a",
"abbr",
"address",
"area",
"article",
"aside",
"audio",
"b",
"base",
"bdi",
"bdo",
"blockquote",
"body",
"br",
"button",
"canvas",
"caption",
"cite",
"code",
"col",
"colgroup",
"command",
"data",
"datalist",
"dd",
"del",
"details",
"dfn",
"dialog",
"div",
"dl",
"dt",
"em",
"embed",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"head",
"header",
"hgroup",
"hr",
"html",
"i",
"iframe",
"img",
"input",
"ins",
"kbd",
"keygen",
"label",
"legend",
"li",
"link",
"map",
"mark",
"menu",
"meta",
"meter",
"nav",
"noscript",
"object",
"ol",
"optgroup",
"option",
"output",
"p",
"param",
"pre",
"progress",
"q",
"rp",
"rt",
"ruby",
"s",
"samp",
"script",
"section",
"select",
"small",
"source",
"span",
"strong",
"style",
"sub",
"summary",
"sup",
"table",
"tbody",
"td",
"textarea",
"tfoot",
"th",
"thead",
"time",
"title",
"tr",
"track",
"u",
"ul",
"var",
"video",
"wbr",
}
var attributes = []string{
"accept",
"accept-charset",
"accesskey",
"action",
"alt",
"async",
"autocomplete",
"autofocus",
"autoplay",
"border",
"challenge",
"charset",
"checked",
"cite",
"class",
"cols",
"colspan",
"command",
"content",
"contenteditable",
"contextmenu",
"controls",
"coords",
"crossorigin",
"data",
"datetime",
"default",
"defer",
"dir",
"dirname",
"disabled",
"download",
"draggable",
"dropzone",
"enctype",
"for",
"form",
"formaction",
"formenctype",
"formmethod",
"formnovalidate",
"formtarget",
"headers",
"height",
"hidden",
"high",
"href",
"hreflang",
"http-equiv",
"icon",
"id",
"inert",
"ismap",
"itemid",
"itemprop",
"itemref",
"itemscope",
"itemtype",
"keytype",
"kind",
"label",
"lang",
"list",
"loop",
"low",
"manifest",
"max",
"maxlength",
"media",
"mediagroup",
"method",
"min",
"multiple",
"muted",
"name",
"novalidate",
"open",
"optimum",
"pattern",
"ping",
"placeholder",
"poster",
"preload",
"radiogroup",
"readonly",
"rel",
"required",
"reversed",
"rows",
"rowspan",
"sandbox",
"spellcheck",
"scope",
"scoped",
"seamless",
"selected",
"shape",
"size",
"sizes",
"span",
"src",
"srcdoc",
"srclang",
"start",
"step",
"style",
"tabindex",
"target",
"title",
"translate",
"type",
"typemustmatch",
"usemap",
"value",
"width",
"wrap",
}
var eventHandlers = []string{
"onabort",
"onafterprint",
"onbeforeprint",
"onbeforeunload",
"onblur",
"oncancel",
"oncanplay",
"oncanplaythrough",
"onchange",
"onclick",
"onclose",
"oncontextmenu",
"oncuechange",
"ondblclick",
"ondrag",
"ondragend",
"ondragenter",
"ondragleave",
"ondragover",
"ondragstart",
"ondrop",
"ondurationchange",
"onemptied",
"onended",
"onerror",
"onfocus",
"onhashchange",
"oninput",
"oninvalid",
"onkeydown",
"onkeypress",
"onkeyup",
"onload",
"onloadeddata",
"onloadedmetadata",
"onloadstart",
"onmessage",
"onmousedown",
"onmousemove",
"onmouseout",
"onmouseover",
"onmouseup",
"onmousewheel",
"onoffline",
"ononline",
"onpagehide",
"onpageshow",
"onpause",
"onplay",
"onplaying",
"onpopstate",
"onprogress",
"onratechange",
"onreset",
"onresize",
"onscroll",
"onseeked",
"onseeking",
"onselect",
"onshow",
"onstalled",
"onstorage",
"onsubmit",
"onsuspend",
"ontimeupdate",
"onunload",
"onvolumechange",
"onwaiting",
}
// extra are ad-hoc values not covered by any of the lists above.
var extra = []string{
"align",
"annotation",
"annotation-xml",
"applet",
"basefont",
"bgsound",
"big",
"blink",
"center",
"color",
"desc",
"face",
"font",
"foreignObject", // HTML is case-insensitive, but SVG-embedded-in-HTML is case-sensitive.
"foreignobject",
"frame",
"frameset",
"image",
"isindex",
"listing",
"malignmark",
"marquee",
"math",
"mglyph",
"mi",
"mn",
"mo",
"ms",
"mtext",
"nobr",
"noembed",
"noframes",
"plaintext",
"prompt",
"public",
"spacer",
"strike",
"svg",
"system",
"tt",
"xmp",
}
vendor/code.google.com/p/go.net/html/atom/table.go
Generated Vendored
+694
View File
@@ -0,0 +1,694 @@
// generated by go run gen.go; DO NOT EDIT
package atom
const (
A Atom = 0x1
Abbr Atom = 0x4
Accept Atom = 0x2106
AcceptCharset Atom = 0x210e
Accesskey Atom = 0x3309
Action Atom = 0x21b06
Address Atom = 0x5d507
Align Atom = 0x1105
Alt Atom = 0x4503
Annotation Atom = 0x18d0a
AnnotationXml Atom = 0x18d0e
Applet Atom = 0x2d106
Area Atom = 0x31804
Article Atom = 0x39907
Aside Atom = 0x4f05
Async Atom = 0x9305
Audio Atom = 0xaf05
Autocomplete Atom = 0xd50c
Autofocus Atom = 0xe109
Autoplay Atom = 0x10c08
B Atom = 0x101
Base Atom = 0x11404
Basefont Atom = 0x11408
Bdi Atom = 0x1a03
Bdo Atom = 0x12503
Bgsound Atom = 0x13807
Big Atom = 0x14403
Blink Atom = 0x14705
Blockquote Atom = 0x14c0a
Body Atom = 0x2f04
Border Atom = 0x15606
Br Atom = 0x202
Button Atom = 0x15c06
Canvas Atom = 0x4b06
Caption Atom = 0x1e007
Center Atom = 0x2df06
Challenge Atom = 0x23e09
Charset Atom = 0x2807
Checked Atom = 0x33f07
Cite Atom = 0x9704
Class Atom = 0x3d905
Code Atom = 0x16f04
Col Atom = 0x17603
Colgroup Atom = 0x17608
Color Atom = 0x18305
Cols Atom = 0x18804
Colspan Atom = 0x18807
Command Atom = 0x19b07
Content Atom = 0x42c07
Contenteditable Atom = 0x42c0f
Contextmenu Atom = 0x3480b
Controls Atom = 0x1ae08
Coords Atom = 0x1ba06
Crossorigin Atom = 0x1c40b
Data Atom = 0x44304
Datalist Atom = 0x44308
Datetime Atom = 0x25b08
Dd Atom = 0x28802
Default Atom = 0x5207
Defer Atom = 0x17105
Del Atom = 0x4d603
Desc Atom = 0x4804
Details Atom = 0x6507
Dfn Atom = 0x8303
Dialog Atom = 0x1b06
Dir Atom = 0x9d03
Dirname Atom = 0x9d07
Disabled Atom = 0x10008
Div Atom = 0x10703
Dl Atom = 0x13e02
Download Atom = 0x40908
Draggable Atom = 0x1a109
Dropzone Atom = 0x3a208
Dt Atom = 0x4e402
Em Atom = 0x7f02
Embed Atom = 0x7f05
Enctype Atom = 0x23007
Face Atom = 0x2dd04
Fieldset Atom = 0x1d508
Figcaption Atom = 0x1dd0a
Figure Atom = 0x1f106
Font Atom = 0x11804
Footer Atom = 0x5906
For Atom = 0x1fd03
ForeignObject Atom = 0x1fd0d
Foreignobject Atom = 0x20a0d
Form Atom = 0x21704
Formaction Atom = 0x2170a
Formenctype Atom = 0x22c0b
Formmethod Atom = 0x2470a
Formnovalidate Atom = 0x2510e
Formtarget Atom = 0x2660a
Frame Atom = 0x8705
Frameset Atom = 0x8708
H1 Atom = 0x13602
H2 Atom = 0x29602
H3 Atom = 0x2c502
H4 Atom = 0x30e02
H5 Atom = 0x4e602
H6 Atom = 0x27002
Head Atom = 0x2fa04
Header Atom = 0x2fa06
Headers Atom = 0x2fa07
Height Atom = 0x27206
Hgroup Atom = 0x27a06
Hidden Atom = 0x28606
High Atom = 0x29304
Hr Atom = 0x13102
Href Atom = 0x29804
Hreflang Atom = 0x29808
Html Atom = 0x27604
HttpEquiv Atom = 0x2a00a
I Atom = 0x601
Icon Atom = 0x42b04
Id Atom = 0x5102
Iframe Atom = 0x2b406
Image Atom = 0x2ba05
Img Atom = 0x2bf03
Inert Atom = 0x4c105
Input Atom = 0x3f605
Ins Atom = 0x1cd03
Isindex Atom = 0x2c707
Ismap Atom = 0x2ce05
Itemid Atom = 0x9806
Itemprop Atom = 0x57e08
Itemref Atom = 0x2d707
Itemscope Atom = 0x2e509
Itemtype Atom = 0x2ef08
Kbd Atom = 0x1903
Keygen Atom = 0x3906
Keytype Atom = 0x51207
Kind Atom = 0xfd04
Label Atom = 0xba05
Lang Atom = 0x29c04
Legend Atom = 0x1a806
Li Atom = 0x1202
Link Atom = 0x14804
List Atom = 0x44704
Listing Atom = 0x44707
Loop Atom = 0xbe04
Low Atom = 0x13f03
Malignmark Atom = 0x100a
Manifest Atom = 0x5b608
Map Atom = 0x2d003
Mark Atom = 0x1604
Marquee Atom = 0x5f207
Math Atom = 0x2f704
Max Atom = 0x30603
Maxlength Atom = 0x30609
Media Atom = 0xa205
Mediagroup Atom = 0xa20a
Menu Atom = 0x34f04
Meta Atom = 0x45604
Meter Atom = 0x26105
Method Atom = 0x24b06
Mglyph Atom = 0x2c006
Mi Atom = 0x9b02
Min Atom = 0x31003
Mn Atom = 0x25402
Mo Atom = 0x47a02
Ms Atom = 0x2e802
Mtext Atom = 0x31305
Multiple Atom = 0x32108
Muted Atom = 0x32905
Name Atom = 0xa004
Nav Atom = 0x3e03
Nobr Atom = 0x7404
Noembed Atom = 0x7d07
Noframes Atom = 0x8508
Noscript Atom = 0x28b08
Novalidate Atom = 0x2550a
Object Atom = 0x21106
Ol Atom = 0xcd02
Onabort Atom = 0x16007
Onafterprint Atom = 0x1e50c
Onbeforeprint Atom = 0x21f0d
Onbeforeunload Atom = 0x5c90e
Onblur Atom = 0x3e206
Oncancel Atom = 0xb308
Oncanplay Atom = 0x12709
Oncanplaythrough Atom = 0x12710
Onchange Atom = 0x3b808
Onclick Atom = 0x2ad07
Onclose Atom = 0x32e07
Oncontextmenu Atom = 0x3460d
Oncuechange Atom = 0x3530b
Ondblclick Atom = 0x35e0a
Ondrag Atom = 0x36806
Ondragend Atom = 0x36809
Ondragenter Atom = 0x3710b
Ondragleave Atom = 0x37c0b
Ondragover Atom = 0x3870a
Ondragstart Atom = 0x3910b
Ondrop Atom = 0x3a006
Ondurationchange Atom = 0x3b010
Onemptied Atom = 0x3a709
Onended Atom = 0x3c007
Onerror Atom = 0x3c707
Onfocus Atom = 0x3ce07
Onhashchange Atom = 0x3e80c
Oninput Atom = 0x3f407
Oninvalid Atom = 0x3fb09
Onkeydown Atom = 0x40409
Onkeypress Atom = 0x4110a
Onkeyup Atom = 0x42107
Onload Atom = 0x43b06
Onloadeddata Atom = 0x43b0c
Onloadedmetadata Atom = 0x44e10
Onloadstart Atom = 0x4640b
Onmessage Atom = 0x46f09
Onmousedown Atom = 0x4780b
Onmousemove Atom = 0x4830b
Onmouseout Atom = 0x48e0a
Onmouseover Atom = 0x49b0b
Onmouseup Atom = 0x4a609
Onmousewheel Atom = 0x4af0c
Onoffline Atom = 0x4bb09
Ononline Atom = 0x4c608
Onpagehide Atom = 0x4ce0a
Onpageshow Atom = 0x4d90a
Onpause Atom = 0x4e807
Onplay Atom = 0x4f206
Onplaying Atom = 0x4f209
Onpopstate Atom = 0x4fb0a
Onprogress Atom = 0x5050a
Onratechange Atom = 0x5190c
Onreset Atom = 0x52507
Onresize Atom = 0x52c08
Onscroll Atom = 0x53a08
Onseeked Atom = 0x54208
Onseeking Atom = 0x54a09
Onselect Atom = 0x55308
Onshow Atom = 0x55d06
Onstalled Atom = 0x56609
Onstorage Atom = 0x56f09
Onsubmit Atom = 0x57808
Onsuspend Atom = 0x58809
Ontimeupdate Atom = 0x1190c
Onunload Atom = 0x59108
Onvolumechange Atom = 0x5990e
Onwaiting Atom = 0x5a709
Open Atom = 0x58404
Optgroup Atom = 0xc008
Optimum Atom = 0x5b007
Option Atom = 0x5c506
Output Atom = 0x49506
P Atom = 0xc01
Param Atom = 0xc05
Pattern Atom = 0x6e07
Ping Atom = 0xab04
Placeholder Atom = 0xc70b
Plaintext Atom = 0xf109
Poster Atom = 0x17d06
Pre Atom = 0x27f03
Preload Atom = 0x27f07
Progress Atom = 0x50708
Prompt Atom = 0x5bf06
Public Atom = 0x42706
Q Atom = 0x15101
Radiogroup Atom = 0x30a
Readonly Atom = 0x31908
Rel Atom = 0x28003
Required Atom = 0x1f508
Reversed Atom = 0x5e08
Rows Atom = 0x7704
Rowspan Atom = 0x7707
Rp Atom = 0x1eb02
Rt Atom = 0x16502
Ruby Atom = 0xd104
S Atom = 0x2c01
Samp Atom = 0x6b04
Sandbox Atom = 0xe907
Scope Atom = 0x2e905
Scoped Atom = 0x2e906
Script Atom = 0x28d06
Seamless Atom = 0x33308
Section Atom = 0x3dd07
Select Atom = 0x55506
Selected Atom = 0x55508
Shape Atom = 0x1b505
Size Atom = 0x53004
Sizes Atom = 0x53005
Small Atom = 0x1bf05
Source Atom = 0x1cf06
Spacer Atom = 0x30006
Span Atom = 0x7a04
Spellcheck Atom = 0x33a0a
Src Atom = 0x3d403
Srcdoc Atom = 0x3d406
Srclang Atom = 0x41a07
Start Atom = 0x39705
Step Atom = 0x5bc04
Strike Atom = 0x50e06
Strong Atom = 0x53406
Style Atom = 0x5db05
Sub Atom = 0x57a03
Summary Atom = 0x5e007
Sup Atom = 0x5e703
Svg Atom = 0x5ea03
System Atom = 0x5ed06
Tabindex Atom = 0x45c08
Table Atom = 0x43605
Target Atom = 0x26a06
Tbody Atom = 0x2e05
Td Atom = 0x4702
Textarea Atom = 0x31408
Tfoot Atom = 0x5805
Th Atom = 0x13002
Thead Atom = 0x2f905
Time Atom = 0x11b04
Title Atom = 0x8e05
Tr Atom = 0xf902
Track Atom = 0xf905
Translate Atom = 0x16609
Tt Atom = 0x7002
Type Atom = 0x23304
Typemustmatch Atom = 0x2330d
U Atom = 0xb01
Ul Atom = 0x5602
Usemap Atom = 0x4ec06
Value Atom = 0x4005
Var Atom = 0x10903
Video Atom = 0x2a905
Wbr Atom = 0x14103
Width Atom = 0x4e205
Wrap Atom = 0x56204
Xmp Atom = 0xef03
)
const hash0 = 0xc17da63e
const maxAtomLen = 16
var table = [1 << 9]Atom{
0x1: 0x4830b, // onmousemove
0x2: 0x5a709, // onwaiting
0x4: 0x5bf06, // prompt
0x7: 0x5b007, // optimum
0x8: 0x1604, // mark
0xa: 0x2d707, // itemref
0xb: 0x4d90a, // onpageshow
0xc: 0x55506, // select
0xd: 0x1a109, // draggable
0xe: 0x3e03, // nav
0xf: 0x19b07, // command
0x11: 0xb01, // u
0x14: 0x2fa07, // headers
0x15: 0x44308, // datalist
0x17: 0x6b04, // samp
0x1a: 0x40409, // onkeydown
0x1b: 0x53a08, // onscroll
0x1c: 0x17603, // col
0x20: 0x57e08, // itemprop
0x21: 0x2a00a, // http-equiv
0x22: 0x5e703, // sup
0x24: 0x1f508, // required
0x2b: 0x27f07, // preload
0x2c: 0x21f0d, // onbeforeprint
0x2d: 0x3710b, // ondragenter
0x2e: 0x4e402, // dt
0x2f: 0x57808, // onsubmit
0x30: 0x13102, // hr
0x31: 0x3460d, // oncontextmenu
0x33: 0x2ba05, // image
0x34: 0x4e807, // onpause
0x35: 0x27a06, // hgroup
0x36: 0xab04, // ping
0x37: 0x55308, // onselect
0x3a: 0x10703, // div
0x40: 0x9b02, // mi
0x41: 0x33308, // seamless
0x42: 0x2807, // charset
0x43: 0x5102, // id
0x44: 0x4fb0a, // onpopstate
0x45: 0x4d603, // del
0x46: 0x5f207, // marquee
0x47: 0x3309, // accesskey
0x49: 0x5906, // footer
0x4a: 0x2d106, // applet
0x4b: 0x2ce05, // ismap
0x51: 0x34f04, // menu
0x52: 0x2f04, // body
0x55: 0x8708, // frameset
0x56: 0x52507, // onreset
0x57: 0x14705, // blink
0x58: 0x8e05, // title
0x59: 0x39907, // article
0x5b: 0x13002, // th
0x5d: 0x15101, // q
0x5e: 0x58404, // open
0x5f: 0x31804, // area
0x61: 0x43b06, // onload
0x62: 0x3f605, // input
0x63: 0x11404, // base
0x64: 0x18807, // colspan
0x65: 0x51207, // keytype
0x66: 0x13e02, // dl
0x68: 0x1d508, // fieldset
0x6a: 0x31003, // min
0x6b: 0x10903, // var
0x6f: 0x2fa06, // header
0x70: 0x16502, // rt
0x71: 0x17608, // colgroup
0x72: 0x25402, // mn
0x74: 0x16007, // onabort
0x75: 0x3906, // keygen
0x76: 0x4bb09, // onoffline
0x77: 0x23e09, // challenge
0x78: 0x2d003, // map
0x7a: 0x30e02, // h4
0x7b: 0x3c707, // onerror
0x7c: 0x30609, // maxlength
0x7d: 0x31305, // mtext
0x7e: 0x5805, // tfoot
0x7f: 0x11804, // font
0x80: 0x100a, // malignmark
0x81: 0x45604, // meta
0x82: 0x9305, // async
0x83: 0x2c502, // h3
0x84: 0x28802, // dd
0x85: 0x29804, // href
0x86: 0xa20a, // mediagroup
0x87: 0x1ba06, // coords
0x88: 0x41a07, // srclang
0x89: 0x35e0a, // ondblclick
0x8a: 0x4005, // value
0x8c: 0xb308, // oncancel
0x8e: 0x33a0a, // spellcheck
0x8f: 0x8705, // frame
0x91: 0x14403, // big
0x94: 0x21b06, // action
0x95: 0x9d03, // dir
0x97: 0x31908, // readonly
0x99: 0x43605, // table
0x9a: 0x5e007, // summary
0x9b: 0x14103, // wbr
0x9c: 0x30a, // radiogroup
0x9d: 0xa004, // name
0x9f: 0x5ed06, // system
0xa1: 0x18305, // color
0xa2: 0x4b06, // canvas
0xa3: 0x27604, // html
0xa5: 0x54a09, // onseeking
0xac: 0x1b505, // shape
0xad: 0x28003, // rel
0xae: 0x12710, // oncanplaythrough
0xaf: 0x3870a, // ondragover
0xb1: 0x1fd0d, // foreignObject
0xb3: 0x7704, // rows
0xb6: 0x44707, // listing
0xb7: 0x49506, // output
0xb9: 0x3480b, // contextmenu
0xbb: 0x13f03, // low
0xbc: 0x1eb02, // rp
0xbd: 0x58809, // onsuspend
0xbe: 0x15c06, // button
0xbf: 0x4804, // desc
0xc1: 0x3dd07, // section
0xc2: 0x5050a, // onprogress
0xc3: 0x56f09, // onstorage
0xc4: 0x2f704, // math
0xc5: 0x4f206, // onplay
0xc7: 0x5602, // ul
0xc8: 0x6e07, // pattern
0xc9: 0x4af0c, // onmousewheel
0xca: 0x36809, // ondragend
0xcb: 0xd104, // ruby
0xcc: 0xc01, // p
0xcd: 0x32e07, // onclose
0xce: 0x26105, // meter
0xcf: 0x13807, // bgsound
0xd2: 0x27206, // height
0xd4: 0x101, // b
0xd5: 0x2ef08, // itemtype
0xd8: 0x1e007, // caption
0xd9: 0x10008, // disabled
0xdc: 0x5ea03, // svg
0xdd: 0x1bf05, // small
0xde: 0x44304, // data
0xe0: 0x4c608, // ononline
0xe1: 0x2c006, // mglyph
0xe3: 0x7f05, // embed
0xe4: 0xf902, // tr
0xe5: 0x4640b, // onloadstart
0xe7: 0x3b010, // ondurationchange
0xed: 0x12503, // bdo
0xee: 0x4702, // td
0xef: 0x4f05, // aside
0xf0: 0x29602, // h2
0xf1: 0x50708, // progress
0xf2: 0x14c0a, // blockquote
0xf4: 0xba05, // label
0xf5: 0x601, // i
0xf7: 0x7707, // rowspan
0xfb: 0x4f209, // onplaying
0xfd: 0x2bf03, // img
0xfe: 0xc008, // optgroup
0xff: 0x42c07, // content
0x101: 0x5190c, // onratechange
0x103: 0x3e80c, // onhashchange
0x104: 0x6507, // details
0x106: 0x40908, // download
0x109: 0xe907, // sandbox
0x10b: 0x42c0f, // contenteditable
0x10d: 0x37c0b, // ondragleave
0x10e: 0x2106, // accept
0x10f: 0x55508, // selected
0x112: 0x2170a, // formaction
0x113: 0x2df06, // center
0x115: 0x44e10, // onloadedmetadata
0x116: 0x14804, // link
0x117: 0x11b04, // time
0x118: 0x1c40b, // crossorigin
0x119: 0x3ce07, // onfocus
0x11a: 0x56204, // wrap
0x11b: 0x42b04, // icon
0x11d: 0x2a905, // video
0x11e: 0x3d905, // class
0x121: 0x5990e, // onvolumechange
0x122: 0x3e206, // onblur
0x123: 0x2e509, // itemscope
0x124: 0x5db05, // style
0x127: 0x42706, // public
0x129: 0x2510e, // formnovalidate
0x12a: 0x55d06, // onshow
0x12c: 0x16609, // translate
0x12d: 0x9704, // cite
0x12e: 0x2e802, // ms
0x12f: 0x1190c, // ontimeupdate
0x130: 0xfd04, // kind
0x131: 0x2660a, // formtarget
0x135: 0x3c007, // onended
0x136: 0x28606, // hidden
0x137: 0x2c01, // s
0x139: 0x2470a, // formmethod
0x13a: 0x44704, // list
0x13c: 0x27002, // h6
0x13d: 0xcd02, // ol
0x13e: 0x3530b, // oncuechange
0x13f: 0x20a0d, // foreignobject
0x143: 0x5c90e, // onbeforeunload
0x145: 0x3a709, // onemptied
0x146: 0x17105, // defer
0x147: 0xef03, // xmp
0x148: 0xaf05, // audio
0x149: 0x1903, // kbd
0x14c: 0x46f09, // onmessage
0x14d: 0x5c506, // option
0x14e: 0x4503, // alt
0x14f: 0x33f07, // checked
0x150: 0x10c08, // autoplay
0x152: 0x202, // br
0x153: 0x2550a, // novalidate
0x156: 0x7d07, // noembed
0x159: 0x2ad07, // onclick
0x15a: 0x4780b, // onmousedown
0x15b: 0x3b808, // onchange
0x15e: 0x3fb09, // oninvalid
0x15f: 0x2e906, // scoped
0x160: 0x1ae08, // controls
0x161: 0x32905, // muted
0x163: 0x4ec06, // usemap
0x164: 0x1dd0a, // figcaption
0x165: 0x36806, // ondrag
0x166: 0x29304, // high
0x168: 0x3d403, // src
0x169: 0x17d06, // poster
0x16b: 0x18d0e, // annotation-xml
0x16c: 0x5bc04, // step
0x16d: 0x4, // abbr
0x16e: 0x1b06, // dialog
0x170: 0x1202, // li
0x172: 0x47a02, // mo
0x175: 0x1fd03, // for
0x176: 0x1cd03, // ins
0x178: 0x53004, // size
0x17a: 0x5207, // default
0x17b: 0x1a03, // bdi
0x17c: 0x4ce0a, // onpagehide
0x17d: 0x9d07, // dirname
0x17e: 0x23304, // type
0x17f: 0x21704, // form
0x180: 0x4c105, // inert
0x181: 0x12709, // oncanplay
0x182: 0x8303, // dfn
0x183: 0x45c08, // tabindex
0x186: 0x7f02, // em
0x187: 0x29c04, // lang
0x189: 0x3a208, // dropzone
0x18a: 0x4110a, // onkeypress
0x18b: 0x25b08, // datetime
0x18c: 0x18804, // cols
0x18d: 0x1, // a
0x18e: 0x43b0c, // onloadeddata
0x191: 0x15606, // border
0x192: 0x2e05, // tbody
0x193: 0x24b06, // method
0x195: 0xbe04, // loop
0x196: 0x2b406, // iframe
0x198: 0x2fa04, // head
0x19e: 0x5b608, // manifest
0x19f: 0xe109, // autofocus
0x1a0: 0x16f04, // code
0x1a1: 0x53406, // strong
0x1a2: 0x32108, // multiple
0x1a3: 0xc05, // param
0x1a6: 0x23007, // enctype
0x1a7: 0x2dd04, // face
0x1a8: 0xf109, // plaintext
0x1a9: 0x13602, // h1
0x1aa: 0x56609, // onstalled
0x1ad: 0x28d06, // script
0x1ae: 0x30006, // spacer
0x1af: 0x52c08, // onresize
0x1b0: 0x49b0b, // onmouseover
0x1b1: 0x59108, // onunload
0x1b2: 0x54208, // onseeked
0x1b4: 0x2330d, // typemustmatch
0x1b5: 0x1f106, // figure
0x1b6: 0x48e0a, // onmouseout
0x1b7: 0x27f03, // pre
0x1b8: 0x4e205, // width
0x1bb: 0x7404, // nobr
0x1be: 0x7002, // tt
0x1bf: 0x1105, // align
0x1c0: 0x3f407, // oninput
0x1c3: 0x42107, // onkeyup
0x1c6: 0x1e50c, // onafterprint
0x1c7: 0x210e, // accept-charset
0x1c8: 0x9806, // itemid
0x1cb: 0x50e06, // strike
0x1cc: 0x57a03, // sub
0x1cd: 0xf905, // track
0x1ce: 0x39705, // start
0x1d0: 0x11408, // basefont
0x1d6: 0x1cf06, // source
0x1d7: 0x1a806, // legend
0x1d8: 0x2f905, // thead
0x1da: 0x2e905, // scope
0x1dd: 0x21106, // object
0x1de: 0xa205, // media
0x1df: 0x18d0a, // annotation
0x1e0: 0x22c0b, // formenctype
0x1e2: 0x28b08, // noscript
0x1e4: 0x53005, // sizes
0x1e5: 0xd50c, // autocomplete
0x1e6: 0x7a04, // span
0x1e7: 0x8508, // noframes
0x1e8: 0x26a06, // target
0x1e9: 0x3a006, // ondrop
0x1ea: 0x3d406, // srcdoc
0x1ec: 0x5e08, // reversed
0x1f0: 0x2c707, // isindex
0x1f3: 0x29808, // hreflang
0x1f5: 0x4e602, // h5
0x1f6: 0x5d507, // address
0x1fa: 0x30603, // max
0x1fb: 0xc70b, // placeholder
0x1fc: 0x31408, // textarea
0x1fe: 0x4a609, // onmouseup
0x1ff: 0x3910b, // ondragstart
}
const atomText = "abbradiogrouparamalignmarkbdialogaccept-charsetbodyaccesskey" +
"genavaluealtdescanvasidefaultfootereversedetailsampatternobr" +
"owspanoembedfnoframesetitleasyncitemidirnamediagroupingaudio" +
"ncancelabelooptgrouplaceholderubyautocompleteautofocusandbox" +
"mplaintextrackindisabledivarautoplaybasefontimeupdatebdoncan" +
"playthrough1bgsoundlowbrbigblinkblockquoteborderbuttonabortr" +
"anslatecodefercolgroupostercolorcolspannotation-xmlcommandra" +
"ggablegendcontrolshapecoordsmallcrossoriginsourcefieldsetfig" +
"captionafterprintfigurequiredforeignObjectforeignobjectforma" +
"ctionbeforeprintformenctypemustmatchallengeformmethodformnov" +
"alidatetimeterformtargeth6heightmlhgroupreloadhiddenoscripth" +
"igh2hreflanghttp-equivideonclickiframeimageimglyph3isindexis" +
"mappletitemrefacenteritemscopeditemtypematheaderspacermaxlen" +
"gth4minmtextareadonlymultiplemutedoncloseamlesspellcheckedon" +
"contextmenuoncuechangeondblclickondragendondragenterondragle" +
"aveondragoverondragstarticleondropzonemptiedondurationchange" +
"onendedonerroronfocusrcdoclassectionbluronhashchangeoninputo" +
"ninvalidonkeydownloadonkeypressrclangonkeyupublicontentedita" +
"bleonloadeddatalistingonloadedmetadatabindexonloadstartonmes" +
"sageonmousedownonmousemoveonmouseoutputonmouseoveronmouseupo" +
"nmousewheelonofflinertononlineonpagehidelonpageshowidth5onpa" +
"usemaponplayingonpopstateonprogresstrikeytypeonratechangeonr" +
"esetonresizestrongonscrollonseekedonseekingonselectedonshowr" +
"aponstalledonstorageonsubmitempropenonsuspendonunloadonvolum" +
"echangeonwaitingoptimumanifestepromptoptionbeforeunloaddress" +
"tylesummarysupsvgsystemarquee"
vendor/code.google.com/p/go.net/html/const.go
Generated Vendored
+100
View File
@@ -0,0 +1,100 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package html
// Section 12.2.3.2 of the HTML5 specification says "The following elements
// have varying levels of special parsing rules".
// http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#the-stack-of-open-elements
var isSpecialElementMap = map[string]bool{
"address": true,
"applet": true,
"area": true,
"article": true,
"aside": true,
"base": true,
"basefont": true,
"bgsound": true,
"blockquote": true,
"body": true,
"br": true,
"button": true,
"caption": true,
"center": true,
"col": true,
"colgroup": true,
"command": true,
"dd": true,
"details": true,
"dir": true,
"div": true,
"dl": true,
"dt": true,
"embed": true,
"fieldset": true,
"figcaption": true,
"figure": true,
"footer": true,
"form": true,
"frame": true,
"frameset": true,
"h1": true,
"h2": true,
"h3": true,
"h4": true,
"h5": true,
"h6": true,
"head": true,
"header": true,
"hgroup": true,
"hr": true,
"html": true,
"iframe": true,
"img": true,
"input": true,
"isindex": true,
"li": true,
"link": true,
"listing": true,
"marquee": true,
"menu": true,
"meta": true,
"nav": true,
"noembed": true,
"noframes": true,
"noscript": true,
"object": true,
"ol": true,
"p": true,
"param": true,
"plaintext": true,
"pre": true,
"script": true,
"section": true,
"select": true,
"style": true,
"summary": true,
"table": true,
"tbody": true,
"td": true,
"textarea": true,
"tfoot": true,
"th": true,
"thead": true,
"title": true,
"tr": true,
"ul": true,
"wbr": true,
"xmp": true,
}
func isSpecialElement(element *Node) bool {
switch element.Namespace {
case "", "html":
return isSpecialElementMap[element.Data]
case "svg":
return element.Data == "foreignObject"
}
return false
}
vendor/code.google.com/p/go.net/html/doc.go
Generated Vendored
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package html implements an HTML5-compliant tokenizer and parser.
Tokenization is done by creating a Tokenizer for an io.Reader r. It is the
caller's responsibility to ensure that r provides UTF-8 encoded HTML.
z := html.NewTokenizer(r)
Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(),
which parses the next token and returns its type, or an error:
for {
tt := z.Next()
if tt == html.ErrorToken {
// ...
return ...
}
// Process the current token.
}
There are two APIs for retrieving the current token. The high-level API is to
call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs
allow optionally calling Raw after Next but before Token, Text, TagName, or
TagAttr. In EBNF notation, the valid call sequence per token is:
Next {Raw} [ Token | Text | TagName {TagAttr} ]
Token returns an independent data structure that completely describes a token.
Entities (such as "&lt;") are unescaped, tag names and attribute keys are
lower-cased, and attributes are collected into a []Attribute. For example:
for {
if z.Next() == html.ErrorToken {
// Returning io.EOF indicates success.
return z.Err()
}
emitToken(z.Token())
}
The low-level API performs fewer allocations and copies, but the contents of
the []byte values returned by Text, TagName and TagAttr may change on the next
call to Next. For example, to extract an HTML page's anchor text:
depth := 0
for {
tt := z.Next()
switch tt {
case ErrorToken:
return z.Err()
case TextToken:
if depth > 0 {
// emitBytes should copy the []byte it receives,
// if it doesn't process it immediately.
emitBytes(z.Text())
}
case StartTagToken, EndTagToken:
tn, _ := z.TagName()
if len(tn) == 1 && tn[0] == 'a' {
if tt == StartTagToken {
depth++
} else {
depth--
}
}
}
}
Parsing is done by calling Parse with an io.Reader, which returns the root of
the parse tree (the document element) as a *Node. It is the caller's
responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
example, to process each anchor node in depth-first order:
doc, err := html.Parse(r)
if err != nil {
// ...
}
var f func(*html.Node)
f = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "a" {
// Do something with n...
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
f(c)
}
}
f(doc)
The relevant specifications include:
http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html and
http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html
*/
package html
// The tokenization algorithm implemented by this package is not a line-by-line
// transliteration of the relatively verbose state-machine in the WHATWG
// specification. A more direct approach is used instead, where the program
// counter implies the state, such as whether it is tokenizing a tag or a text
// node. Specification compliance is verified by checking expected and actual
// outputs over a test suite rather than aiming for algorithmic fidelity.
// TODO(nigeltao): Does a DOM API belong in this package or a separate one?
// TODO(nigeltao): How does parsing interact with a JavaScript engine?
vendor/code.google.com/p/go.net/html/doctype.go
Generated Vendored
+156
View File
@@ -0,0 +1,156 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package html
import (
"strings"
)
// parseDoctype parses the data from a DoctypeToken into a name,
// public identifier, and system identifier. It returns a Node whose Type
// is DoctypeNode, whose Data is the name, and which has attributes
// named "system" and "public" for the two identifiers if they were present.
// quirks is whether the document should be parsed in "quirks mode".
func parseDoctype(s string) (n *Node, quirks bool) {
n = &Node{Type: DoctypeNode}
// Find the name.
space := strings.IndexAny(s, whitespace)
if space == -1 {
space = len(s)
}
n.Data = s[:space]
// The comparison to "html" is case-sensitive.
if n.Data != "html" {
quirks = true
}
n.Data = strings.ToLower(n.Data)
s = strings.TrimLeft(s[space:], whitespace)
if len(s) < 6 {
// It can't start with "PUBLIC" or "SYSTEM".
// Ignore the rest of the string.
return n, quirks || s != ""
}
key := strings.ToLower(s[:6])
s = s[6:]
for key == "public" || key == "system" {
s = strings.TrimLeft(s, whitespace)
if s == "" {
break
}
quote := s[0]
if quote != '"' && quote != '\'' {
break
}
s = s[1:]
q := strings.IndexRune(s, rune(quote))
var id string
if q == -1 {
id = s
s = ""
} else {
id = s[:q]
s = s[q+1:]
}
n.Attr = append(n.Attr, Attribute{Key: key, Val: id})
if key == "public" {
key = "system"
} else {
key = ""
}
}
if key != "" || s != "" {
quirks = true
} else if len(n.Attr) > 0 {
if n.Attr[0].Key == "public" {
public := strings.ToLower(n.Attr[0].Val)
switch public {
case "-//w3o//dtd w3 html strict 3.0//en//", "-/w3d/dtd html 4.0 transitional/en", "html":
quirks = true
default:
for _, q := range quirkyIDs {
if strings.HasPrefix(public, q) {
quirks = true
break
}
}
}
// The following two public IDs only cause quirks mode if there is no system ID.
if len(n.Attr) == 1 && (strings.HasPrefix(public, "-//w3c//dtd html 4.01 frameset//") ||
strings.HasPrefix(public, "-//w3c//dtd html 4.01 transitional//")) {
quirks = true
}
}
if lastAttr := n.Attr[len(n.Attr)-1]; lastAttr.Key == "system" &&
strings.ToLower(lastAttr.Val) == "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd" {
quirks = true
}
}
return n, quirks
}
// quirkyIDs is a list of public doctype identifiers that cause a document
// to be interpreted in quirks mode. The identifiers should be in lower case.
var quirkyIDs = []string{
"+//silmaril//dtd html pro v0r11 19970101//",
"-//advasoft ltd//dtd html 3.0 aswedit + extensions//",
"-//as//dtd html 3.0 aswedit + extensions//",
"-//ietf//dtd html 2.0 level 1//",
"-//ietf//dtd html 2.0 level 2//",
"-//ietf//dtd html 2.0 strict level 1//",
"-//ietf//dtd html 2.0 strict level 2//",
"-//ietf//dtd html 2.0 strict//",
"-//ietf//dtd html 2.0//",
"-//ietf//dtd html 2.1e//",
"-//ietf//dtd html 3.0//",
"-//ietf//dtd html 3.2 final//",
"-//ietf//dtd html 3.2//",
"-//ietf//dtd html 3//",
"-//ietf//dtd html level 0//",
"-//ietf//dtd html level 1//",
"-//ietf//dtd html level 2//",
"-//ietf//dtd html level 3//",
"-//ietf//dtd html strict level 0//",
"-//ietf//dtd html strict level 1//",
"-//ietf//dtd html strict level 2//",
"-//ietf//dtd html strict level 3//",
"-//ietf//dtd html strict//",
"-//ietf//dtd html//",
"-//metrius//dtd metrius presentational//",
"-//microsoft//dtd internet explorer 2.0 html strict//",
"-//microsoft//dtd internet explorer 2.0 html//",
"-//microsoft//dtd internet explorer 2.0 tables//",
"-//microsoft//dtd internet explorer 3.0 html strict//",
"-//microsoft//dtd internet explorer 3.0 html//",
"-//microsoft//dtd internet explorer 3.0 tables//",
"-//netscape comm. corp.//dtd html//",
"-//netscape comm. corp.//dtd strict html//",
"-//o'reilly and associates//dtd html 2.0//",
"-//o'reilly and associates//dtd html extended 1.0//",
"-//o'reilly and associates//dtd html extended relaxed 1.0//",
"-//softquad software//dtd hotmetal pro 6.0::19990601::extensions to html 4.0//",
"-//softquad//dtd hotmetal pro 4.0::19971010::extensions to html 4.0//",
"-//spyglass//dtd html 2.0 extended//",
"-//sq//dtd html 2.0 hotmetal + extensions//",
"-//sun microsystems corp.//dtd hotjava html//",
"-//sun microsystems corp.//dtd hotjava strict html//",
"-//w3c//dtd html 3 1995-03-24//",
"-//w3c//dtd html 3.2 draft//",
"-//w3c//dtd html 3.2 final//",
"-//w3c//dtd html 3.2//",
"-//w3c//dtd html 3.2s draft//",
"-//w3c//dtd html 4.0 frameset//",
"-//w3c//dtd html 4.0 transitional//",
"-//w3c//dtd html experimental 19960712//",
"-//w3c//dtd html experimental 970421//",
"-//w3c//dtd w3 html//",
"-//w3o//dtd w3 html 3.0//",
"-//webtechs//dtd mozilla html 2.0//",
"-//webtechs//dtd mozilla html//",
}
vendor/code.google.com/p/go.net/html/entity.go
Generated Vendored
+2253
View File
File diff suppressed because it is too large. Load diff
vendor/code.google.com/p/go.net/html/escape.go
Generated Vendored
+258
View File
@@ -0,0 +1,258 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package html
import (
"bytes"
"strings"
"unicode/utf8"
)
// These replacements permit compatibility with old numeric entities that
// assumed Windows-1252 encoding.
// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
var replacementTable = [...]rune{
'\u20AC', // First entry is what 0x80 should be replaced with.
'\u0081',
'\u201A',
'\u0192',
'\u201E',
'\u2026',
'\u2020',
'\u2021',
'\u02C6',
'\u2030',
'\u0160',
'\u2039',
'\u0152',
'\u008D',
'\u017D',
'\u008F',
'\u0090',
'\u2018',
'\u2019',
'\u201C',
'\u201D',
'\u2022',
'\u2013',
'\u2014',
'\u02DC',
'\u2122',
'\u0161',
'\u203A',
'\u0153',
'\u009D',
'\u017E',
'\u0178', // Last entry is 0x9F.
// 0x00->'\uFFFD' is handled programmatically.
// 0x0D->'\u000D' is a no-op.
}
// unescapeEntity reads an entity like "&lt;" from b[src:] and writes the
// corresponding "<" to b[dst:], returning the incremented dst and src cursors.
// Precondition: b[src] == '&' && dst <= src.
// attribute should be true if parsing an attribute value.
func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) {
// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
// i starts at 1 because we already know that s[0] == '&'.
i, s := 1, b[src:]
if len(s) <= 1 {
b[dst] = b[src]
return dst + 1, src + 1
}
if s[i] == '#' {
if len(s) <= 3 { // We need to have at least "&#.".
b[dst] = b[src]
return dst + 1, src + 1
}
i++
c := s[i]
hex := false
if c == 'x' || c == 'X' {
hex = true
i++
}
x := '\x00'
for i < len(s) {
c = s[i]
i++
if hex {
if '0' <= c && c <= '9' {
x = 16*x + rune(c) - '0'
continue
} else if 'a' <= c && c <= 'f' {
x = 16*x + rune(c) - 'a' + 10
continue
} else if 'A' <= c && c <= 'F' {
x = 16*x + rune(c) - 'A' + 10
continue
}
} else if '0' <= c && c <= '9' {
x = 10*x + rune(c) - '0'
continue
}
if c != ';' {
i--
}
break
}
if i <= 3 { // No characters matched.
b[dst] = b[src]
return dst + 1, src + 1
}
if 0x80 <= x && x <= 0x9F {
// Replace characters from Windows-1252 with UTF-8 equivalents.
x = replacementTable[x-0x80]
} else if x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF {
// Replace invalid characters with the replacement character.
x = '\uFFFD'
}
return dst + utf8.EncodeRune(b[dst:], x), src + i
}
// Consume the maximum number of characters possible, with the
// consumed characters matching one of the named references.
for i < len(s) {
c := s[i]
i++
// Lower-cased characters are more common in entities, so we check for them first.
if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
continue
}
if c != ';' {
i--
}
break
}
entityName := string(s[1:i])
if entityName == "" {
// No-op.
} else if attribute && entityName[len(entityName)-1] != ';' && len(s) > i && s[i] == '=' {
// No-op.
} else if x := entity[entityName]; x != 0 {
return dst + utf8.EncodeRune(b[dst:], x), src + i
} else if x := entity2[entityName]; x[0] != 0 {
dst1 := dst + utf8.EncodeRune(b[dst:], x[0])
return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i
} else if !attribute {
maxLen := len(entityName) - 1
if maxLen > longestEntityWithoutSemicolon {
maxLen = longestEntityWithoutSemicolon
}
for j := maxLen; j > 1; j-- {
if x := entity[entityName[:j]]; x != 0 {
return dst + utf8.EncodeRune(b[dst:], x), src + j + 1
}
}
}
dst1, src1 = dst+i, src+i
copy(b[dst:dst1], b[src:src1])
return dst1, src1
}
// unescape unescapes b's entities in-place, so that "a&lt;b" becomes "a<b".
// attribute should be true if parsing an attribute value.
func unescape(b []byte, attribute bool) []byte {
for i, c := range b {
if c == '&' {
dst, src := unescapeEntity(b, i, i, attribute)
for src < len(b) {
c := b[src]
if c == '&' {
dst, src = unescapeEntity(b, dst, src, attribute)
} else {
b[dst] = c
dst, src = dst+1, src+1
}
}
return b[0:dst]
}
}
return b
}
// lower lower-cases the A-Z bytes in b in-place, so that "aBc" becomes "abc".
func lower(b []byte) []byte {
for i, c := range b {
if 'A' <= c && c <= 'Z' {
b[i] = c + 'a' - 'A'
}
}
return b
}
const escapedChars = "&'<>\"\r"
func escape(w writer, s string) error {
i := strings.IndexAny(s, escapedChars)
for i != -1 {
if _, err := w.WriteString(s[:i]); err != nil {
return err
}
var esc string
switch s[i] {
case '&':
esc = "&amp;"
case '\'':
// "&#39;" is shorter than "&apos;" and apos was not in HTML until HTML5.
esc = "&#39;"
case '<':
esc = "&lt;"
case '>':
esc = "&gt;"
case '"':
// "&#34;" is shorter than "&quot;".
esc = "&#34;"
case '\r':
esc = "&#13;"
default:
panic("unrecognized escape character")
}
s = s[i+1:]
if _, err := w.WriteString(esc); err != nil {
return err
}
i = strings.IndexAny(s, escapedChars)
}
_, err := w.WriteString(s)
return err
}
// EscapeString escapes special characters like "<" to become "&lt;". It
// escapes only five such characters: <, >, &, ' and ".
// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
// always true.
func EscapeString(s string) string {
if strings.IndexAny(s, escapedChars) == -1 {
return s
}
var buf bytes.Buffer
escape(&buf, s)
return buf.String()
}
// UnescapeString unescapes entities like "&lt;" to become "<". It unescapes a
// larger range of entities than EscapeString escapes. For example, "&aacute;"
// unescapes to "á", as does "&#225;" and "&xE1;".
// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
// always true.
func UnescapeString(s string) string {
for _, c := range s {
if c == '&' {
return string(unescape([]byte(s), false))
}
}
return s
}
vendor/code.google.com/p/go.net/html/foreign.go
Generated Vendored
+226
View File
@@ -0,0 +1,226 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package html
import (
"strings"
)
func adjustAttributeNames(aa []Attribute, nameMap map[string]string) {
for i := range aa {
if newName, ok := nameMap[aa[i].Key]; ok {
aa[i].Key = newName
}
}
}
func adjustForeignAttributes(aa []Attribute) {
for i, a := range aa {
if a.Key == "" || a.Key[0] != 'x' {
continue
}
switch a.Key {
case "xlink:actuate", "xlink:arcrole", "xlink:href", "xlink:role", "xlink:show",
"xlink:title", "xlink:type", "xml:base", "xml:lang", "xml:space", "xmlns:xlink":
j := strings.Index(a.Key, ":")
aa[i].Namespace = a.Key[:j]
aa[i].Key = a.Key[j+1:]
}
}
}
func htmlIntegrationPoint(n *Node) bool {
if n.Type != ElementNode {
return false
}
switch n.Namespace {
case "math":
if n.Data == "annotation-xml" {
for _, a := range n.Attr {
if a.Key == "encoding" {
val := strings.ToLower(a.Val)
if val == "text/html" || val == "application/xhtml+xml" {
return true
}
}
}
}
case "svg":
switch n.Data {
case "desc", "foreignObject", "title":
return true
}
}
return false
}
func mathMLTextIntegrationPoint(n *Node) bool {
if n.Namespace != "math" {
return false
}
switch n.Data {
case "mi", "mo", "mn", "ms", "mtext":
return true
}
return false
}
// Section 12.2.5.5.
var breakout = map[string]bool{
"b": true,
"big": true,
"blockquote": true,
"body": true,
"br": true,
"center": true,
"code": true,
"dd": true,
"div": true,
"dl": true,
"dt": true,
"em": true,
"embed": true,
"h1": true,
"h2": true,
"h3": true,
"h4": true,
"h5": true,
"h6": true,
"head": true,
"hr": true,
"i": true,
"img": true,
"li": true,
"listing": true,
"menu": true,
"meta": true,
"nobr": true,
"ol": true,
"p": true,
"pre": true,
"ruby": true,
"s": true,
"small": true,
"span": true,
"strong": true,
"strike": true,
"sub": true,
"sup": true,
"table": true,
"tt": true,
"u": true,
"ul": true,
"var": true,
}
// Section 12.2.5.5.
var svgTagNameAdjustments = map[string]string{
"altglyph": "altGlyph",
"altglyphdef": "altGlyphDef",
"altglyphitem": "altGlyphItem",
"animatecolor": "animateColor",
"animatemotion": "animateMotion",
"animatetransform": "animateTransform",
"clippath": "clipPath",
"feblend": "feBlend",
"fecolormatrix": "feColorMatrix",
"fecomponenttransfer": "feComponentTransfer",
"fecomposite": "feComposite",
"feconvolvematrix": "feConvolveMatrix",
"fediffuselighting": "feDiffuseLighting",
"fedisplacementmap": "feDisplacementMap",
"fedistantlight": "feDistantLight",
"feflood": "feFlood",
"fefunca": "feFuncA",
"fefuncb": "feFuncB",
"fefuncg": "feFuncG",
"fefuncr": "feFuncR",
"fegaussianblur": "feGaussianBlur",
"feimage": "feImage",
"femerge": "feMerge",
"femergenode": "feMergeNode",
"femorphology": "feMorphology",
"feoffset": "feOffset",
"fepointlight": "fePointLight",
"fespecularlighting": "feSpecularLighting",
"fespotlight": "feSpotLight",
"fetile": "feTile",
"feturbulence": "feTurbulence",
"foreignobject": "foreignObject",
"glyphref": "glyphRef",
"lineargradient": "linearGradient",
"radialgradient": "radialGradient",
"textpath": "textPath",
}
// Section 12.2.5.1
var mathMLAttributeAdjustments = map[string]string{
"definitionurl": "definitionURL",
}
var svgAttributeAdjustments = map[string]string{
"attributename": "attributeName",
"attributetype": "attributeType",
"basefrequency": "baseFrequency",
"baseprofile": "baseProfile",
"calcmode": "calcMode",
"clippathunits": "clipPathUnits",
"contentscripttype": "contentScriptType",
"contentstyletype": "contentStyleType",
"diffuseconstant": "diffuseConstant",
"edgemode": "edgeMode",
"externalresourcesrequired": "externalResourcesRequired",
"filterres": "filterRes",
"filterunits": "filterUnits",
"glyphref": "glyphRef",
"gradienttransform": "gradientTransform",
"gradientunits": "gradientUnits",
"kernelmatrix": "kernelMatrix",
"kernelunitlength": "kernelUnitLength",
"keypoints": "keyPoints",
"keysplines": "keySplines",
"keytimes": "keyTimes",
"lengthadjust": "lengthAdjust",
"limitingconeangle": "limitingConeAngle",
"markerheight": "markerHeight",
"markerunits": "markerUnits",
"markerwidth": "markerWidth",
"maskcontentunits": "maskContentUnits",
"maskunits": "maskUnits",
"numoctaves": "numOctaves",
"pathlength": "pathLength",
"patterncontentunits": "patternContentUnits",
"patterntransform": "patternTransform",
"patternunits": "patternUnits",
"pointsatx": "pointsAtX",
"pointsaty": "pointsAtY",
"pointsatz": "pointsAtZ",
"preservealpha": "preserveAlpha",
"preserveaspectratio": "preserveAspectRatio",
"primitiveunits": "primitiveUnits",
"refx": "refX",
"refy": "refY",
"repeatcount": "repeatCount",
"repeatdur": "repeatDur",
"requiredextensions": "requiredExtensions",
"requiredfeatures": "requiredFeatures",
"specularconstant": "specularConstant",
"specularexponent": "specularExponent",
"spreadmethod": "spreadMethod",
"startoffset": "startOffset",
"stddeviation": "stdDeviation",
"stitchtiles": "stitchTiles",
"surfacescale": "surfaceScale",
"systemlanguage": "systemLanguage",
"tablevalues": "tableValues",
"targetx": "targetX",
"targety": "targetY",
"textlength": "textLength",
"viewbox": "viewBox",
"viewtarget": "viewTarget",
"xchannelselector": "xChannelSelector",
"ychannelselector": "yChannelSelector",
"zoomandpan": "zoomAndPan",
}
vendor/code.google.com/p/go.net/html/node.go
Generated Vendored
+193
View File
@@ -0,0 +1,193 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package html
import (
"code.google.com/p/go.net/html/atom"
)
// A NodeType is the type of a Node.
type NodeType uint32
const (
ErrorNode NodeType = iota
TextNode
DocumentNode
ElementNode
CommentNode
DoctypeNode
scopeMarkerNode
)
// Section 12.2.3.3 says "scope markers are inserted when entering applet
// elements, buttons, object elements, marquees, table cells, and table
// captions, and are used to prevent formatting from 'leaking'".
var scopeMarker = Node{Type: scopeMarkerNode}
// A Node consists of a NodeType and some Data (tag name for element nodes,
// content for text) and are part of a tree of Nodes. Element nodes may also
// have a Namespace and contain a slice of Attributes. Data is unescaped, so
// that it looks like "a<b" rather than "a&lt;b". For element nodes, DataAtom
// is the atom for Data, or zero if Data is not a known tag name.
//
// An empty Namespace implies a "http://www.w3.org/1999/xhtml" namespace.
// Similarly, "math" is short for "http://www.w3.org/1998/Math/MathML", and
// "svg" is short for "http://www.w3.org/2000/svg".
type Node struct {
Parent, FirstChild, LastChild, PrevSibling, NextSibling *Node
Type NodeType
DataAtom atom.Atom
Data string
Namespace string
Attr []Attribute
}
// InsertBefore inserts newChild as a child of n, immediately before oldChild
// in the sequence of n's children. oldChild may be nil, in which case newChild
// is appended to the end of n's children.
//
// It will panic if newChild already has a parent or siblings.
func (n *Node) InsertBefore(newChild, oldChild *Node) {
if newChild.Parent != nil || newChild.PrevSibling != nil || newChild.NextSibling != nil {
panic("html: InsertBefore called for an attached child Node")
}
var prev, next *Node
if oldChild != nil {
prev, next = oldChild.PrevSibling, oldChild
} else {
prev = n.LastChild
}
if prev != nil {
prev.NextSibling = newChild
} else {
n.FirstChild = newChild
}
if next != nil {
next.PrevSibling = newChild
} else {
n.LastChild = newChild
}
newChild.Parent = n
newChild.PrevSibling = prev
newChild.NextSibling = next
}
// AppendChild adds a node c as a child of n.
//
// It will panic if c already has a parent or siblings.
func (n *Node) AppendChild(c *Node) {
if c.Parent != nil || c.PrevSibling != nil || c.NextSibling != nil {
panic("html: AppendChild called for an attached child Node")
}
last := n.LastChild
if last != nil {
last.NextSibling = c
} else {
n.FirstChild = c
}
n.LastChild = c
c.Parent = n
c.PrevSibling = last
}
// RemoveChild removes a node c that is a child of n. Afterwards, c will have
// no parent and no siblings.
//
// It will panic if c's parent is not n.
func (n *Node) RemoveChild(c *Node) {
if c.Parent != n {
panic("html: RemoveChild called for a non-child Node")
}
if n.FirstChild == c {
n.FirstChild = c.NextSibling
}
if c.NextSibling != nil {
c.NextSibling.PrevSibling = c.PrevSibling
}
if n.LastChild == c {
n.LastChild = c.PrevSibling
}
if c.PrevSibling != nil {
c.PrevSibling.NextSibling = c.NextSibling
}
c.Parent = nil
c.PrevSibling = nil
c.NextSibling = nil
}
// reparentChildren reparents all of src's child nodes to dst.
func reparentChildren(dst, src *Node) {
for {
child := src.FirstChild
if child == nil {
break
}
src.RemoveChild(child)
dst.AppendChild(child)
}
}
// clone returns a new node with the same type, data and attributes.
// The clone has no parent, no siblings and no children.
func (n *Node) clone() *Node {
m := &Node{
Type: n.Type,
DataAtom: n.DataAtom,
Data: n.Data,
Attr: make([]Attribute, len(n.Attr)),
}
copy(m.Attr, n.Attr)
return m
}
// nodeStack is a stack of nodes.
type nodeStack []*Node
// pop pops the stack. It will panic if s is empty.
func (s *nodeStack) pop() *Node {
i := len(*s)
n := (*s)[i-1]
*s = (*s)[:i-1]
return n
}
// top returns the most recently pushed node, or nil if s is empty.
func (s *nodeStack) top() *Node {
if i := len(*s); i > 0 {
return (*s)[i-1]
}
return nil
}
// index returns the index of the top-most occurrence of n in the stack, or -1
// if n is not present.
func (s *nodeStack) index(n *Node) int {
for i := len(*s) - 1; i >= 0; i-- {
if (*s)[i] == n {
return i
}
}
return -1
}
// insert inserts a node at the given index.
func (s *nodeStack) insert(i int, n *Node) {
(*s) = append(*s, nil)
copy((*s)[i+1:], (*s)[i:])
(*s)[i] = n
}
// remove removes a node from the stack. It is a no-op if n is not present.
func (s *nodeStack) remove(n *Node) {
i := s.index(n)
if i == -1 {
return
}
copy((*s)[i:], (*s)[i+1:])
j := len(*s) - 1
(*s)[j] = nil
*s = (*s)[:j]
}
vendor/code.google.com/p/go.net/html/parse.go
Generated Vendored
+2092
View File
File diff suppressed because it is too large. Load diff
vendor/code.google.com/p/go.net/html/render.go
Generated Vendored
+271
View File
@@ -0,0 +1,271 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package html
import (
"bufio"
"errors"
"fmt"
"io"
"strings"
)
type writer interface {
io.Writer
WriteByte(c byte) error // in Go 1.1, use io.ByteWriter
WriteString(string) (int, error)
}
// Render renders the parse tree n to the given writer.
//
// Rendering is done on a 'best effort' basis: calling Parse on the output of
// Render will always result in something similar to the original tree, but it
// is not necessarily an exact clone unless the original tree was 'well-formed'.
// 'Well-formed' is not easily specified; the HTML5 specification is
// complicated.
//
// Calling Parse on arbitrary input typically results in a 'well-formed' parse
// tree. However, it is possible for Parse to yield a 'badly-formed' parse tree.
// For example, in a 'well-formed' parse tree, no <a> element is a child of
// another <a> element: parsing "<a><a>" results in two sibling elements.
// Similarly, in a 'well-formed' parse tree, no <a> element is a child of a
// <table> element: parsing "<p><table><a>" results in a <p> with two sibling
// children; the <a> is reparented to the <table>'s parent. However, calling
// Parse on "<a><table><a>" does not return an error, but the result has an <a>
// element with an <a> child, and is therefore not 'well-formed'.
//
// Programmatically constructed trees are typically also 'well-formed', but it
// is possible to construct a tree that looks innocuous but, when rendered and
// re-parsed, results in a different tree. A simple example is that a solitary
// text node would become a tree containing <html>, <head> and <body> elements.
// Another example is that the programmatic equivalent of "a<head>b</head>c"
// becomes "<html><head><head/><body>abc</body></html>".
func Render(w io.Writer, n *Node) error {
if x, ok := w.(writer); ok {
return render(x, n)
}
buf := bufio.NewWriter(w)
if err := render(buf, n); err != nil {
return err
}
return buf.Flush()
}
// plaintextAbort is returned from render1 when a <plaintext> element
// has been rendered. No more end tags should be rendered after that.
var plaintextAbort = errors.New("html: internal error (plaintext abort)")
func render(w writer, n *Node) error {
err := render1(w, n)
if err == plaintextAbort {
err = nil
}
return err
}
func render1(w writer, n *Node) error {
// Render non-element nodes; these are the easy cases.
switch n.Type {
case ErrorNode:
return errors.New("html: cannot render an ErrorNode node")
case TextNode:
return escape(w, n.Data)
case DocumentNode:
for c := n.FirstChild; c != nil; c = c.NextSibling {
if err := render1(w, c); err != nil {
return err
}
}
return nil
case ElementNode:
// No-op.
case CommentNode:
if _, err := w.WriteString("<!--"); err != nil {
return err
}
if _, err := w.WriteString(n.Data); err != nil {
return err
}
if _, err := w.WriteString("-->"); err != nil {
return err
}
return nil
case DoctypeNode:
if _, err := w.WriteString("<!DOCTYPE "); err != nil {
return err
}
if _, err := w.WriteString(n.Data); err != nil {
return err
}
if n.Attr != nil {
var p, s string
for _, a := range n.Attr {
switch a.Key {
case "public":
p = a.Val
case "system":
s = a.Val
}
}
if p != "" {
if _, err := w.WriteString(" PUBLIC "); err != nil {
return err
}
if err := writeQuoted(w, p); err != nil {
return err
}
if s != "" {
if err := w.WriteByte(' '); err != nil {
return err
}
if err := writeQuoted(w, s); err != nil {
return err
}
}
} else if s != "" {
if _, err := w.WriteString(" SYSTEM "); err != nil {
return err
}
if err := writeQuoted(w, s); err != nil {
return err
}
}
}
return w.WriteByte('>')
default:
return errors.New("html: unknown node type")
}
// Render the <xxx> opening tag.
if err := w.WriteByte('<'); err != nil {
return err
}
if _, err := w.WriteString(n.Data); err != nil {
return err
}
for _, a := range n.Attr {
if err := w.WriteByte(' '); err != nil {
return err
}
if a.Namespace != "" {
if _, err := w.WriteString(a.Namespace); err != nil {
return err
}
if err := w.WriteByte(':'); err != nil {
return err
}
}
if _, err := w.WriteString(a.Key); err != nil {
return err
}
if _, err := w.WriteString(`="`); err != nil {
return err
}
if err := escape(w, a.Val); err != nil {
return err
}
if err := w.WriteByte('"'); err != nil {
return err
}
}
if voidElements[n.Data] {
if n.FirstChild != nil {
return fmt.Errorf("html: void element <%s> has child nodes", n.Data)
}
_, err := w.WriteString("/>")
return err
}
if err := w.WriteByte('>'); err != nil {
return err
}
// Add initial newline where there is danger of a newline beging ignored.
if c := n.FirstChild; c != nil && c.Type == TextNode && strings.HasPrefix(c.Data, "\n") {
switch n.Data {
case "pre", "listing", "textarea":
if err := w.WriteByte('\n'); err != nil {
return err
}
}
}
// Render any child nodes.
switch n.Data {
case "iframe", "noembed", "noframes", "noscript", "plaintext", "script", "style", "xmp":
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == TextNode {
if _, err := w.WriteString(c.Data); err != nil {
return err
}
} else {
if err := render1(w, c); err != nil {
return err
}
}
}
if n.Data == "plaintext" {
// Don't render anything else. <plaintext> must be the
// last element in the file, with no closing tag.
return plaintextAbort
}
default:
for c := n.FirstChild; c != nil; c = c.NextSibling {
if err := render1(w, c); err != nil {
return err
}
}
}
// Render the </xxx> closing tag.
if _, err := w.WriteString("</"); err != nil {
return err
}
if _, err := w.WriteString(n.Data); err != nil {
return err
}
return w.WriteByte('>')
}
// writeQuoted writes s to w surrounded by quotes. Normally it will use double
// quotes, but if s contains a double quote, it will use single quotes.
// It is used for writing the identifiers in a doctype declaration.
// In valid HTML, they can't contain both types of quotes.
func writeQuoted(w writer, s string) error {
var q byte = '"'
if strings.Contains(s, `"`) {
q = '\''
}
if err := w.WriteByte(q); err != nil {
return err
}
if _, err := w.WriteString(s); err != nil {
return err
}
if err := w.WriteByte(q); err != nil {
return err
}
return nil
}
// Section 12.1.2, "Elements", gives this list of void elements. Void elements
// are those that can't have any contents.
var voidElements = map[string]bool{
"area": true,
"base": true,
"br": true,
"col": true,
"command": true,
"embed": true,
"hr": true,
"img": true,
"input": true,
"keygen": true,
"link": true,
"meta": true,
"param": true,
"source": true,
"track": true,
"wbr": true,
}
vendor/code.google.com/p/go.net/html/token.go
Generated Vendored
+1198
View File
File diff suppressed because it is too large. Load diff
vendor/github.com/PuerkitoBio/goquery/LICENSE
Generated Vendored
+12
View File
@@ -0,0 +1,12 @@
Copyright (c) 2012-2013, Martin Angers & Contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/PuerkitoBio/goquery/README.md
Generated Vendored
+109
View File
@@ -0,0 +1,109 @@
# goquery - a little like that j-thing, only in Go
GoQuery brings a syntax and a set of features similar to [jQuery][] to the [Go language][go]. It is based on the experimental html package and the CSS Selector library [cascadia][]. Since the experimental html parser returns tokens (nodes), and not a full-featured DOM object, jQuery's manipulation and modification functions have been left off (no point in modifying data in the parsed tree of the HTML, it has no effect).
Supported functions are query-oriented features (`hasClass()`, `attr()` and the likes), as well as traversing functions that make sense given what we have to work with. This makes GoQuery a great library for scraping web pages.
Syntax-wise, it is as close as possible to jQuery, with the same function names when possible, and that warm and fuzzy chainable interface. jQuery being the ultra-popular library that it is, I felt that writing a similar HTML-manipulating library was better to follow its API than to start anew (in the same spirit as Go's `fmt` package), even though some of its methods are less than intuitive (looking at you, [index()][index]...).
## Installation
$ go get github.com/PuerkitoBio/goquery
(optional) To run unit tests:
$ cd $GOPATH/src/github.com/PuerkitoBio/goquery
$ go test
(optional) To run benchmarks:
$ cd $GOPATH/src/github.com/PuerkitoBio/goquery
$ go test -bench=".*"
## Changelog
* **v0.3.2** : Add `NewDocumentFromReader()` (thanks jweir) which allows creating a goquery document from an io.Reader.
* **v0.3.1** : Add `NewDocumentFromResponse()` (thanks assassingj) which allows creating a goquery document from an http response.
* **v0.3.0** : Add `EachWithBreak()` which allows to break out of an `Each()` loop by returning false. This function was added instead of changing the existing `Each()` to avoid breaking compatibility.
* **v0.2.1** : Make go-getable, now that [go.net/html is Go1.0-compatible][gonet] (thanks to @matrixik for pointing this out).
* **v0.2.0** : Add support for negative indices in Slice(). **BREAKING CHANGE** `Document.Root` is removed, `Document` is now a `Selection` itself (a selection of one, the root element, just like `Document.Root` was before). Add jQuery's Closest() method.
* **v0.1.1** : Add benchmarks to use as baseline for refactorings, refactor Next...() and Prev...() methods to use the new html package's linked list features (Next/PrevSibling, FirstChild). Good performance boost (40+% in some cases).
* **v0.1.0** : Initial release.
## API
GoQuery exposes two classes, `Document` and `Selection`. Unlike jQuery, which is loaded as part of a DOM document, and thus acts on its containing document, GoQuery doesn't know which HTML document to act upon. So it needs to be told, and that's what the `Document` class is for. It holds the root document node as the initial Selection object to manipulate.
jQuery often has many variants for the same function (no argument, a selector string argument, a jQuery object argument, a DOM element argument, ...). Instead of exposing the same features in GoQuery as a single method with variadic empty interface arguments, I use statically-typed signatures following this naming convention:
* When the jQuery equivalent can be called with no argument, it has the same name as jQuery for the no argument signature (e.g.: `Prev()`), and the version with a selector string argument is called `XxxFiltered()` (e.g.: `PrevFiltered()`)
* When the jQuery equivalent **requires** one argument, the same name as jQuery is used for the selector string version (e.g.: `Is()`)
* The signatures accepting a jQuery object as argument are defined in GoQuery as `XxxSelection()` and take a `*Selection` object as argument (e.g.: `FilterSelection()`)
* The signatures accepting a DOM element as argument in jQuery are defined in GoQuery as `XxxNodes()` and take a variadic argument of type `*html.Node` (e.g.: `FilterNodes()`)
* Finally, the signatures accepting a function as argument in jQuery are defined in GoQuery as `XxxFunction()` and take a function as argument (e.g.: `FilterFunction()`)
GoQuery's complete [godoc reference documentation can be found here][doc].
Please note that Cascadia's selectors do NOT necessarily match all supported selectors of jQuery (Sizzle). See the [cascadia project][cascadia] for details.
## Examples
Taken from example_test.go:
```Go
import (
"fmt"
// In real use, this import would be required (not in this example, since it
// is part of the goquery package)
//"github.com/PuerkitoBio/goquery"
"strconv"
)
// This example scrapes the 10 reviews shown on the home page of MetalReview.com,
// the best metal review site on the web :) (and no, I'm not affiliated to them!)
func ExampleScrape_MetalReview() {
// Load the HTML document (in real use, the type would be *goquery.Document)
var doc *Document
var e error
if doc, e = NewDocument("http://metalreview.com"); e != nil {
panic(e.Error())
}
// Find the review items (the type of the Selection would be *goquery.Selection)
doc.Find(".slider-row:nth-child(1) .slider-item").Each(func(i int, s *Selection) {
var band, title string
var score float64
// For each item found, get the band, title and score, and print it
band = s.Find("strong").Text()
title = s.Find("em").Text()
if score, e = strconv.ParseFloat(s.Find(".score").Text(), 64); e != nil {
// Not a valid float, ignore score
fmt.Printf("Review %d: %s - %s.\n", i, band, title)
} else {
// Print all, including score
fmt.Printf("Review %d: %s - %s (%2.1f).\n", i, band, title, score)
}
})
// To see the output of the Example while running the test suite (go test), simply
// remove the leading "x" before Output on the next line. This will cause the
// example to fail (all the "real" tests should pass).
// xOutput: voluntarily fail the Example output.
}
```
## License
The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia's license is [here][caslic].
[jquery]: http://jquery.com/
[go]: http://golang.org/
[cascadia]: http://code.google.com/p/cascadia/
[bsd]: http://opensource.org/licenses/BSD-3-Clause
[golic]: http://golang.org/LICENSE
[caslic]: http://code.google.com/p/cascadia/source/browse/LICENSE
[doc]: http://godoc.org/github.com/PuerkitoBio/goquery
[index]: http://api.jquery.com/index/
[gonet]: http://code.google.com/p/go/source/detail?r=f7f5159120f51ba0070774d3c5907969b5fe7858&repo=net
vendor/github.com/PuerkitoBio/goquery/array.go
Generated Vendored
+85
View File
@@ -0,0 +1,85 @@
package goquery
import (
"code.google.com/p/go.net/html"
)
// First() reduces the set of matched elements to the first in the set.
// It returns a new Selection object.
func (this *Selection) First() *Selection {
return this.Eq(0)
}
// Last() reduces the set of matched elements to the last in the set.
// It returns a new Selection object.
func (this *Selection) Last() *Selection {
return this.Eq(-1)
}
// Eq() reduces the set of matched elements to the one at the specified index.
// If a negative index is given, it counts backwards starting at the end of the
// set. It returns a new Selection object, and an empty Selection object if the
// index is invalid.
func (this *Selection) Eq(index int) *Selection {
if index < 0 {
index += len(this.Nodes)
}
return this.Slice(index, index+1)
}
// Slice() reduces the set of matched elements to a subset specified by a range
// of indices.
func (this *Selection) Slice(start int, end int) *Selection {
if start < 0 {
start += len(this.Nodes)
}
if end < 0 {
end += len(this.Nodes)
}
return pushStack(this, this.Nodes[start:end])
}
// Get() retrieves the underlying node at the specified index.
// Get() without parameter is not implemented, since the node array is available
// on the Selection object.
func (this *Selection) Get(index int) *html.Node {
if index < 0 {
index += len(this.Nodes) // Negative index gets from the end
}
return this.Nodes[index]
}
// Index() returns the position of the first element within the Selection object
// relative to its sibling elements.
func (this *Selection) Index() int {
if len(this.Nodes) > 0 {
return newSingleSelection(this.Nodes[0], this.document).PrevAll().Length()
}
return -1
}
// IndexSelector() returns the position of the first element within the
// Selection object relative to the elements matched by the selector, or -1 if
// not found.
func (this *Selection) IndexSelector(selector string) int {
if len(this.Nodes) > 0 {
sel := this.document.Find(selector)
return indexInSlice(sel.Nodes, this.Nodes[0])
}
return -1
}
// IndexOfNode() returns the position of the specified node within the Selection
// object, or -1 if not found.
func (this *Selection) IndexOfNode(node *html.Node) int {
return indexInSlice(this.Nodes, node)
}
// IndexOfSelection() returns the position of the first node in the specified
// Selection object within this Selection object, or -1 if not found.
func (this *Selection) IndexOfSelection(s *Selection) int {
if s != nil && len(s.Nodes) > 0 {
return indexInSlice(this.Nodes, s.Nodes[0])
}
return -1
}
vendor/github.com/PuerkitoBio/goquery/doc.go
Generated Vendored
+91
View File
@@ -0,0 +1,91 @@
// Copyright (c) 2012-2013, Martin Angers & Contributors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
// * Neither the name of the author nor the names of its contributors may be used to
// endorse or promote products derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
// WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/*
Package goquery implements features similar to jQuery, including the chainable
syntax, to manipulate and query an HTML document (the modification functions of jQuery are not included).
It uses Cascadia as CSS selector (similar to Sizzle for jQuery).
To provide a chainable interface, error management is strict, and goquery panics
if an invalid Cascadia selector is used (this is consistent with the behavior of
jQuery/Sizzle/document.querySelectorAll, where an error is thrown). This is
necessary since multiple return values cannot be used to allow a chainable
interface.
It is hosted on GitHub, along with additional documentation in the README.md
file: https://github.com/puerkitobio/goquery
The various methods are split into files based on the category of behavior:
* array.go : array-like positional manipulation of the selection.
- Eq()
- First()
- Get()
- Index...()
- Last()
- Slice()
* expand.go : methods that expand or augment the selection's set.
- Add...()
- AndSelf()
- Union(), which is an alias for AddSelection()
* filter.go : filtering methods, that reduce the selection's set.
- End()
- Filter...()
- Has...()
- Intersection(), which is an alias of FilterSelection()
- Not...()
* iteration.go : methods to loop over the selection's nodes.
- Each()
- EachWithBreak()
- Map()
* property.go : methods that inspect and get the node's properties values.
- Attr()
- Html()
- Length()
- Size(), which is an alias for Length()
- Text()
* query.go : methods that query, or reflect, a node's identity.
- Contains()
- HasClass()
- Is...()
* traversal.go : methods to traverse the HTML document tree.
- Children...()
- Contents()
- Find...()
- Next...()
- Parent[s]...()
- Prev...()
- Siblings...()
* type.go : definition of the types exposed by GoQuery.
- Document
- Selection
*/
package goquery
vendor/github.com/PuerkitoBio/goquery/expand.go
Generated Vendored
+40
View File
@@ -0,0 +1,40 @@
package goquery
import (
"code.google.com/p/go.net/html"
)
// Add() adds the selector string's matching nodes to those in the current
// selection and returns a new Selection object.
// The selector string is run in the context of the document of the current
// Selection object.
func (this *Selection) Add(selector string) *Selection {
return this.AddNodes(findWithSelector([]*html.Node{this.document.rootNode}, selector)...)
}
// AddSelection() adds the specified Selection object's nodes to those in the
// current selection and returns a new Selection object.
func (this *Selection) AddSelection(sel *Selection) *Selection {
if sel == nil {
return this.AddNodes()
}
return this.AddNodes(sel.Nodes...)
}
// Union() is an alias for AddSelection().
func (this *Selection) Union(sel *Selection) *Selection {
return this.AddSelection(sel)
}
// AddNodes() adds the specified nodes to those in the
// current selection and returns a new Selection object.
func (this *Selection) AddNodes(nodes ...*html.Node) *Selection {
return pushStack(this, appendWithoutDuplicates(this.Nodes, nodes))
}
// AndSelf() adds the previous set of elements on the stack to the current set.
// It returns a new Selection object containing the current Selection combined
// with the previous one.
func (this *Selection) AndSelf() *Selection {
return this.AddSelection(this.prevSel)
}
vendor/github.com/PuerkitoBio/goquery/filter.go
Generated Vendored
+141
View File
@@ -0,0 +1,141 @@
package goquery
import (
"code.google.com/p/cascadia"
"code.google.com/p/go.net/html"
)
// Filter() reduces the set of matched elements to those that match the selector string.
// It returns a new Selection object for this subset of matching elements.
func (this *Selection) Filter(selector string) *Selection {
return pushStack(this, winnow(this, selector, true))
}
// Not() removes elements from the Selection that match the selector string.
// It returns a new Selection object with the matching elements removed.
func (this *Selection) Not(selector string) *Selection {
return pushStack(this, winnow(this, selector, false))
}
// FilterFunction() reduces the set of matched elements to those that pass the function's test.
// It returns a new Selection object for this subset of elements.
func (this *Selection) FilterFunction(f func(int, *Selection) bool) *Selection {
return pushStack(this, winnowFunction(this, f, true))
}
// Not() removes elements from the Selection that pass the function's test.
// It returns a new Selection object with the matching elements removed.
func (this *Selection) NotFunction(f func(int, *Selection) bool) *Selection {
return pushStack(this, winnowFunction(this, f, false))
}
// FilterNodes() reduces the set of matched elements to those that match the specified nodes.
// It returns a new Selection object for this subset of elements.
func (this *Selection) FilterNodes(nodes ...*html.Node) *Selection {
return pushStack(this, winnowNodes(this, nodes, true))
}
// Not() removes elements from the Selection that match the specified nodes.
// It returns a new Selection object with the matching elements removed.
func (this *Selection) NotNodes(nodes ...*html.Node) *Selection {
return pushStack(this, winnowNodes(this, nodes, false))
}
// FilterSelection() reduces the set of matched elements to those that match a
// node in the specified Selection object.
// It returns a new Selection object for this subset of elements.
func (this *Selection) FilterSelection(s *Selection) *Selection {
if s == nil {
return pushStack(this, winnowNodes(this, nil, true))
}
return pushStack(this, winnowNodes(this, s.Nodes, true))
}
// Not() removes elements from the Selection that match a node in the specified
// Selection object.
// It returns a new Selection object with the matching elements removed.
func (this *Selection) NotSelection(s *Selection) *Selection {
if s == nil {
return pushStack(this, winnowNodes(this, nil, false))
}
return pushStack(this, winnowNodes(this, s.Nodes, false))
}
// Intersection() is an alias for FilterSelection().
func (this *Selection) Intersection(s *Selection) *Selection {
return this.FilterSelection(s)
}
// Has() reduces the set of matched elements to those that have a descendant
// that matches the selector.
// It returns a new Selection object with the matching elements.
func (this *Selection) Has(selector string) *Selection {
return this.HasSelection(this.document.Find(selector))
}
// HasNodes() reduces the set of matched elements to those that have a
// descendant that matches one of the nodes.
// It returns a new Selection object with the matching elements.
func (this *Selection) HasNodes(nodes ...*html.Node) *Selection {
return this.FilterFunction(func(_ int, s *Selection) bool {
// Add all nodes that contain one of the specified nodes
for _, n := range nodes {
if s.Contains(n) {
return true
}
}
return false
})
}
// HasSelection() reduces the set of matched elements to those that have a
// descendant that matches one of the nodes of the specified Selection object.
// It returns a new Selection object with the matching elements.
func (this *Selection) HasSelection(sel *Selection) *Selection {
if sel == nil {
return this.HasNodes()
}
return this.HasNodes(sel.Nodes...)
}
// End() ends the most recent filtering operation in the current chain and
// returns the set of matched elements to its previous state.
func (this *Selection) End() *Selection {
if this.prevSel != nil {
return this.prevSel
}
return newEmptySelection(this.document)
}
// Filter based on a selector string, and the indicator to keep (Filter) or
// to get rid of (Not) the matching elements.
func winnow(sel *Selection, selector string, keep bool) []*html.Node {
cs := cascadia.MustCompile(selector)
// Optimize if keep is requested
if keep {
return cs.Filter(sel.Nodes)
} else {
// Use grep
return grep(sel, func(i int, s *Selection) bool {
return !cs.Match(s.Get(0))
})
}
return nil
}
// Filter based on an array of nodes, and the indicator to keep (Filter) or
// to get rid of (Not) the matching elements.
func winnowNodes(sel *Selection, nodes []*html.Node, keep bool) []*html.Node {
return grep(sel, func(i int, s *Selection) bool {
return isInSlice(nodes, s.Get(0)) == keep
})
}
// Filter based on a function test, and the indicator to keep (Filter) or
// to get rid of (Not) the matching elements.
func winnowFunction(sel *Selection, f func(int, *Selection) bool, keep bool) []*html.Node {
return grep(sel, func(i int, s *Selection) bool {
return f(i, s) == keep
})
}
vendor/github.com/PuerkitoBio/goquery/iteration.go
Generated Vendored
+33
View File
@@ -0,0 +1,33 @@
package goquery
// Each() iterates over a Selection object, executing a function for each
// matched element. It returns the current Selection object.
func (this *Selection) Each(f func(int, *Selection)) *Selection {
for i, n := range this.Nodes {
f(i, newSingleSelection(n, this.document))
}
return this
}
// EachWithBreak() iterates over a Selection object, executing a function for each
// matched element. It is identical to `Each()` except that it is possible to break
// out of the loop by returning `false` in the callback function. It returns the
// current Selection object.
func (this *Selection) EachWithBreak(f func(int, *Selection) bool) *Selection {
for i, n := range this.Nodes {
if !f(i, newSingleSelection(n, this.document)) {
return this
}
}
return this
}
// Map() passes each element in the current matched set through a function,
// producing a slice of string holding the returned values.
func (this *Selection) Map(f func(int, *Selection) string) (result []string) {
for i, n := range this.Nodes {
result = append(result, f(i, newSingleSelection(n, this.document)))
}
return result
}
vendor/github.com/PuerkitoBio/goquery/property.go
Generated Vendored
+90
View File
@@ -0,0 +1,90 @@
package goquery
import (
"bytes"
"code.google.com/p/go.net/html"
)
// Attr() gets the specified attribute's value for the first element in the
// Selection. To get the value for each element individually, use a looping
// construct such as Each() or Map() method.
func (this *Selection) Attr(attrName string) (val string, exists bool) {
if len(this.Nodes) == 0 {
return
}
return getAttributeValue(attrName, this.Nodes[0])
}
// Text() gets the combined text contents of each element in the set of matched
// elements, including their descendants.
func (this *Selection) Text() string {
var buf bytes.Buffer
// Slightly optimized vs calling Each(): no single selection object created
for _, n := range this.Nodes {
buf.WriteString(getNodeText(n))
}
return buf.String()
}
// Size() is an alias for Length().
func (this *Selection) Size() int {
return this.Length()
}
// Length() returns the number of elements in the Selection object.
func (this *Selection) Length() int {
return len(this.Nodes)
}
// Html() gets the HTML contents of the first element in the set of matched
// elements. It includes text and comment nodes.
func (this *Selection) Html() (ret string, e error) {
// Since there is no .innerHtml, the HTML content must be re-created from
// the nodes usint html.Render().
var buf bytes.Buffer
if len(this.Nodes) > 0 {
for c := this.Nodes[0].FirstChild; c != nil; c = c.NextSibling {
e = html.Render(&buf, c)
if e != nil {
return
}
}
ret = buf.String()
}
return
}
// Get the specified node's text content.
func getNodeText(node *html.Node) string {
if node.Type == html.TextNode {
// Keep newlines and spaces, like jQuery
return node.Data
} else if node.FirstChild != nil {
var buf bytes.Buffer
for c := node.FirstChild; c != nil; c = c.NextSibling {
buf.WriteString(getNodeText(c))
}
return buf.String()
}
return ""
}
// Private function to get the specified attribute's value from a node.
func getAttributeValue(attrName string, n *html.Node) (val string, exists bool) {
if n == nil {
return
}
for _, a := range n.Attr {
if a.Key == attrName {
val = a.Val
exists = true
return
}
}
return
}
vendor/github.com/PuerkitoBio/goquery/query.go
Generated Vendored
+71
View File
@@ -0,0 +1,71 @@
package goquery
import (
"code.google.com/p/cascadia"
"code.google.com/p/go.net/html"
"regexp"
"strings"
)
var rxClassTrim = regexp.MustCompile("[\t\r\n]")
// Is() checks the current matched set of elements against a selector and
// returns true if at least one of these elements matches.
func (this *Selection) Is(selector string) bool {
if len(this.Nodes) > 0 {
// Attempt a match with the selector
cs := cascadia.MustCompile(selector)
if len(this.Nodes) == 1 {
return cs.Match(this.Nodes[0])
} else {
return len(cs.Filter(this.Nodes)) > 0
}
}
return false
}
// IsFunction() checks the current matched set of elements against a predicate and
// returns true if at least one of these elements matches.
func (this *Selection) IsFunction(f func(int, *Selection) bool) bool {
return this.FilterFunction(f).Length() > 0
}
// IsSelection() checks the current matched set of elements against a Selection object
// and returns true if at least one of these elements matches.
func (this *Selection) IsSelection(s *Selection) bool {
return this.FilterSelection(s).Length() > 0
}
// IsNodes() checks the current matched set of elements against the specified nodes
// and returns true if at least one of these elements matches.
func (this *Selection) IsNodes(nodes ...*html.Node) bool {
return this.FilterNodes(nodes...).Length() > 0
}
// HasClass() determines whether any of the matched elements are assigned the
// given class.
func (this *Selection) HasClass(class string) bool {
class = " " + class + " "
for _, n := range this.Nodes {
// Applies only to element nodes
if n.Type == html.ElementNode {
if elClass, ok := getAttributeValue("class", n); ok {
elClass = rxClassTrim.ReplaceAllString(" "+elClass+" ", " ")
if strings.Index(elClass, class) > -1 {
return true
}
}
}
}
return false
}
// Contains() returns true if the specified Node is within,
// at any depth, one of the nodes in the Selection object.
// It is NOT inclusive, to behave like jQuery's implementation, and
// unlike Javascript's .contains(), so if the contained
// node is itself in the selection, it returns false.
func (this *Selection) Contains(n *html.Node) bool {
return sliceContains(this.Nodes, n)
}
vendor/github.com/PuerkitoBio/goquery/traversal.go
Generated Vendored
+537
View File
@@ -0,0 +1,537 @@
package goquery
import (
"code.google.com/p/cascadia"
"code.google.com/p/go.net/html"
)
type siblingType int
// Sibling type, used internally when iterating over children at the same
// level (siblings) to specify which nodes are requested.
const (
siblingPrevUntil siblingType = iota - 3
siblingPrevAll
siblingPrev
siblingAll
siblingNext
siblingNextAll
siblingNextUntil
siblingAllIncludingNonElements
)
// Find() gets the descendants of each element in the current set of matched
// elements, filtered by a selector. It returns a new Selection object
// containing these matched elements.
func (this *Selection) Find(selector string) *Selection {
return pushStack(this, findWithSelector(this.Nodes, selector))
}
// FindSelection() gets the descendants of each element in the current
// Selection, filtered by a Selection. It returns a new Selection object
// containing these matched elements.
func (this *Selection) FindSelection(sel *Selection) *Selection {
if sel == nil {
return pushStack(this, nil)
}
return this.FindNodes(sel.Nodes...)
}
// FindNodes() gets the descendants of each element in the current
// Selection, filtered by some nodes. It returns a new Selection object
// containing these matched elements.
func (this *Selection) FindNodes(nodes ...*html.Node) *Selection {
return pushStack(this, mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
if sliceContains(this.Nodes, n) {
return []*html.Node{n}
}
return nil
}))
}
// Contents() gets the children of each element in the Selection,
// including text and comment nodes. It returns a new Selection object
// containing these elements.
func (this *Selection) Contents() *Selection {
return pushStack(this, getChildrenNodes(this.Nodes, siblingAllIncludingNonElements))
}
// ContentsFiltered() gets the children of each element in the Selection,
// filtered by the specified selector. It returns a new Selection
// object containing these elements. Since selectors only act on Element nodes,
// this function is an alias to ChildrenFiltered() unless the selector is empty,
// in which case it is an alias to Contents().
func (this *Selection) ContentsFiltered(selector string) *Selection {
if selector != "" {
return this.ChildrenFiltered(selector)
}
return this.Contents()
}
// Children() gets the child elements of each element in the Selection.
// It returns a new Selection object containing these elements.
func (this *Selection) Children() *Selection {
return pushStack(this, getChildrenNodes(this.Nodes, siblingAll))
}
// ChildrenFiltered() gets the child elements of each element in the Selection,
// filtered by the specified selector. It returns a new
// Selection object containing these elements.
func (this *Selection) ChildrenFiltered(selector string) *Selection {
return filterAndPush(this, getChildrenNodes(this.Nodes, siblingAll), selector)
}
// Parent() gets the parent of each element in the Selection. It returns a
// new Selection object containing the matched elements.
func (this *Selection) Parent() *Selection {
return pushStack(this, getParentNodes(this.Nodes))
}
// ParentFiltered() gets the parent of each element in the Selection filtered by a
// selector. It returns a new Selection object containing the matched elements.
func (this *Selection) ParentFiltered(selector string) *Selection {
return filterAndPush(this, getParentNodes(this.Nodes), selector)
}
// Closest() gets the first element that matches the selector by testing the
// element itself and traversing up through its ancestors in the DOM tree.
func (this *Selection) Closest(selector string) *Selection {
cs := cascadia.MustCompile(selector)
return pushStack(this, mapNodes(this.Nodes, func(i int, n *html.Node) []*html.Node {
// For each node in the selection, test the node itself, then each parent
// until a match is found.
for ; n != nil; n = n.Parent {
if cs.Match(n) {
return []*html.Node{n}
}
}
return nil
}))
}
// ClosestNodes() gets the first element that matches one of the nodes by testing the
// element itself and traversing up through its ancestors in the DOM tree.
func (this *Selection) ClosestNodes(nodes ...*html.Node) *Selection {
return pushStack(this, mapNodes(this.Nodes, func(i int, n *html.Node) []*html.Node {
// For each node in the selection, test the node itself, then each parent
// until a match is found.
for ; n != nil; n = n.Parent {
if isInSlice(nodes, n) {
return []*html.Node{n}
}
}
return nil
}))
}
// ClosestSelection() gets the first element that matches one of the nodes in the
// Selection by testing the element itself and traversing up through its ancestors
// in the DOM tree.
func (this *Selection) ClosestSelection(s *Selection) *Selection {
if s == nil {
return pushStack(this, nil)
}
return this.ClosestNodes(s.Nodes...)
}
// Parents() gets the ancestors of each element in the current Selection. It
// returns a new Selection object with the matched elements.
func (this *Selection) Parents() *Selection {
return pushStack(this, getParentsNodes(this.Nodes, "", nil))
}
// ParentsFiltered() gets the ancestors of each element in the current
// Selection. It returns a new Selection object with the matched elements.
func (this *Selection) ParentsFiltered(selector string) *Selection {
return filterAndPush(this, getParentsNodes(this.Nodes, "", nil), selector)
}
// ParentsUntil() gets the ancestors of each element in the Selection, up to but
// not including the element matched by the selector. It returns a new Selection
// object containing the matched elements.
func (this *Selection) ParentsUntil(selector string) *Selection {
return pushStack(this, getParentsNodes(this.Nodes, selector, nil))
}
// ParentsUntilSelection() gets the ancestors of each element in the Selection,
// up to but not including the elements in the specified Selection. It returns a
// new Selection object containing the matched elements.
func (this *Selection) ParentsUntilSelection(sel *Selection) *Selection {
if sel == nil {
return this.Parents()
}
return this.ParentsUntilNodes(sel.Nodes...)
}
// ParentsUntilNodes() gets the ancestors of each element in the Selection,
// up to but not including the specified nodes. It returns a
// new Selection object containing the matched elements.
func (this *Selection) ParentsUntilNodes(nodes ...*html.Node) *Selection {
return pushStack(this, getParentsNodes(this.Nodes, "", nodes))
}
// ParentsFilteredUntil() is like ParentsUntil(), with the option to filter the
// results based on a selector string. It returns a new Selection
// object containing the matched elements.
func (this *Selection) ParentsFilteredUntil(filterSelector string, untilSelector string) *Selection {
return filterAndPush(this, getParentsNodes(this.Nodes, untilSelector, nil), filterSelector)
}
// ParentsFilteredUntilSelection() is like ParentsUntilSelection(), with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (this *Selection) ParentsFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
if sel == nil {
return this.ParentsFiltered(filterSelector)
}
return this.ParentsFilteredUntilNodes(filterSelector, sel.Nodes...)
}
// ParentsFilteredUntilNodes() is like ParentsUntilNodes(), with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (this *Selection) ParentsFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
return filterAndPush(this, getParentsNodes(this.Nodes, "", nodes), filterSelector)
}
// Siblings() gets the siblings of each element in the Selection. It returns
// a new Selection object containing the matched elements.
func (this *Selection) Siblings() *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingAll, "", nil))
}
// SiblingsFiltered() gets the siblings of each element in the Selection
// filtered by a selector. It returns a new Selection object containing the
// matched elements.
func (this *Selection) SiblingsFiltered(selector string) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingAll, "", nil), selector)
}
// Next() gets the immediately following sibling of each element in the
// Selection. It returns a new Selection object containing the matched elements.
func (this *Selection) Next() *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingNext, "", nil))
}
// NextFiltered() gets the immediately following sibling of each element in the
// Selection filtered by a selector. It returns a new Selection object
// containing the matched elements.
func (this *Selection) NextFiltered(selector string) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingNext, "", nil), selector)
}
// NextAll() gets all the following siblings of each element in the
// Selection. It returns a new Selection object containing the matched elements.
func (this *Selection) NextAll() *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingNextAll, "", nil))
}
// NextAllFiltered() gets all the following siblings of each element in the
// Selection filtered by a selector. It returns a new Selection object
// containing the matched elements.
func (this *Selection) NextAllFiltered(selector string) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingNextAll, "", nil), selector)
}
// Prev() gets the immediately preceding sibling of each element in the
// Selection. It returns a new Selection object containing the matched elements.
func (this *Selection) Prev() *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingPrev, "", nil))
}
// PrevFiltered() gets the immediately preceding sibling of each element in the
// Selection filtered by a selector. It returns a new Selection object
// containing the matched elements.
func (this *Selection) PrevFiltered(selector string) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingPrev, "", nil), selector)
}
// PrevAll() gets all the preceding siblings of each element in the
// Selection. It returns a new Selection object containing the matched elements.
func (this *Selection) PrevAll() *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingPrevAll, "", nil))
}
// PrevAllFiltered() gets all the preceding siblings of each element in the
// Selection filtered by a selector. It returns a new Selection object
// containing the matched elements.
func (this *Selection) PrevAllFiltered(selector string) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingPrevAll, "", nil), selector)
}
// NextUntil() gets all following siblings of each element up to but not
// including the element matched by the selector. It returns a new Selection
// object containing the matched elements.
func (this *Selection) NextUntil(selector string) *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingNextUntil,
selector, nil))
}
// NextUntilSelection() gets all following siblings of each element up to but not
// including the element matched by the Selection. It returns a new Selection
// object containing the matched elements.
func (this *Selection) NextUntilSelection(sel *Selection) *Selection {
if sel == nil {
return this.NextAll()
}
return this.NextUntilNodes(sel.Nodes...)
}
// NextUntilNodes() gets all following siblings of each element up to but not
// including the element matched by the nodes. It returns a new Selection
// object containing the matched elements.
func (this *Selection) NextUntilNodes(nodes ...*html.Node) *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingNextUntil,
"", nodes))
}
// PrevUntil() gets all preceding siblings of each element up to but not
// including the element matched by the selector. It returns a new Selection
// object containing the matched elements.
func (this *Selection) PrevUntil(selector string) *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingPrevUntil,
selector, nil))
}
// PrevUntilSelection() gets all preceding siblings of each element up to but not
// including the element matched by the Selection. It returns a new Selection
// object containing the matched elements.
func (this *Selection) PrevUntilSelection(sel *Selection) *Selection {
if sel == nil {
return this.PrevAll()
}
return this.PrevUntilNodes(sel.Nodes...)
}
// PrevUntilNodes() gets all preceding siblings of each element up to but not
// including the element matched by the nodes. It returns a new Selection
// object containing the matched elements.
func (this *Selection) PrevUntilNodes(nodes ...*html.Node) *Selection {
return pushStack(this, getSiblingNodes(this.Nodes, siblingPrevUntil,
"", nodes))
}
// NextFilteredUntil() is like NextUntil(), with the option to filter
// the results based on a selector string.
// It returns a new Selection object containing the matched elements.
func (this *Selection) NextFilteredUntil(filterSelector string, untilSelector string) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingNextUntil,
untilSelector, nil), filterSelector)
}
// NextFilteredUntilSelection() is like NextUntilSelection(), with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (this *Selection) NextFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
if sel == nil {
return this.NextFiltered(filterSelector)
}
return this.NextFilteredUntilNodes(filterSelector, sel.Nodes...)
}
// NextFilteredUntilNodes() is like NextUntilNodes(), with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (this *Selection) NextFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingNextUntil,
"", nodes), filterSelector)
}
// PrevFilteredUntil() is like PrevUntil(), with the option to filter
// the results based on a selector string.
// It returns a new Selection object containing the matched elements.
func (this *Selection) PrevFilteredUntil(filterSelector string, untilSelector string) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingPrevUntil,
untilSelector, nil), filterSelector)
}
// PrevFilteredUntilSelection() is like PrevUntilSelection(), with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (this *Selection) PrevFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
if sel == nil {
return this.PrevFiltered(filterSelector)
}
return this.PrevFilteredUntilNodes(filterSelector, sel.Nodes...)
}
// PrevFilteredUntilNodes() is like PrevUntilNodes(), with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (this *Selection) PrevFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
return filterAndPush(this, getSiblingNodes(this.Nodes, siblingPrevUntil,
"", nodes), filterSelector)
}
// Filter and push filters the nodes based on a selector, and pushes the results
// on the stack, with the srcSel as previous selection.
func filterAndPush(srcSel *Selection, nodes []*html.Node, selector string) *Selection {
// Create a temporary Selection with the specified nodes to filter using winnow
sel := &Selection{nodes, srcSel.document, nil}
// Filter based on selector and push on stack
return pushStack(srcSel, winnow(sel, selector, true))
}
// Internal implementation of Find that return raw nodes.
func findWithSelector(nodes []*html.Node, selector string) []*html.Node {
// Compile the selector once
sel := cascadia.MustCompile(selector)
// Map nodes to find the matches within the children of each node
return mapNodes(nodes, func(i int, n *html.Node) (result []*html.Node) {
// Go down one level, becausejQuery's Find() selects only within descendants
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode {
result = append(result, sel.MatchAll(c)...)
}
}
return
})
}
// Internal implementation to get all parent nodes, stopping at the specified
// node (or nil if no stop).
func getParentsNodes(nodes []*html.Node, stopSelector string, stopNodes []*html.Node) []*html.Node {
return mapNodes(nodes, func(i int, n *html.Node) (result []*html.Node) {
for p := n.Parent; p != nil; p = p.Parent {
sel := newSingleSelection(p, nil)
if stopSelector != "" {
if sel.Is(stopSelector) {
break
}
} else if len(stopNodes) > 0 {
if sel.IsNodes(stopNodes...) {
break
}
}
if p.Type == html.ElementNode {
result = append(result, p)
}
}
return
})
}
// Internal implementation of sibling nodes that return a raw slice of matches.
func getSiblingNodes(nodes []*html.Node, st siblingType, untilSelector string, untilNodes []*html.Node) []*html.Node {
var f func(*html.Node) bool
// If the requested siblings are ...Until(), create the test function to
// determine if the until condition is reached (returns true if it is)
if st == siblingNextUntil || st == siblingPrevUntil {
f = func(n *html.Node) bool {
if untilSelector != "" {
// Selector-based condition
sel := newSingleSelection(n, nil)
return sel.Is(untilSelector)
} else if len(untilNodes) > 0 {
// Nodes-based condition
sel := newSingleSelection(n, nil)
return sel.IsNodes(untilNodes...)
}
return false
}
}
return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
return getChildrenWithSiblingType(n.Parent, st, n, f)
})
}
// Gets the children nodes of each node in the specified slice of nodes,
// based on the sibling type request.
func getChildrenNodes(nodes []*html.Node, st siblingType) []*html.Node {
return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
return getChildrenWithSiblingType(n, st, nil, nil)
})
}
// Gets the children of the specified parent, based on the requested sibling
// type, skipping a specified node if required.
func getChildrenWithSiblingType(parent *html.Node, st siblingType, skipNode *html.Node,
untilFunc func(*html.Node) bool) (result []*html.Node) {
// Create the iterator function
var iter = func(cur *html.Node) (ret *html.Node) {
// Based on the sibling type requested, iterate the right way
for {
switch st {
case siblingAll, siblingAllIncludingNonElements:
if cur == nil {
// First iteration, start with first child of parent
// Skip node if required
if ret = parent.FirstChild; ret == skipNode && skipNode != nil {
ret = skipNode.NextSibling
}
} else {
// Skip node if required
if ret = cur.NextSibling; ret == skipNode && skipNode != nil {
ret = skipNode.NextSibling
}
}
case siblingPrev, siblingPrevAll, siblingPrevUntil:
if cur == nil {
// Start with previous sibling of the skip node
ret = skipNode.PrevSibling
} else {
ret = cur.PrevSibling
}
case siblingNext, siblingNextAll, siblingNextUntil:
if cur == nil {
// Start with next sibling of the skip node
ret = skipNode.NextSibling
} else {
ret = cur.NextSibling
}
default:
panic("Invalid sibling type.")
}
if ret == nil || ret.Type == html.ElementNode || st == siblingAllIncludingNonElements {
return
} else {
// Not a valid node, try again from this one
cur = ret
}
}
panic("Unreachable code reached.")
}
for c := iter(nil); c != nil; c = iter(c) {
// If this is an ...Until() case, test before append (returns true
// if the until condition is reached)
if st == siblingNextUntil || st == siblingPrevUntil {
if untilFunc(c) {
return
}
}
result = append(result, c)
if st == siblingNext || st == siblingPrev {
// Only one node was requested (immediate next or previous), so exit
return
}
}
return
}
// Internal implementation of parent nodes that return a raw slice of Nodes.
func getParentNodes(nodes []*html.Node) []*html.Node {
return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
if n.Parent != nil && n.Parent.Type == html.ElementNode {
return []*html.Node{n.Parent}
}
return nil
})
}
// Internal map function used by many traversing methods. Takes the source nodes
// to iterate on and the mapping function that returns an array of nodes.
// Returns an array of nodes mapped by calling the callback function once for
// each node in the source nodes.
func mapNodes(nodes []*html.Node, f func(int, *html.Node) []*html.Node) (result []*html.Node) {
for i, n := range nodes {
if vals := f(i, n); len(vals) > 0 {
result = appendWithoutDuplicates(result, vals)
}
}
return
}
vendor/github.com/PuerkitoBio/goquery/type.go
Generated Vendored
+93
View File
@@ -0,0 +1,93 @@
package goquery
import (
"code.google.com/p/go.net/html"
"io"
"net/http"
"net/url"
)
// Document represents an HTML document to be manipulated. Unlike jQuery, which
// is loaded as part of a DOM document, and thus acts upon its containing
// document, GoQuery doesn't know which HTML document to act upon. So it needs
// to be told, and that's what the Document class is for. It holds the root
// document node to manipulate, and can make selections on this document.
type Document struct {
*Selection
Url *url.URL
rootNode *html.Node
}
// NewDocumentFromNode() is a Document constructor that takes a root html Node
// as argument.
func NewDocumentFromNode(root *html.Node) (d *Document) {
return newDocument(root, nil)
}
// NewDocument() is a Document constructor that takes a string URL as argument.
// It loads the specified document, parses it, and stores the root Document
// node, ready to be manipulated.
func NewDocument(url string) (d *Document, e error) {
// Load the URL
res, e := http.Get(url)
if e != nil {
return
}
return NewDocumentFromResponse(res)
}
// NewDocumentFromReader() returns a Document from a generic reader.
// It returns an error as second value if the reader's data cannot be parsed
// as html. It does *not* check if the reader is also an io.Closer, so the
// provided reader is never closed by this call, it is the responsibility
// of the caller to close it if required.
func NewDocumentFromReader(r io.Reader) (d *Document, e error) {
root, e := html.Parse(r)
if e != nil {
return nil, e
}
return newDocument(root, nil), nil
}
// NewDocumentFromResponse() is another Document constructor that takes an http resonse as argument.
// It loads the specified response's document, parses it, and stores the root Document
// node, ready to be manipulated.
func NewDocumentFromResponse(res *http.Response) (d *Document, e error) {
defer res.Body.Close()
// Parse the HTML into nodes
root, e := html.Parse(res.Body)
if e != nil {
return
}
// Create and fill the document
return newDocument(root, res.Request.URL), nil
}
// Private constructor, make sure all fields are correctly filled.
func newDocument(root *html.Node, url *url.URL) *Document {
// Create and fill the document
d := &Document{nil, url, root}
d.Selection = newSingleSelection(root, d)
return d
}
// Selection represents a collection of nodes matching some criteria. The
// initial Selection can be created by using Document.Find(), and then
// manipulated using the jQuery-like chainable syntax and methods.
type Selection struct {
Nodes []*html.Node
document *Document
prevSel *Selection
}
// Helper constructor to create an empty selection
func newEmptySelection(doc *Document) *Selection {
return &Selection{nil, doc, nil}
}
// Helper constructor to create a selection of only one node
func newSingleSelection(node *html.Node, doc *Document) *Selection {
return &Selection{[]*html.Node{node}, doc, nil}
}
vendor/github.com/PuerkitoBio/goquery/utilities.go
Generated Vendored
+87
View File
@@ -0,0 +1,87 @@
package goquery
import (
"code.google.com/p/go.net/html"
)
func getChildren(n *html.Node) (result []*html.Node) {
for c := n.FirstChild; c != nil; c = c.NextSibling {
result = append(result, c)
}
return
}
// Loop through all container nodes to search for the target node.
func sliceContains(container []*html.Node, contained *html.Node) bool {
for _, n := range container {
if nodeContains(n, contained) {
return true
}
}
return false
}
// Checks if the contained node is within the container node.
func nodeContains(container *html.Node, contained *html.Node) bool {
// Check if the parent of the contained node is the container node, traversing
// upward until the top is reached, or the container is found.
for contained = contained.Parent; contained != nil; contained = contained.Parent {
if container == contained {
return true
}
}
return false
}
// Checks if the target node is in the slice of nodes.
func isInSlice(slice []*html.Node, node *html.Node) bool {
return indexInSlice(slice, node) > -1
}
// Returns the index of the target node in the slice, or -1.
func indexInSlice(slice []*html.Node, node *html.Node) int {
if node != nil {
for i, n := range slice {
if n == node {
return i
}
}
}
return -1
}
// Appends the new nodes to the target slice, making sure no duplicate is added.
// There is no check to the original state of the target slice, so it may still
// contain duplicates. The target slice is returned because append() may create
// a new underlying array.
func appendWithoutDuplicates(target []*html.Node,
nodes []*html.Node) []*html.Node {
for _, n := range nodes {
if !isInSlice(target, n) {
target = append(target, n)
}
}
return target
}
// Loop through a selection, returning only those nodes that pass the predicate
// function.
func grep(sel *Selection, predicate func(i int, s *Selection) bool) (result []*html.Node) {
for i, n := range sel.Nodes {
if predicate(i, newSingleSelection(n, sel.document)) {
result = append(result, n)
}
}
return
}
// Creates a new Selection object based on the specified nodes, and keeps the
// source Selection object on the stack (linked list).
func pushStack(fromSel *Selection, nodes []*html.Node) (result *Selection) {
result = &Selection{nodes, fromSel.document, fromSel}
return
}
vendor/github.com/agtorre/gocolorize/LICENSE.txt
Generated Vendored
+22
View File
@@ -0,0 +1,22 @@
# This is the MIT license
# Copyright (c) 2013 Aaron G. Torres All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
vendor/github.com/agtorre/gocolorize/README.md
Generated Vendored
+135
View File
@@ -0,0 +1,135 @@
#Gocolorize
Gocolorize is a package that allows Go programs to provide ANSI coloring in a stateful manner. Gocolorize is ideal for logging or cli applications.
![colored tests passing](https://raw.github.com/agtorre/gocolorize/master/screenshot/tests.png)
[![wercker status](https://app.wercker.com/status/ee73c1abee9900c475def6ff9a142237/s "wercker status")](https://app.wercker.com/project/bykey/ee73c1abee9900c475def6ff9a142237)
##Features
- Stateful ANSI coloring
- Supports Foreground and background colors
- Supports a nuber of text properties such as bold or underline
- Color multiple arguments
- Color multiple interfaces, including complex types
- Tests with 100% coverage
- Working examples
- Disable ability for portability
##Install Gocolorize
To install:
$ go get github.com/agtorre/gocolorize
##Usage
Ways to initialize a Colorize object:
```go
//It can be done like this
var c gocolorize.Colorize
c.SetFg(gocolorize.Red)
c.SetBg(gocolorize.Black)
//Or this
c := gocolorize.Colorize{Fg: gocolorize.Red, Bg: gocolorize.Black}
//Or this
c := gocolorize.NewColor("red:black")
```
Once you have an object:
```go
//Call Paint to take inputs and return a colored string
c.Paint("This", "accepts", "multiple", "arguments", "and", "types:", 1, 1.25, "etc")
//If you want a short-hand closure
p = c.Paint
p("Neat")
//To print it:
Fmt.Println(p("test"))
//It can also be appended to other strings, used in logging to stdout, etc.
a := "test " + p("case")
//The closure allows you to reuse the original object, for example
p = c.Paint
c.SetFg(gocolorize.Green)
p2 = c.Paint
Fmt.Println(p("different" + " " + p2("colors")))
```
Object Properties:
```go
//These will only apply if there is a Fg and Bg respectively
c.ToggleFgIntensity()
c.ToggleBgIntensity()
//Set additional attributes
c.ToggleBold()
c.ToggleBlink()
c.ToggleUnderLine()
c.ToggleInverse()
//To disable or renable everything color (for example on Windows)
//the other functions will still work, they'll just return plain
//text for portability
gocolorize.SetPlain(true)
```
##NewColor String Format
```go
"foregroundColor+attributes:backgroundColor+attributes"
```
Colors:
* black
* red
* green
* yellow
* blue
* magenta
* cyan
* white
Attributes:
* b = bold foreground
* B = blink foreground
* u = underline foreground
* h = high intensity (bright) foreground, background
* i = inverse
##Examples
See examples directory for examples:
$ cd examples/
$ go run song.go
$ go run logging.go
##Tests
Tests are another good place to see examples. In order to run tests:
$ go test -cover
##Portability
ANSI coloring will not work in default Windows environments and may not work in other environments correctly. In order to allow compatibility with these environments, you can call:
```go
gocolorize.SetPlain(true)
```
Once toggled, the library will still function, but it will not color the output.
## References
Wikipedia ANSI escape codes [Colors](http://en.wikipedia.org/wiki/ANSI_escape_code#Colors)
[A stylesheet author's guide to terminal colors](http://wynnnetherland.com/journal/a-stylesheet-author-s-guide-to-terminal-colors)
## Special Thanks
https://github.com/mgutz/ansi was an inspiration for the latest version. I did a lot of rewriting, removed the 'paints' module, and did a lot of general cleanup. I learned a lot reading this and using this code.
vendor/github.com/agtorre/gocolorize/gocolorize.go
Generated Vendored
+233
View File
@@ -0,0 +1,233 @@
package gocolorize
import (
"fmt"
"strings"
)
//internal usage
var plain = false
const (
start = "\033["
reset = "\033[0m"
bold = "1;"
blink = "5;"
underline = "4;"
inverse = "7;"
normalIntensityFg = 30
highIntensityFg = 90
normalIntensityBg = 40
highIntensityBg = 100
)
//The rest is external facing
//Color can be used for Fg and Bg
type Color int
const (
ColorNone = iota
//placeholder here so we don't confuse with ColorNone
Red
Green
Yellow
Blue
Magenta
Cyan
White
Black Color = -1
)
var colors = map[string]Color{
"red": Red,
"green": Green,
"yellow": Yellow,
"blue": Blue,
"magenta": Magenta,
"cyan": Cyan,
"white": White,
"black": Black,
}
//Can set 1 or more of these properties
//This struct holds the state
type Property struct {
Bold bool
Blink bool
Underline bool
Inverse bool
Fgi bool
Bgi bool
}
//Where the magic happens
type Colorize struct {
Value []interface{}
Fg Color
Bg Color
Prop Property
}
//returns a value you can stick into print, of type string
func (c Colorize) Paint(v ...interface{}) string {
c.Value = v
return fmt.Sprint(c)
}
func propsString(p Property) string {
var result string
if p.Bold {
result += bold
}
if p.Blink {
result += blink
}
if p.Underline {
result += underline
}
if p.Inverse {
result += inverse
}
return result
}
// Format allows ColorText to satisfy the fmt.Formatter interface. The format
// behaviour is the same as for fmt.Print.
func (ct Colorize) Format(fs fmt.State, c rune) {
var base int
//fmt.Println(ct.Fg, ct.Fgi, ct.Bg, ct.Bgi, ct.Prop)
//First Handle the Fg styles and options
if ct.Fg != ColorNone && !plain {
if ct.Prop.Fgi {
base = int(highIntensityFg)
} else {
base = int(normalIntensityFg)
}
if ct.Fg == Black {
base = base
} else {
base = base + int(ct.Fg)
}
fmt.Fprint(fs, start, "0;", propsString(ct.Prop), base, "m")
}
//Next Handle the Bg styles and options
if ct.Bg != ColorNone && !plain {
if ct.Prop.Bgi {
base = int(highIntensityBg)
} else {
base = int(normalIntensityBg)
}
if ct.Bg == Black {
base = base
} else {
base = base + int(ct.Bg)
}
//We still want to honor props if only the background is set
if ct.Fg == ColorNone {
fmt.Fprint(fs, start, propsString(ct.Prop), base, "m")
//fmt.Fprint(fs, start, base, "m")
} else {
fmt.Fprint(fs, start, base, "m")
}
}
// I simplified this to be a bit less efficient,
// but more robust, it will work with anything that
// printf("%v") will support
for i, v := range ct.Value {
fmt.Fprintf(fs, fmt.Sprint(v))
if i < len(ct.Value)-1 {
fmt.Fprintf(fs, " ")
}
}
//after we finish go back to a clean state
if !plain {
fmt.Fprint(fs, reset)
}
}
func NewColor(style string) Colorize {
//Thank you https://github.com/mgutz/ansi for
//this code example and for a bunch of other ideas
foreground_background := strings.Split(style, ":")
foreground := strings.Split(foreground_background[0], "+")
fg := colors[foreground[0]]
fgStyle := ""
if len(foreground) > 1 {
fgStyle = foreground[1]
}
var bg Color
bgStyle := ""
if len(foreground_background) > 1 {
background := strings.Split(foreground_background[1], "+")
bg = colors[background[0]]
if len(background) > 1 {
bgStyle = background[1]
}
}
c := Colorize{Fg: fg, Bg: bg}
if len(fgStyle) > 0 {
if strings.Contains(fgStyle, "b") {
c.ToggleBold()
}
if strings.Contains(fgStyle, "B") {
c.ToggleBlink()
}
if strings.Contains(fgStyle, "u") {
c.ToggleUnderline()
}
if strings.Contains(fgStyle, "i") {
c.ToggleInverse()
}
if strings.Contains(fgStyle, "h") {
c.ToggleFgIntensity()
}
}
if len(bgStyle) > 0 {
if strings.Contains(bgStyle, "h") {
c.ToggleBgIntensity()
}
}
return c
}
func (C *Colorize) SetColor(c Color) {
C.Fg = c
}
func (C *Colorize) SetBgColor(b Color) {
C.Bg = b
}
func (C *Colorize) ToggleFgIntensity() {
C.Prop.Fgi = !C.Prop.Fgi
}
func (C *Colorize) ToggleBgIntensity() {
C.Prop.Bgi = !C.Prop.Bgi
}
func (C *Colorize) ToggleBold() {
C.Prop.Bold = !C.Prop.Bold
}
func (C *Colorize) ToggleBlink() {
C.Prop.Blink = !C.Prop.Blink
}
func (C *Colorize) ToggleUnderline() {
C.Prop.Underline = !C.Prop.Underline
}
func (C *Colorize) ToggleInverse() {
C.Prop.Inverse = !C.Prop.Inverse
}
func SetPlain(p bool) {
plain = p
}
vendor/github.com/agtorre/gocolorize/wercker.yml
Generated Vendored
+46
View File
@@ -0,0 +1,46 @@
# This references the default golang container from
# the Docker Hub: https://registry.hub.docker.com/u/library/golang/
# If you want Google's container you would reference google/golang
# Read more about containers on our dev center
# http://devcenter.wercker.com/docs/containers/index.html
box: golang
# This is the build pipeline. Pipelines are the core of wercker
# Read more about pipelines on our dev center
# http://devcenter.wercker.com/docs/pipelines/index.html
# You can also use services such as databases. Read more on our dev center:
# http://devcenter.wercker.com/docs/services/index.html
# services:
# - postgres
# http://devcenter.wercker.com/docs/services/postgresql.html
# - mongo
# http://devcenter.wercker.com/docs/services/mongodb.html
build:
# The steps that will be executed on build
# Steps make up the actions in your pipeline
# Read more about steps on our dev center:
# http://devcenter.wercker.com/docs/steps/index.html
steps:
# Sets the go workspace and places you package
# at the right place in the workspace tree
- setup-go-workspace
# Gets the dependencies
- script:
name: go get
code: |
go get
# Build the project
- script:
name: go build
code: |
go build ./...
# Test the project
- script:
name: go test
code: |
go test ./...
vendor/github.com/klauspost/compress/LICENSE
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/klauspost/compress/flate/copy.go
Generated Vendored
+32
View File
@@ -0,0 +1,32 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// forwardCopy is like the built-in copy function except that it always goes
// forward from the start, even if the dst and src overlap.
// It is equivalent to:
// for i := 0; i < n; i++ {
// mem[dst+i] = mem[src+i]
// }
func forwardCopy(mem []byte, dst, src, n int) {
if dst <= src {
copy(mem[dst:dst+n], mem[src:src+n])
return
}
for {
if dst >= src+n {
copy(mem[dst:dst+n], mem[src:src+n])
return
}
// There is some forward overlap. The destination
// will be filled with a repeated pattern of mem[src:src+k].
// We copy one instance of the pattern here, then repeat.
// Each time around this loop k will double.
k := dst - src
copy(mem[dst:dst+k], mem[src:src+k])
n -= k
dst += k
}
}
vendor/github.com/klauspost/compress/flate/crc32_amd64.go
Generated Vendored
+41
View File
@@ -0,0 +1,41 @@
//+build !noasm
//+build !appengine
// Copyright 2015, Klaus Post, see LICENSE for details.
package flate
import (
"github.com/klauspost/cpuid"
)
// crc32sse returns a hash for the first 4 bytes of the slice
// len(a) must be >= 4.
//go:noescape
func crc32sse(a []byte) uint32
// crc32sseAll calculates hashes for each 4-byte set in a.
// dst must be east len(a) - 4 in size.
// The size is not checked by the assembly.
//go:noescape
func crc32sseAll(a []byte, dst []uint32)
// matchLenSSE4 returns the number of matching bytes in a and b
// up to length 'max'. Both slices must be at least 'max'
// bytes in size.
//
// TODO: drop the "SSE4" name, since it doesn't use any SSE instructions.
//
//go:noescape
func matchLenSSE4(a, b []byte, max int) int
// histogram accumulates a histogram of b in h.
// h must be at least 256 entries in length,
// and must be cleared before calling this function.
//go:noescape
func histogram(b []byte, h []int32)
// Detect SSE 4.2 feature.
func init() {
useSSE42 = cpuid.CPU.SSE42()
}
vendor/github.com/klauspost/compress/flate/crc32_amd64.s
Generated Vendored
+213
View File
@@ -0,0 +1,213 @@
//+build !noasm
//+build !appengine
// Copyright 2015, Klaus Post, see LICENSE for details.
// func crc32sse(a []byte) uint32
TEXT ·crc32sse(SB), 4, $0
MOVQ a+0(FP), R10
XORQ BX, BX
// CRC32 dword (R10), EBX
BYTE $0xF2; BYTE $0x41; BYTE $0x0f
BYTE $0x38; BYTE $0xf1; BYTE $0x1a
MOVL BX, ret+24(FP)
RET
// func crc32sseAll(a []byte, dst []uint32)
TEXT ·crc32sseAll(SB), 4, $0
MOVQ a+0(FP), R8 // R8: src
MOVQ a_len+8(FP), R10 // input length
MOVQ dst+24(FP), R9 // R9: dst
SUBQ $4, R10
JS end
JZ one_crc
MOVQ R10, R13
SHRQ $2, R10 // len/4
ANDQ $3, R13 // len&3
XORQ BX, BX
ADDQ $1, R13
TESTQ R10, R10
JZ rem_loop
crc_loop:
MOVQ (R8), R11
XORQ BX, BX
XORQ DX, DX
XORQ DI, DI
MOVQ R11, R12
SHRQ $8, R11
MOVQ R12, AX
MOVQ R11, CX
SHRQ $16, R12
SHRQ $16, R11
MOVQ R12, SI
// CRC32 EAX, EBX
BYTE $0xF2; BYTE $0x0f
BYTE $0x38; BYTE $0xf1; BYTE $0xd8
// CRC32 ECX, EDX
BYTE $0xF2; BYTE $0x0f
BYTE $0x38; BYTE $0xf1; BYTE $0xd1
// CRC32 ESI, EDI
BYTE $0xF2; BYTE $0x0f
BYTE $0x38; BYTE $0xf1; BYTE $0xfe
MOVL BX, (R9)
MOVL DX, 4(R9)
MOVL DI, 8(R9)
XORQ BX, BX
MOVL R11, AX
// CRC32 EAX, EBX
BYTE $0xF2; BYTE $0x0f
BYTE $0x38; BYTE $0xf1; BYTE $0xd8
MOVL BX, 12(R9)
ADDQ $16, R9
ADDQ $4, R8
XORQ BX, BX
SUBQ $1, R10
JNZ crc_loop
rem_loop:
MOVL (R8), AX
// CRC32 EAX, EBX
BYTE $0xF2; BYTE $0x0f
BYTE $0x38; BYTE $0xf1; BYTE $0xd8
MOVL BX, (R9)
ADDQ $4, R9
ADDQ $1, R8
XORQ BX, BX
SUBQ $1, R13
JNZ rem_loop
end:
RET
one_crc:
MOVQ $1, R13
XORQ BX, BX
JMP rem_loop
// func matchLenSSE4(a, b []byte, max int) int
TEXT ·matchLenSSE4(SB), 4, $0
MOVQ a_base+0(FP), SI
MOVQ b_base+24(FP), DI
MOVQ DI, DX
MOVQ max+48(FP), CX
cmp8:
// As long as we are 8 or more bytes before the end of max, we can load and
// compare 8 bytes at a time. If those 8 bytes are equal, repeat.
CMPQ CX, $8
JLT cmp1
MOVQ (SI), AX
MOVQ (DI), BX
CMPQ AX, BX
JNE bsf
ADDQ $8, SI
ADDQ $8, DI
SUBQ $8, CX
JMP cmp8
bsf:
// If those 8 bytes were not equal, XOR the two 8 byte values, and return
// the index of the first byte that differs. The BSF instruction finds the
// least significant 1 bit, the amd64 architecture is little-endian, and
// the shift by 3 converts a bit index to a byte index.
XORQ AX, BX
BSFQ BX, BX
SHRQ $3, BX
ADDQ BX, DI
// Subtract off &b[0] to convert from &b[ret] to ret, and return.
SUBQ DX, DI
MOVQ DI, ret+56(FP)
RET
cmp1:
// In the slices' tail, compare 1 byte at a time.
CMPQ CX, $0
JEQ matchLenEnd
MOVB (SI), AX
MOVB (DI), BX
CMPB AX, BX
JNE matchLenEnd
ADDQ $1, SI
ADDQ $1, DI
SUBQ $1, CX
JMP cmp1
matchLenEnd:
// Subtract off &b[0] to convert from &b[ret] to ret, and return.
SUBQ DX, DI
MOVQ DI, ret+56(FP)
RET
// func histogram(b []byte, h []int32)
TEXT ·histogram(SB), 4, $0
MOVQ b+0(FP), SI // SI: &b
MOVQ b_len+8(FP), R9 // R9: len(b)
MOVQ h+24(FP), DI // DI: Histogram
MOVQ R9, R8
SHRQ $3, R8
JZ hist1
XORQ R11, R11
loop_hist8:
MOVQ (SI), R10
MOVB R10, R11
INCL (DI)(R11*4)
SHRQ $8, R10
MOVB R10, R11
INCL (DI)(R11*4)
SHRQ $8, R10
MOVB R10, R11
INCL (DI)(R11*4)
SHRQ $8, R10
MOVB R10, R11
INCL (DI)(R11*4)
SHRQ $8, R10
MOVB R10, R11
INCL (DI)(R11*4)
SHRQ $8, R10
MOVB R10, R11
INCL (DI)(R11*4)
SHRQ $8, R10
MOVB R10, R11
INCL (DI)(R11*4)
SHRQ $8, R10
INCL (DI)(R10*4)
ADDQ $8, SI
DECQ R8
JNZ loop_hist8
hist1:
ANDQ $7, R9
JZ end_hist
XORQ R10, R10
loop_hist1:
MOVB (SI), R10
INCL (DI)(R10*4)
INCQ SI
DECQ R9
JNZ loop_hist1
end_hist:
RET
vendor/github.com/klauspost/compress/flate/crc32_noasm.go
Generated Vendored
+35
View File
@@ -0,0 +1,35 @@
//+build !amd64 noasm appengine
// Copyright 2015, Klaus Post, see LICENSE for details.
package flate
func init() {
useSSE42 = false
}
// crc32sse should never be called.
func crc32sse(a []byte) uint32 {
panic("no assembler")
}
// crc32sseAll should never be called.
func crc32sseAll(a []byte, dst []uint32) {
panic("no assembler")
}
// matchLenSSE4 should never be called.
func matchLenSSE4(a, b []byte, max int) int {
panic("no assembler")
return 0
}
// histogram accumulates a histogram of b in h.
//
// len(h) must be >= 256, and h's elements must be all zeroes.
func histogram(b []byte, h []int32) {
h = h[:256]
for _, t := range b {
h[t]++
}
}
vendor/github.com/klauspost/compress/flate/deflate.go
Generated Vendored
+1353
View File
File diff suppressed because it is too large. Load diff
vendor/github.com/klauspost/compress/flate/dict_decoder.go
Generated Vendored
+184
View File
@@ -0,0 +1,184 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// dictDecoder implements the LZ77 sliding dictionary as used in decompression.
// LZ77 decompresses data through sequences of two forms of commands:
//
// * Literal insertions: Runs of one or more symbols are inserted into the data
// stream as is. This is accomplished through the writeByte method for a
// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
// Any valid stream must start with a literal insertion if no preset dictionary
// is used.
//
// * Backward copies: Runs of one or more symbols are copied from previously
// emitted data. Backward copies come as the tuple (dist, length) where dist
// determines how far back in the stream to copy from and length determines how
// many bytes to copy. Note that it is valid for the length to be greater than
// the distance. Since LZ77 uses forward copies, that situation is used to
// perform a form of run-length encoding on repeated runs of symbols.
// The writeCopy and tryWriteCopy are used to implement this command.
//
// For performance reasons, this implementation performs little to no sanity
// checks about the arguments. As such, the invariants documented for each
// method call must be respected.
type dictDecoder struct {
hist []byte // Sliding window history
// Invariant: 0 <= rdPos <= wrPos <= len(hist)
wrPos int // Current output position in buffer
rdPos int // Have emitted hist[:rdPos] already
full bool // Has a full window length been written yet?
}
// init initializes dictDecoder to have a sliding window dictionary of the given
// size. If a preset dict is provided, it will initialize the dictionary with
// the contents of dict.
func (dd *dictDecoder) init(size int, dict []byte) {
*dd = dictDecoder{hist: dd.hist}
if cap(dd.hist) < size {
dd.hist = make([]byte, size)
}
dd.hist = dd.hist[:size]
if len(dict) > len(dd.hist) {
dict = dict[len(dict)-len(dd.hist):]
}
dd.wrPos = copy(dd.hist, dict)
if dd.wrPos == len(dd.hist) {
dd.wrPos = 0
dd.full = true
}
dd.rdPos = dd.wrPos
}
// histSize reports the total amount of historical data in the dictionary.
func (dd *dictDecoder) histSize() int {
if dd.full {
return len(dd.hist)
}
return dd.wrPos
}
// availRead reports the number of bytes that can be flushed by readFlush.
func (dd *dictDecoder) availRead() int {
return dd.wrPos - dd.rdPos
}
// availWrite reports the available amount of output buffer space.
func (dd *dictDecoder) availWrite() int {
return len(dd.hist) - dd.wrPos
}
// writeSlice returns a slice of the available buffer to write data to.
//
// This invariant will be kept: len(s) <= availWrite()
func (dd *dictDecoder) writeSlice() []byte {
return dd.hist[dd.wrPos:]
}
// writeMark advances the writer pointer by cnt.
//
// This invariant must be kept: 0 <= cnt <= availWrite()
func (dd *dictDecoder) writeMark(cnt int) {
dd.wrPos += cnt
}
// writeByte writes a single byte to the dictionary.
//
// This invariant must be kept: 0 < availWrite()
func (dd *dictDecoder) writeByte(c byte) {
dd.hist[dd.wrPos] = c
dd.wrPos++
}
// writeCopy copies a string at a given (dist, length) to the output.
// This returns the number of bytes copied and may be less than the requested
// length if the available space in the output buffer is too small.
//
// This invariant must be kept: 0 < dist <= histSize()
func (dd *dictDecoder) writeCopy(dist, length int) int {
dstBase := dd.wrPos
dstPos := dstBase
srcPos := dstPos - dist
endPos := dstPos + length
if endPos > len(dd.hist) {
endPos = len(dd.hist)
}
// Copy non-overlapping section after destination position.
//
// This section is non-overlapping in that the copy length for this section
// is always less than or equal to the backwards distance. This can occur
// if a distance refers to data that wraps-around in the buffer.
// Thus, a backwards copy is performed here; that is, the exact bytes in
// the source prior to the copy is placed in the destination.
if srcPos < 0 {
srcPos += len(dd.hist)
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:])
srcPos = 0
}
// Copy possibly overlapping section before destination position.
//
// This section can overlap if the copy length for this section is larger
// than the backwards distance. This is allowed by LZ77 so that repeated
// strings can be succinctly represented using (dist, length) pairs.
// Thus, a forwards copy is performed here; that is, the bytes copied is
// possibly dependent on the resulting bytes in the destination as the copy
// progresses along. This is functionally equivalent to the following:
//
// for i := 0; i < endPos-dstPos; i++ {
// dd.hist[dstPos+i] = dd.hist[srcPos+i]
// }
// dstPos = endPos
//
for dstPos < endPos {
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
}
dd.wrPos = dstPos
return dstPos - dstBase
}
// tryWriteCopy tries to copy a string at a given (distance, length) to the
// output. This specialized version is optimized for short distances.
//
// This method is designed to be inlined for performance reasons.
//
// This invariant must be kept: 0 < dist <= histSize()
func (dd *dictDecoder) tryWriteCopy(dist, length int) int {
dstPos := dd.wrPos
endPos := dstPos + length
if dstPos < dist || endPos > len(dd.hist) {
return 0
}
dstBase := dstPos
srcPos := dstPos - dist
// Copy possibly overlapping section before destination position.
loop:
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
if dstPos < endPos {
goto loop // Avoid for-loop so that this function can be inlined
}
dd.wrPos = dstPos
return dstPos - dstBase
}
// readFlush returns a slice of the historical buffer that is ready to be
// emitted to the user. The data returned by readFlush must be fully consumed
// before calling any other dictDecoder methods.
func (dd *dictDecoder) readFlush() []byte {
toRead := dd.hist[dd.rdPos:dd.wrPos]
dd.rdPos = dd.wrPos
if dd.wrPos == len(dd.hist) {
dd.wrPos, dd.rdPos = 0, 0
dd.full = true
}
return toRead
}
vendor/github.com/klauspost/compress/flate/gen.go
Generated Vendored
+265
View File
@@ -0,0 +1,265 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
// This program generates fixedhuff.go
// Invoke as
//
// go run gen.go -output fixedhuff.go
package main
import (
"bytes"
"flag"
"fmt"
"go/format"
"io/ioutil"
"log"
)
var filename = flag.String("output", "fixedhuff.go", "output file name")
const maxCodeLen = 16
// Note: the definition of the huffmanDecoder struct is copied from
// inflate.go, as it is private to the implementation.
// chunk & 15 is number of bits
// chunk >> 4 is value, including table link
const (
huffmanChunkBits = 9
huffmanNumChunks = 1 << huffmanChunkBits
huffmanCountMask = 15
huffmanValueShift = 4
)
type huffmanDecoder struct {
min int // the minimum code length
chunks [huffmanNumChunks]uint32 // chunks as described above
links [][]uint32 // overflow links
linkMask uint32 // mask the width of the link table
}
// Initialize Huffman decoding tables from array of code lengths.
// Following this function, h is guaranteed to be initialized into a complete
// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
// degenerate case where the tree has only a single symbol with length 1. Empty
// trees are permitted.
func (h *huffmanDecoder) init(bits []int) bool {
// Sanity enables additional runtime tests during Huffman
// table construction. It's intended to be used during
// development to supplement the currently ad-hoc unit tests.
const sanity = false
if h.min != 0 {
*h = huffmanDecoder{}
}
// Count number of codes of each length,
// compute min and max length.
var count [maxCodeLen]int
var min, max int
for _, n := range bits {
if n == 0 {
continue
}
if min == 0 || n < min {
min = n
}
if n > max {
max = n
}
count[n]++
}
// Empty tree. The decompressor.huffSym function will fail later if the tree
// is used. Technically, an empty tree is only valid for the HDIST tree and
// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
// is guaranteed to fail since it will attempt to use the tree to decode the
// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
// guaranteed to fail later since the compressed data section must be
// composed of at least one symbol (the end-of-block marker).
if max == 0 {
return true
}
code := 0
var nextcode [maxCodeLen]int
for i := min; i <= max; i++ {
code <<= 1
nextcode[i] = code
code += count[i]
}
// Check that the coding is complete (i.e., that we've
// assigned all 2-to-the-max possible bit sequences).
// Exception: To be compatible with zlib, we also need to
// accept degenerate single-code codings. See also
// TestDegenerateHuffmanCoding.
if code != 1<<uint(max) && !(code == 1 && max == 1) {
return false
}
h.min = min
if max > huffmanChunkBits {
numLinks := 1 << (uint(max) - huffmanChunkBits)
h.linkMask = uint32(numLinks - 1)
// create link tables
link := nextcode[huffmanChunkBits+1] >> 1
h.links = make([][]uint32, huffmanNumChunks-link)
for j := uint(link); j < huffmanNumChunks; j++ {
reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8
reverse >>= uint(16 - huffmanChunkBits)
off := j - uint(link)
if sanity && h.chunks[reverse] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[reverse] = uint32(off<<huffmanValueShift | (huffmanChunkBits + 1))
h.links[off] = make([]uint32, numLinks)
}
}
for i, n := range bits {
if n == 0 {
continue
}
code := nextcode[n]
nextcode[n]++
chunk := uint32(i<<huffmanValueShift | n)
reverse := int(reverseByte[code>>8]) | int(reverseByte[code&0xff])<<8
reverse >>= uint(16 - n)
if n <= huffmanChunkBits {
for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
// We should never need to overwrite
// an existing chunk. Also, 0 is
// never a valid chunk, because the
// lower 4 "count" bits should be
// between 1 and 15.
if sanity && h.chunks[off] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[off] = chunk
}
} else {
j := reverse & (huffmanNumChunks - 1)
if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
// Longer codes should have been
// associated with a link table above.
panic("impossible: not an indirect chunk")
}
value := h.chunks[j] >> huffmanValueShift
linktab := h.links[value]
reverse >>= huffmanChunkBits
for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
if sanity && linktab[off] != 0 {
panic("impossible: overwriting existing chunk")
}
linktab[off] = chunk
}
}
}
if sanity {
// Above we've sanity checked that we never overwrote
// an existing entry. Here we additionally check that
// we filled the tables completely.
for i, chunk := range h.chunks {
if chunk == 0 {
// As an exception, in the degenerate
// single-code case, we allow odd
// chunks to be missing.
if code == 1 && i%2 == 1 {
continue
}
panic("impossible: missing chunk")
}
}
for _, linktab := range h.links {
for _, chunk := range linktab {
if chunk == 0 {
panic("impossible: missing chunk")
}
}
}
}
return true
}
func main() {
flag.Parse()
var h huffmanDecoder
var bits [288]int
initReverseByte()
for i := 0; i < 144; i++ {
bits[i] = 8
}
for i := 144; i < 256; i++ {
bits[i] = 9
}
for i := 256; i < 280; i++ {
bits[i] = 7
}
for i := 280; i < 288; i++ {
bits[i] = 8
}
h.init(bits[:])
if h.links != nil {
log.Fatal("Unexpected links table in fixed Huffman decoder")
}
var buf bytes.Buffer
fmt.Fprintf(&buf, `// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.`+"\n\n")
fmt.Fprintln(&buf, "package flate")
fmt.Fprintln(&buf)
fmt.Fprintln(&buf, "// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT")
fmt.Fprintln(&buf)
fmt.Fprintln(&buf, "var fixedHuffmanDecoder = huffmanDecoder{")
fmt.Fprintf(&buf, "\t%d,\n", h.min)
fmt.Fprintln(&buf, "\t[huffmanNumChunks]uint32{")
for i := 0; i < huffmanNumChunks; i++ {
if i&7 == 0 {
fmt.Fprintf(&buf, "\t\t")
} else {
fmt.Fprintf(&buf, " ")
}
fmt.Fprintf(&buf, "0x%04x,", h.chunks[i])
if i&7 == 7 {
fmt.Fprintln(&buf)
}
}
fmt.Fprintln(&buf, "\t},")
fmt.Fprintln(&buf, "\tnil, 0,")
fmt.Fprintln(&buf, "}")
data, err := format.Source(buf.Bytes())
if err != nil {
log.Fatal(err)
}
err = ioutil.WriteFile(*filename, data, 0644)
if err != nil {
log.Fatal(err)
}
}
var reverseByte [256]byte
func initReverseByte() {
for x := 0; x < 256; x++ {
var result byte
for i := uint(0); i < 8; i++ {
result |= byte(((x >> i) & 1) << (7 - i))
}
reverseByte[x] = result
}
}
vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go
Generated Vendored
+701
View File
@@ -0,0 +1,701 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"io"
)
const (
// The largest offset code.
offsetCodeCount = 30
// The special code used to mark the end of a block.
endBlockMarker = 256
// The first length code.
lengthCodesStart = 257
// The number of codegen codes.
codegenCodeCount = 19
badCode = 255
// bufferFlushSize indicates the buffer size
// after which bytes are flushed to the writer.
// Should preferably be a multiple of 6, since
// we accumulate 6 bytes between writes to the buffer.
bufferFlushSize = 240
// bufferSize is the actual output byte buffer size.
// It must have additional headroom for a flush
// which can contain up to 8 bytes.
bufferSize = bufferFlushSize + 8
)
// The number of extra bits needed by length code X - LENGTH_CODES_START.
var lengthExtraBits = []int8{
/* 257 */ 0, 0, 0,
/* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2,
/* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
/* 280 */ 4, 5, 5, 5, 5, 0,
}
// The length indicated by length code X - LENGTH_CODES_START.
var lengthBase = []uint32{
0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
64, 80, 96, 112, 128, 160, 192, 224, 255,
}
// offset code word extra bits.
var offsetExtraBits = []int8{
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
/* extended window */
14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20,
}
var offsetBase = []uint32{
/* normal deflate */
0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
/* extended window */
0x008000, 0x00c000, 0x010000, 0x018000, 0x020000,
0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000,
0x100000, 0x180000, 0x200000, 0x300000,
}
// The odd order in which the codegen code sizes are written.
var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
type huffmanBitWriter struct {
// writer is the underlying writer.
// Do not use it directly; use the write method, which ensures
// that Write errors are sticky.
writer io.Writer
// Data waiting to be written is bytes[0:nbytes]
// and then the low nbits of bits.
bits uint64
nbits uint
bytes [bufferSize]byte
codegenFreq [codegenCodeCount]int32
nbytes int
literalFreq []int32
offsetFreq []int32
codegen []uint8
literalEncoding *huffmanEncoder
offsetEncoding *huffmanEncoder
codegenEncoding *huffmanEncoder
err error
}
func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
return &huffmanBitWriter{
writer: w,
literalFreq: make([]int32, maxNumLit),
offsetFreq: make([]int32, offsetCodeCount),
codegen: make([]uint8, maxNumLit+offsetCodeCount+1),
literalEncoding: newHuffmanEncoder(maxNumLit),
codegenEncoding: newHuffmanEncoder(codegenCodeCount),
offsetEncoding: newHuffmanEncoder(offsetCodeCount),
}
}
func (w *huffmanBitWriter) reset(writer io.Writer) {
w.writer = writer
w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
w.bytes = [bufferSize]byte{}
}
func (w *huffmanBitWriter) flush() {
if w.err != nil {
w.nbits = 0
return
}
n := w.nbytes
for w.nbits != 0 {
w.bytes[n] = byte(w.bits)
w.bits >>= 8
if w.nbits > 8 { // Avoid underflow
w.nbits -= 8
} else {
w.nbits = 0
}
n++
}
w.bits = 0
w.write(w.bytes[:n])
w.nbytes = 0
}
func (w *huffmanBitWriter) write(b []byte) {
if w.err != nil {
return
}
_, w.err = w.writer.Write(b)
}
func (w *huffmanBitWriter) writeBits(b int32, nb uint) {
if w.err != nil {
return
}
w.bits |= uint64(b) << w.nbits
w.nbits += nb
if w.nbits >= 48 {
bits := w.bits
w.bits >>= 48
w.nbits -= 48
n := w.nbytes
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n >= bufferFlushSize {
w.write(w.bytes[:n])
n = 0
}
w.nbytes = n
}
}
func (w *huffmanBitWriter) writeBytes(bytes []byte) {
if w.err != nil {
return
}
n := w.nbytes
if w.nbits&7 != 0 {
w.err = InternalError("writeBytes with unfinished bits")
return
}
for w.nbits != 0 {
w.bytes[n] = byte(w.bits)
w.bits >>= 8
w.nbits -= 8
n++
}
if n != 0 {
w.write(w.bytes[:n])
}
w.nbytes = 0
w.write(bytes)
}
// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
// the literal and offset lengths arrays (which are concatenated into a single
// array). This method generates that run-length encoding.
//
// The result is written into the codegen array, and the frequencies
// of each code is written into the codegenFreq array.
// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
// information. Code badCode is an end marker
//
// numLiterals The number of literals in literalEncoding
// numOffsets The number of offsets in offsetEncoding
// litenc, offenc The literal and offset encoder to use
func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) {
for i := range w.codegenFreq {
w.codegenFreq[i] = 0
}
// Note that we are using codegen both as a temporary variable for holding
// a copy of the frequencies, and as the place where we put the result.
// This is fine because the output is always shorter than the input used
// so far.
codegen := w.codegen // cache
// Copy the concatenated code sizes to codegen. Put a marker at the end.
cgnl := codegen[:numLiterals]
for i := range cgnl {
cgnl[i] = uint8(litEnc.codes[i].len)
}
cgnl = codegen[numLiterals : numLiterals+numOffsets]
for i := range cgnl {
cgnl[i] = uint8(offEnc.codes[i].len)
}
codegen[numLiterals+numOffsets] = badCode
size := codegen[0]
count := 1
outIndex := 0
for inIndex := 1; size != badCode; inIndex++ {
// INVARIANT: We have seen "count" copies of size that have not yet
// had output generated for them.
nextSize := codegen[inIndex]
if nextSize == size {
count++
continue
}
// We need to generate codegen indicating "count" of size.
if size != 0 {
codegen[outIndex] = size
outIndex++
w.codegenFreq[size]++
count--
for count >= 3 {
n := 6
if n > count {
n = count
}
codegen[outIndex] = 16
outIndex++
codegen[outIndex] = uint8(n - 3)
outIndex++
w.codegenFreq[16]++
count -= n
}
} else {
for count >= 11 {
n := 138
if n > count {
n = count
}
codegen[outIndex] = 18
outIndex++
codegen[outIndex] = uint8(n - 11)
outIndex++
w.codegenFreq[18]++
count -= n
}
if count >= 3 {
// count >= 3 && count <= 10
codegen[outIndex] = 17
outIndex++
codegen[outIndex] = uint8(count - 3)
outIndex++
w.codegenFreq[17]++
count = 0
}
}
count--
for ; count >= 0; count-- {
codegen[outIndex] = size
outIndex++
w.codegenFreq[size]++
}
// Set up invariant for next time through the loop.
size = nextSize
count = 1
}
// Marker indicating the end of the codegen.
codegen[outIndex] = badCode
}
// dynamicSize returns the size of dynamically encoded data in bits.
func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) {
numCodegens = len(w.codegenFreq)
for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
numCodegens--
}
header := 3 + 5 + 5 + 4 + (3 * numCodegens) +
w.codegenEncoding.bitLength(w.codegenFreq[:]) +
int(w.codegenFreq[16])*2 +
int(w.codegenFreq[17])*3 +
int(w.codegenFreq[18])*7
size = header +
litEnc.bitLength(w.literalFreq) +
offEnc.bitLength(w.offsetFreq) +
extraBits
return size, numCodegens
}
// fixedSize returns the size of dynamically encoded data in bits.
func (w *huffmanBitWriter) fixedSize(extraBits int) int {
return 3 +
fixedLiteralEncoding.bitLength(w.literalFreq) +
fixedOffsetEncoding.bitLength(w.offsetFreq) +
extraBits
}
// storedSize calculates the stored size, including header.
// The function returns the size in bits and whether the block
// fits inside a single block.
func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
if in == nil {
return 0, false
}
if len(in) <= maxStoreBlockSize {
return (len(in) + 5) * 8, true
}
return 0, false
}
func (w *huffmanBitWriter) writeCode(c hcode) {
if w.err != nil {
return
}
w.bits |= uint64(c.code) << w.nbits
w.nbits += uint(c.len)
if w.nbits >= 48 {
bits := w.bits
w.bits >>= 48
w.nbits -= 48
n := w.nbytes
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n >= bufferFlushSize {
w.write(w.bytes[:n])
n = 0
}
w.nbytes = n
}
}
// Write the header of a dynamic Huffman block to the output stream.
//
// numLiterals The number of literals specified in codegen
// numOffsets The number of offsets specified in codegen
// numCodegens The number of codegens used in codegen
func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) {
if w.err != nil {
return
}
var firstBits int32 = 4
if isEof {
firstBits = 5
}
w.writeBits(firstBits, 3)
w.writeBits(int32(numLiterals-257), 5)
w.writeBits(int32(numOffsets-1), 5)
w.writeBits(int32(numCodegens-4), 4)
for i := 0; i < numCodegens; i++ {
value := uint(w.codegenEncoding.codes[codegenOrder[i]].len)
w.writeBits(int32(value), 3)
}
i := 0
for {
var codeWord int = int(w.codegen[i])
i++
if codeWord == badCode {
break
}
w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
switch codeWord {
case 16:
w.writeBits(int32(w.codegen[i]), 2)
i++
break
case 17:
w.writeBits(int32(w.codegen[i]), 3)
i++
break
case 18:
w.writeBits(int32(w.codegen[i]), 7)
i++
break
}
}
}
func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
if w.err != nil {
return
}
var flag int32
if isEof {
flag = 1
}
w.writeBits(flag, 3)
w.flush()
w.writeBits(int32(length), 16)
w.writeBits(int32(^uint16(length)), 16)
}
func (w *huffmanBitWriter) writeFixedHeader(isEof bool) {
if w.err != nil {
return
}
// Indicate that we are a fixed Huffman block
var value int32 = 2
if isEof {
value = 3
}
w.writeBits(value, 3)
}
// writeBlock will write a block of tokens with the smallest encoding.
// The original input can be supplied, and if the huffman encoded data
// is larger than the original bytes, the data will be written as a
// stored block.
// If the input is nil, the tokens will always be Huffman encoded.
func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
if w.err != nil {
return
}
tokens = append(tokens, endBlockMarker)
numLiterals, numOffsets := w.indexTokens(tokens)
var extraBits int
storedSize, storable := w.storedSize(input)
if storable {
// We only bother calculating the costs of the extra bits required by
// the length of offset fields (which will be the same for both fixed
// and dynamic encoding), if we need to compare those two encodings
// against stored encoding.
for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ {
// First eight length codes have extra size = 0.
extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart])
}
for offsetCode := 4; offsetCode < numOffsets; offsetCode++ {
// First four offset codes have extra size = 0.
extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode])
}
}
// Figure out smallest code.
// Fixed Huffman baseline.
var literalEncoding = fixedLiteralEncoding
var offsetEncoding = fixedOffsetEncoding
var size = w.fixedSize(extraBits)
// Dynamic Huffman?
var numCodegens int
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
if dynamicSize < size {
size = dynamicSize
literalEncoding = w.literalEncoding
offsetEncoding = w.offsetEncoding
}
// Stored bytes?
if storable && storedSize < size {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Huffman.
if literalEncoding == fixedLiteralEncoding {
w.writeFixedHeader(eof)
} else {
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
}
// Write the tokens.
w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes)
}
// writeBlockDynamic encodes a block using a dynamic Huffman table.
// This should be used if the symbols used have a disproportionate
// histogram distribution.
// If input is supplied and the compression savings are below 1/16th of the
// input size the block is stored.
func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) {
if w.err != nil {
return
}
tokens = append(tokens, endBlockMarker)
numLiterals, numOffsets := w.indexTokens(tokens)
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0)
// Store bytes, if we don't get a reasonable improvement.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Write Huffman table.
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
// Write the tokens.
w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes)
}
// indexTokens indexes a slice of tokens, and updates
// literalFreq and offsetFreq, and generates literalEncoding
// and offsetEncoding.
// The number of literal and offset tokens is returned.
func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) {
for i := range w.literalFreq {
w.literalFreq[i] = 0
}
for i := range w.offsetFreq {
w.offsetFreq[i] = 0
}
for _, t := range tokens {
if t < matchType {
w.literalFreq[t.literal()]++
continue
}
length := t.length()
offset := t.offset()
w.literalFreq[lengthCodesStart+lengthCode(length)]++
w.offsetFreq[offsetCode(offset)]++
}
// get the number of literals
numLiterals = len(w.literalFreq)
for w.literalFreq[numLiterals-1] == 0 {
numLiterals--
}
// get the number of offsets
numOffsets = len(w.offsetFreq)
for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
numOffsets--
}
if numOffsets == 0 {
// We haven't found a single match. If we want to go with the dynamic encoding,
// we should count at least one offset to be sure that the offset huffman tree could be encoded.
w.offsetFreq[0] = 1
numOffsets = 1
}
w.literalEncoding.generate(w.literalFreq, 15)
w.offsetEncoding.generate(w.offsetFreq, 15)
return
}
// writeTokens writes a slice of tokens to the output.
// codes for literal and offset encoding must be supplied.
func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) {
if w.err != nil {
return
}
for _, t := range tokens {
if t < matchType {
w.writeCode(leCodes[t.literal()])
continue
}
// Write the length
length := t.length()
lengthCode := lengthCode(length)
w.writeCode(leCodes[lengthCode+lengthCodesStart])
extraLengthBits := uint(lengthExtraBits[lengthCode])
if extraLengthBits > 0 {
extraLength := int32(length - lengthBase[lengthCode])
w.writeBits(extraLength, extraLengthBits)
}
// Write the offset
offset := t.offset()
offsetCode := offsetCode(offset)
w.writeCode(oeCodes[offsetCode])
extraOffsetBits := uint(offsetExtraBits[offsetCode])
if extraOffsetBits > 0 {
extraOffset := int32(offset - offsetBase[offsetCode])
w.writeBits(extraOffset, extraOffsetBits)
}
}
}
// huffOffset is a static offset encoder used for huffman only encoding.
// It can be reused since we will not be encoding offset values.
var huffOffset *huffmanEncoder
func init() {
w := newHuffmanBitWriter(nil)
w.offsetFreq[0] = 1
huffOffset = newHuffmanEncoder(offsetCodeCount)
huffOffset.generate(w.offsetFreq, 15)
}
// writeBlockHuff encodes a block of bytes as either
// Huffman encoded literals or uncompressed bytes if the
// results only gains very little from compression.
func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) {
if w.err != nil {
return
}
// Clear histogram
for i := range w.literalFreq {
w.literalFreq[i] = 0
}
// Add everything as literals
histogram(input, w.literalFreq)
w.literalFreq[endBlockMarker] = 1
const numLiterals = endBlockMarker + 1
const numOffsets = 1
w.literalEncoding.generate(w.literalFreq, 15)
// Figure out smallest code.
// Always use dynamic Huffman or Store
var numCodegens int
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0)
// Store bytes, if we don't get a reasonable improvement.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Huffman.
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
encoding := w.literalEncoding.codes[:257]
n := w.nbytes
for _, t := range input {
// Bitwriting inlined, ~30% speedup
c := encoding[t]
w.bits |= uint64(c.code) << w.nbits
w.nbits += uint(c.len)
if w.nbits < 48 {
continue
}
// Store 6 bytes
bits := w.bits
w.bits >>= 48
w.nbits -= 48
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n < bufferFlushSize {
continue
}
w.write(w.bytes[:n])
if w.err != nil {
return // Return early in the event of write failures
}
n = 0
}
w.nbytes = n
w.writeCode(encoding[endBlockMarker])
}
vendor/github.com/klauspost/compress/flate/huffman_code.go
Generated Vendored
+344
View File
@@ -0,0 +1,344 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"math"
"sort"
)
// hcode is a huffman code with a bit code and bit length.
type hcode struct {
code, len uint16
}
type huffmanEncoder struct {
codes []hcode
freqcache []literalNode
bitCount [17]int32
lns byLiteral // stored to avoid repeated allocation in generate
lfs byFreq // stored to avoid repeated allocation in generate
}
type literalNode struct {
literal uint16
freq int32
}
// A levelInfo describes the state of the constructed tree for a given depth.
type levelInfo struct {
// Our level. for better printing
level int32
// The frequency of the last node at this level
lastFreq int32
// The frequency of the next character to add to this level
nextCharFreq int32
// The frequency of the next pair (from level below) to add to this level.
// Only valid if the "needed" value of the next lower level is 0.
nextPairFreq int32
// The number of chains remaining to generate for this level before moving
// up to the next level
needed int32
}
// set sets the code and length of an hcode.
func (h *hcode) set(code uint16, length uint16) {
h.len = length
h.code = code
}
func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
func newHuffmanEncoder(size int) *huffmanEncoder {
return &huffmanEncoder{codes: make([]hcode, size)}
}
// Generates a HuffmanCode corresponding to the fixed literal table
func generateFixedLiteralEncoding() *huffmanEncoder {
h := newHuffmanEncoder(maxNumLit)
codes := h.codes
var ch uint16
for ch = 0; ch < maxNumLit; ch++ {
var bits uint16
var size uint16
switch {
case ch < 144:
// size 8, 000110000 .. 10111111
bits = ch + 48
size = 8
break
case ch < 256:
// size 9, 110010000 .. 111111111
bits = ch + 400 - 144
size = 9
break
case ch < 280:
// size 7, 0000000 .. 0010111
bits = ch - 256
size = 7
break
default:
// size 8, 11000000 .. 11000111
bits = ch + 192 - 280
size = 8
}
codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
}
return h
}
func generateFixedOffsetEncoding() *huffmanEncoder {
h := newHuffmanEncoder(30)
codes := h.codes
for ch := range codes {
codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
}
return h
}
var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
func (h *huffmanEncoder) bitLength(freq []int32) int {
var total int
for i, f := range freq {
if f != 0 {
total += int(f) * int(h.codes[i].len)
}
}
return total
}
const maxBitsLimit = 16
// Return the number of literals assigned to each bit size in the Huffman encoding
//
// This method is only called when list.length >= 3
// The cases of 0, 1, and 2 literals are handled by special case code.
//
// list An array of the literals with non-zero frequencies
// and their associated frequencies. The array is in order of increasing
// frequency, and has as its last element a special element with frequency
// MaxInt32
// maxBits The maximum number of bits that should be used to encode any literal.
// Must be less than 16.
// return An integer array in which array[i] indicates the number of literals
// that should be encoded in i bits.
func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
if maxBits >= maxBitsLimit {
panic("flate: maxBits too large")
}
n := int32(len(list))
list = list[0 : n+1]
list[n] = maxNode()
// The tree can't have greater depth than n - 1, no matter what. This
// saves a little bit of work in some small cases
if maxBits > n-1 {
maxBits = n - 1
}
// Create information about each of the levels.
// A bogus "Level 0" whose sole purpose is so that
// level1.prev.needed==0. This makes level1.nextPairFreq
// be a legitimate value that never gets chosen.
var levels [maxBitsLimit]levelInfo
// leafCounts[i] counts the number of literals at the left
// of ancestors of the rightmost node at level i.
// leafCounts[i][j] is the number of literals at the left
// of the level j ancestor.
var leafCounts [maxBitsLimit][maxBitsLimit]int32
for level := int32(1); level <= maxBits; level++ {
// For every level, the first two items are the first two characters.
// We initialize the levels as if we had already figured this out.
levels[level] = levelInfo{
level: level,
lastFreq: list[1].freq,
nextCharFreq: list[2].freq,
nextPairFreq: list[0].freq + list[1].freq,
}
leafCounts[level][level] = 2
if level == 1 {
levels[level].nextPairFreq = math.MaxInt32
}
}
// We need a total of 2*n - 2 items at top level and have already generated 2.
levels[maxBits].needed = 2*n - 4
level := maxBits
for {
l := &levels[level]
if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
// We've run out of both leafs and pairs.
// End all calculations for this level.
// To make sure we never come back to this level or any lower level,
// set nextPairFreq impossibly large.
l.needed = 0
levels[level+1].nextPairFreq = math.MaxInt32
level++
continue
}
prevFreq := l.lastFreq
if l.nextCharFreq < l.nextPairFreq {
// The next item on this row is a leaf node.
n := leafCounts[level][level] + 1
l.lastFreq = l.nextCharFreq
// Lower leafCounts are the same of the previous node.
leafCounts[level][level] = n
l.nextCharFreq = list[n].freq
} else {
// The next item on this row is a pair from the previous row.
// nextPairFreq isn't valid until we generate two
// more values in the level below
l.lastFreq = l.nextPairFreq
// Take leaf counts from the lower level, except counts[level] remains the same.
copy(leafCounts[level][:level], leafCounts[level-1][:level])
levels[l.level-1].needed = 2
}
if l.needed--; l.needed == 0 {
// We've done everything we need to do for this level.
// Continue calculating one level up. Fill in nextPairFreq
// of that level with the sum of the two nodes we've just calculated on
// this level.
if l.level == maxBits {
// All done!
break
}
levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
level++
} else {
// If we stole from below, move down temporarily to replenish it.
for levels[level-1].needed > 0 {
level--
}
}
}
// Somethings is wrong if at the end, the top level is null or hasn't used
// all of the leaves.
if leafCounts[maxBits][maxBits] != n {
panic("leafCounts[maxBits][maxBits] != n")
}
bitCount := h.bitCount[:maxBits+1]
bits := 1
counts := &leafCounts[maxBits]
for level := maxBits; level > 0; level-- {
// chain.leafCount gives the number of literals requiring at least "bits"
// bits to encode.
bitCount[bits] = counts[level] - counts[level-1]
bits++
}
return bitCount
}
// Look at the leaves and assign them a bit count and an encoding as specified
// in RFC 1951 3.2.2
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
code := uint16(0)
for n, bits := range bitCount {
code <<= 1
if n == 0 || bits == 0 {
continue
}
// The literals list[len(list)-bits] .. list[len(list)-bits]
// are encoded using "bits" bits, and get the values
// code, code + 1, .... The code values are
// assigned in literal order (not frequency order).
chunk := list[len(list)-int(bits):]
h.lns.sort(chunk)
for _, node := range chunk {
h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
code++
}
list = list[0 : len(list)-int(bits)]
}
}
// Update this Huffman Code object to be the minimum code for the specified frequency count.
//
// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
// maxBits The maximum number of bits to use for any literal.
func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
if h.freqcache == nil {
// Allocate a reusable buffer with the longest possible frequency table.
// Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
// The largest of these is maxNumLit, so we allocate for that case.
h.freqcache = make([]literalNode, maxNumLit+1)
}
list := h.freqcache[:len(freq)+1]
// Number of non-zero literals
count := 0
// Set list to be the set of all non-zero literals and their frequencies
for i, f := range freq {
if f != 0 {
list[count] = literalNode{uint16(i), f}
count++
} else {
list[count] = literalNode{}
h.codes[i].len = 0
}
}
list[len(freq)] = literalNode{}
list = list[:count]
if count <= 2 {
// Handle the small cases here, because they are awkward for the general case code. With
// two or fewer literals, everything has bit length 1.
for i, node := range list {
// "list" is in order of increasing literal value.
h.codes[node.literal].set(uint16(i), 1)
}
return
}
h.lfs.sort(list)
// Get the number of literals for each bit count
bitCount := h.bitCounts(list, maxBits)
// And do the assignment
h.assignEncodingAndSize(bitCount, list)
}
type byLiteral []literalNode
func (s *byLiteral) sort(a []literalNode) {
*s = byLiteral(a)
sort.Sort(s)
}
func (s byLiteral) Len() int { return len(s) }
func (s byLiteral) Less(i, j int) bool {
return s[i].literal < s[j].literal
}
func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
type byFreq []literalNode
func (s *byFreq) sort(a []literalNode) {
*s = byFreq(a)
sort.Sort(s)
}
func (s byFreq) Len() int { return len(s) }
func (s byFreq) Less(i, j int) bool {
if s[i].freq == s[j].freq {
return s[i].literal < s[j].literal
}
return s[i].freq < s[j].freq
}
func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
vendor/github.com/klauspost/compress/flate/inflate.go
Generated Vendored
+868
View File
@@ -0,0 +1,868 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package flate implements the DEFLATE compressed data format, described in
// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file
// formats.
package flate
import (
"bufio"
"io"
"strconv"
"sync"
)
const (
maxCodeLen = 16 // max length of Huffman code
maxCodeLenMask = 15 // mask for max length of Huffman code
// The next three numbers come from the RFC section 3.2.7, with the
// additional proviso in section 3.2.5 which implies that distance codes
// 30 and 31 should never occur in compressed data.
maxNumLit = 286
maxNumDist = 30
numCodes = 19 // number of codes in Huffman meta-code
)
// Initialize the fixedHuffmanDecoder only once upon first use.
var fixedOnce sync.Once
var fixedHuffmanDecoder huffmanDecoder
// A CorruptInputError reports the presence of corrupt input at a given offset.
type CorruptInputError int64
func (e CorruptInputError) Error() string {
return "flate: corrupt input before offset " + strconv.FormatInt(int64(e), 10)
}
// An InternalError reports an error in the flate code itself.
type InternalError string
func (e InternalError) Error() string { return "flate: internal error: " + string(e) }
// A ReadError reports an error encountered while reading input.
//
// Deprecated: No longer returned.
type ReadError struct {
Offset int64 // byte offset where error occurred
Err error // error returned by underlying Read
}
func (e *ReadError) Error() string {
return "flate: read error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error()
}
// A WriteError reports an error encountered while writing output.
//
// Deprecated: No longer returned.
type WriteError struct {
Offset int64 // byte offset where error occurred
Err error // error returned by underlying Write
}
func (e *WriteError) Error() string {
return "flate: write error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error()
}
// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to
// to switch to a new underlying Reader. This permits reusing a ReadCloser
// instead of allocating a new one.
type Resetter interface {
// Reset discards any buffered data and resets the Resetter as if it was
// newly initialized with the given reader.
Reset(r io.Reader, dict []byte) error
}
// The data structure for decoding Huffman tables is based on that of
// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits),
// For codes smaller than the table width, there are multiple entries
// (each combination of trailing bits has the same value). For codes
// larger than the table width, the table contains a link to an overflow
// table. The width of each entry in the link table is the maximum code
// size minus the chunk width.
//
// Note that you can do a lookup in the table even without all bits
// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
// have the property that shorter codes come before longer ones, the
// bit length estimate in the result is a lower bound on the actual
// number of bits.
//
// See the following:
// http://www.gzip.org/algorithm.txt
// chunk & 15 is number of bits
// chunk >> 4 is value, including table link
const (
huffmanChunkBits = 9
huffmanNumChunks = 1 << huffmanChunkBits
huffmanCountMask = 15
huffmanValueShift = 4
)
type huffmanDecoder struct {
min int // the minimum code length
chunks *[huffmanNumChunks]uint32 // chunks as described above
links [][]uint32 // overflow links
linkMask uint32 // mask the width of the link table
}
// Initialize Huffman decoding tables from array of code lengths.
// Following this function, h is guaranteed to be initialized into a complete
// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
// degenerate case where the tree has only a single symbol with length 1. Empty
// trees are permitted.
func (h *huffmanDecoder) init(bits []int) bool {
// Sanity enables additional runtime tests during Huffman
// table construction. It's intended to be used during
// development to supplement the currently ad-hoc unit tests.
const sanity = false
if h.chunks == nil {
h.chunks = &[huffmanNumChunks]uint32{}
}
if h.min != 0 {
*h = huffmanDecoder{chunks: h.chunks, links: h.links}
}
// Count number of codes of each length,
// compute min and max length.
var count [maxCodeLen]int
var min, max int
for _, n := range bits {
if n == 0 {
continue
}
if min == 0 || n < min {
min = n
}
if n > max {
max = n
}
count[n&maxCodeLenMask]++
}
// Empty tree. The decompressor.huffSym function will fail later if the tree
// is used. Technically, an empty tree is only valid for the HDIST tree and
// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
// is guaranteed to fail since it will attempt to use the tree to decode the
// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
// guaranteed to fail later since the compressed data section must be
// composed of at least one symbol (the end-of-block marker).
if max == 0 {
return true
}
code := 0
var nextcode [maxCodeLen]int
for i := min; i <= max; i++ {
code <<= 1
nextcode[i&maxCodeLenMask] = code
code += count[i&maxCodeLenMask]
}
// Check that the coding is complete (i.e., that we've
// assigned all 2-to-the-max possible bit sequences).
// Exception: To be compatible with zlib, we also need to
// accept degenerate single-code codings. See also
// TestDegenerateHuffmanCoding.
if code != 1<<uint(max) && !(code == 1 && max == 1) {
return false
}
h.min = min
chunks := h.chunks[:]
for i := range chunks {
chunks[i] = 0
}
if max > huffmanChunkBits {
numLinks := 1 << (uint(max) - huffmanChunkBits)
h.linkMask = uint32(numLinks - 1)
// create link tables
link := nextcode[huffmanChunkBits+1] >> 1
if cap(h.links) < huffmanNumChunks-link {
h.links = make([][]uint32, huffmanNumChunks-link)
} else {
h.links = h.links[:huffmanNumChunks-link]
}
for j := uint(link); j < huffmanNumChunks; j++ {
reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8
reverse >>= uint(16 - huffmanChunkBits)
off := j - uint(link)
if sanity && h.chunks[reverse] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[reverse] = uint32(off<<huffmanValueShift | (huffmanChunkBits + 1))
if cap(h.links[off]) < numLinks {
h.links[off] = make([]uint32, numLinks)
} else {
links := h.links[off][:0]
h.links[off] = links[:numLinks]
}
}
} else {
h.links = h.links[:0]
}
for i, n := range bits {
if n == 0 {
continue
}
code := nextcode[n]
nextcode[n]++
chunk := uint32(i<<huffmanValueShift | n)
reverse := int(reverseByte[code>>8]) | int(reverseByte[code&0xff])<<8
reverse >>= uint(16 - n)
if n <= huffmanChunkBits {
for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
// We should never need to overwrite
// an existing chunk. Also, 0 is
// never a valid chunk, because the
// lower 4 "count" bits should be
// between 1 and 15.
if sanity && h.chunks[off] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[off] = chunk
}
} else {
j := reverse & (huffmanNumChunks - 1)
if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
// Longer codes should have been
// associated with a link table above.
panic("impossible: not an indirect chunk")
}
value := h.chunks[j] >> huffmanValueShift
linktab := h.links[value]
reverse >>= huffmanChunkBits
for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
if sanity && linktab[off] != 0 {
panic("impossible: overwriting existing chunk")
}
linktab[off] = chunk
}
}
}
if sanity {
// Above we've sanity checked that we never overwrote
// an existing entry. Here we additionally check that
// we filled the tables completely.
for i, chunk := range h.chunks {
if chunk == 0 {
// As an exception, in the degenerate
// single-code case, we allow odd
// chunks to be missing.
if code == 1 && i%2 == 1 {
continue
}
panic("impossible: missing chunk")
}
}
for _, linktab := range h.links {
for _, chunk := range linktab {
if chunk == 0 {
panic("impossible: missing chunk")
}
}
}
}
return true
}
// The actual read interface needed by NewReader.
// If the passed in io.Reader does not also have ReadByte,
// the NewReader will introduce its own buffering.
type Reader interface {
io.Reader
io.ByteReader
}
// Decompress state.
type decompressor struct {
// Input source.
r Reader
roffset int64
// Input bits, in top of b.
b uint32
nb uint
// Huffman decoders for literal/length, distance.
h1, h2 huffmanDecoder
// Length arrays used to define Huffman codes.
bits *[maxNumLit + maxNumDist]int
codebits *[numCodes]int
// Output history, buffer.
dict dictDecoder
// Temporary buffer (avoids repeated allocation).
buf [4]byte
// Next step in the decompression,
// and decompression state.
step func(*decompressor)
stepState int
final bool
err error
toRead []byte
hl, hd *huffmanDecoder
copyLen int
copyDist int
}
func (f *decompressor) nextBlock() {
for f.nb < 1+2 {
if f.err = f.moreBits(); f.err != nil {
return
}
}
f.final = f.b&1 == 1
f.b >>= 1
typ := f.b & 3
f.b >>= 2
f.nb -= 1 + 2
switch typ {
case 0:
f.dataBlock()
case 1:
// compressed, fixed Huffman tables
f.hl = &fixedHuffmanDecoder
f.hd = nil
f.huffmanBlock()
case 2:
// compressed, dynamic Huffman tables
if f.err = f.readHuffman(); f.err != nil {
break
}
f.hl = &f.h1
f.hd = &f.h2
f.huffmanBlock()
default:
// 3 is reserved.
f.err = CorruptInputError(f.roffset)
}
}
func (f *decompressor) Read(b []byte) (int, error) {
for {
if len(f.toRead) > 0 {
n := copy(b, f.toRead)
f.toRead = f.toRead[n:]
if len(f.toRead) == 0 {
return n, f.err
}
return n, nil
}
if f.err != nil {
return 0, f.err
}
f.step(f)
if f.err != nil && len(f.toRead) == 0 {
f.toRead = f.dict.readFlush() // Flush what's left in case of error
}
}
}
// Support the io.WriteTo interface for io.Copy and friends.
func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
total := int64(0)
flushed := false
for {
if len(f.toRead) > 0 {
n, err := w.Write(f.toRead)
total += int64(n)
if err != nil {
f.err = err
return total, err
}
if n != len(f.toRead) {
return total, io.ErrShortWrite
}
f.toRead = f.toRead[:0]
}
if f.err != nil && flushed {
if f.err == io.EOF {
return total, nil
}
return total, f.err
}
if f.err == nil {
f.step(f)
}
if len(f.toRead) == 0 && f.err != nil && !flushed {
f.toRead = f.dict.readFlush() // Flush what's left in case of error
flushed = true
}
}
}
func (f *decompressor) Close() error {
if f.err == io.EOF {
return nil
}
return f.err
}
// RFC 1951 section 3.2.7.
// Compression with dynamic Huffman codes
var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
func (f *decompressor) readHuffman() error {
// HLIT[5], HDIST[5], HCLEN[4].
for f.nb < 5+5+4 {
if err := f.moreBits(); err != nil {
return err
}
}
nlit := int(f.b&0x1F) + 257
if nlit > maxNumLit {
return CorruptInputError(f.roffset)
}
f.b >>= 5
ndist := int(f.b&0x1F) + 1
if ndist > maxNumDist {
return CorruptInputError(f.roffset)
}
f.b >>= 5
nclen := int(f.b&0xF) + 4
// numCodes is 19, so nclen is always valid.
f.b >>= 4
f.nb -= 5 + 5 + 4
// (HCLEN+4)*3 bits: code lengths in the magic codeOrder order.
for i := 0; i < nclen; i++ {
for f.nb < 3 {
if err := f.moreBits(); err != nil {
return err
}
}
f.codebits[codeOrder[i]] = int(f.b & 0x7)
f.b >>= 3
f.nb -= 3
}
for i := nclen; i < len(codeOrder); i++ {
f.codebits[codeOrder[i]] = 0
}
if !f.h1.init(f.codebits[0:]) {
return CorruptInputError(f.roffset)
}
// HLIT + 257 code lengths, HDIST + 1 code lengths,
// using the code length Huffman code.
for i, n := 0, nlit+ndist; i < n; {
x, err := f.huffSym(&f.h1)
if err != nil {
return err
}
if x < 16 {
// Actual length.
f.bits[i] = x
i++
continue
}
// Repeat previous length or zero.
var rep int
var nb uint
var b int
switch x {
default:
return InternalError("unexpected length code")
case 16:
rep = 3
nb = 2
if i == 0 {
return CorruptInputError(f.roffset)
}
b = f.bits[i-1]
case 17:
rep = 3
nb = 3
b = 0
case 18:
rep = 11
nb = 7
b = 0
}
for f.nb < nb {
if err := f.moreBits(); err != nil {
return err
}
}
rep += int(f.b & uint32(1<<nb-1))
f.b >>= nb
f.nb -= nb
if i+rep > n {
return CorruptInputError(f.roffset)
}
for j := 0; j < rep; j++ {
f.bits[i] = b
i++
}
}
if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) {
return CorruptInputError(f.roffset)
}
// As an optimization, we can initialize the min bits to read at a time
// for the HLIT tree to the length of the EOB marker since we know that
// every block must terminate with one. This preserves the property that
// we never read any extra bytes after the end of the DEFLATE stream.
if f.h1.min < f.bits[endBlockMarker] {
f.h1.min = f.bits[endBlockMarker]
}
return nil
}
// Decode a single Huffman block from f.
// hl and hd are the Huffman states for the lit/length values
// and the distance values, respectively. If hd == nil, using the
// fixed distance encoding associated with fixed Huffman blocks.
func (f *decompressor) huffmanBlock() {
const (
stateInit = iota // Zero value must be stateInit
stateDict
)
switch f.stepState {
case stateInit:
goto readLiteral
case stateDict:
goto copyHistory
}
readLiteral:
// Read literal and/or (length, distance) according to RFC section 3.2.3.
{
v, err := f.huffSym(f.hl)
if err != nil {
f.err = err
return
}
var n uint // number of bits extra
var length int
switch {
case v < 256:
f.dict.writeByte(byte(v))
if f.dict.availWrite() == 0 {
f.toRead = f.dict.readFlush()
f.step = (*decompressor).huffmanBlock
f.stepState = stateInit
return
}
goto readLiteral
case v == 256:
f.finishBlock()
return
// otherwise, reference to older data
case v < 265:
length = v - (257 - 3)
n = 0
case v < 269:
length = v*2 - (265*2 - 11)
n = 1
case v < 273:
length = v*4 - (269*4 - 19)
n = 2
case v < 277:
length = v*8 - (273*8 - 35)
n = 3
case v < 281:
length = v*16 - (277*16 - 67)
n = 4
case v < 285:
length = v*32 - (281*32 - 131)
n = 5
case v < maxNumLit:
length = 258
n = 0
default:
f.err = CorruptInputError(f.roffset)
return
}
if n > 0 {
for f.nb < n {
if err = f.moreBits(); err != nil {
f.err = err
return
}
}
length += int(f.b & uint32(1<<n-1))
f.b >>= n
f.nb -= n
}
var dist int
if f.hd == nil {
for f.nb < 5 {
if err = f.moreBits(); err != nil {
f.err = err
return
}
}
dist = int(reverseByte[(f.b&0x1F)<<3])
f.b >>= 5
f.nb -= 5
} else {
if dist, err = f.huffSym(f.hd); err != nil {
f.err = err
return
}
}
switch {
case dist < 4:
dist++
case dist < maxNumDist:
nb := uint(dist-2) >> 1
// have 1 bit in bottom of dist, need nb more.
extra := (dist & 1) << nb
for f.nb < nb {
if err = f.moreBits(); err != nil {
f.err = err
return
}
}
extra |= int(f.b & uint32(1<<nb-1))
f.b >>= nb
f.nb -= nb
dist = 1<<(nb+1) + 1 + extra
default:
f.err = CorruptInputError(f.roffset)
return
}
// No check on length; encoding can be prescient.
if dist > f.dict.histSize() {
f.err = CorruptInputError(f.roffset)
return
}
f.copyLen, f.copyDist = length, dist
goto copyHistory
}
copyHistory:
// Perform a backwards copy according to RFC section 3.2.3.
{
cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen)
if cnt == 0 {
cnt = f.dict.writeCopy(f.copyDist, f.copyLen)
}
f.copyLen -= cnt
if f.dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = f.dict.readFlush()
f.step = (*decompressor).huffmanBlock // We need to continue this work
f.stepState = stateDict
return
}
goto readLiteral
}
}
// Copy a single uncompressed data block from input to output.
func (f *decompressor) dataBlock() {
// Uncompressed.
// Discard current half-byte.
f.nb = 0
f.b = 0
// Length then ones-complement of length.
nr, err := io.ReadFull(f.r, f.buf[0:4])
f.roffset += int64(nr)
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
f.err = err
return
}
n := int(f.buf[0]) | int(f.buf[1])<<8
nn := int(f.buf[2]) | int(f.buf[3])<<8
if uint16(nn) != uint16(^n) {
f.err = CorruptInputError(f.roffset)
return
}
if n == 0 {
f.toRead = f.dict.readFlush()
f.finishBlock()
return
}
f.copyLen = n
f.copyData()
}
// copyData copies f.copyLen bytes from the underlying reader into f.hist.
// It pauses for reads when f.hist is full.
func (f *decompressor) copyData() {
buf := f.dict.writeSlice()
if len(buf) > f.copyLen {
buf = buf[:f.copyLen]
}
cnt, err := io.ReadFull(f.r, buf)
f.roffset += int64(cnt)
f.copyLen -= cnt
f.dict.writeMark(cnt)
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
f.err = err
return
}
if f.dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = f.dict.readFlush()
f.step = (*decompressor).copyData
return
}
f.finishBlock()
}
func (f *decompressor) finishBlock() {
if f.final {
if f.dict.availRead() > 0 {
f.toRead = f.dict.readFlush()
}
f.err = io.EOF
}
f.step = (*decompressor).nextBlock
}
func (f *decompressor) moreBits() error {
c, err := f.r.ReadByte()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
}
f.roffset++
f.b |= uint32(c) << f.nb
f.nb += 8
return nil
}
// Read the next Huffman-encoded symbol from f according to h.
func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) {
// Since a huffmanDecoder can be empty or be composed of a degenerate tree
// with single element, huffSym must error on these two edge cases. In both
// cases, the chunks slice will be 0 for the invalid sequence, leading it
// satisfy the n == 0 check below.
n := uint(h.min)
for {
for f.nb < n {
if err := f.moreBits(); err != nil {
return 0, err
}
}
chunk := h.chunks[f.b&(huffmanNumChunks-1)]
n = uint(chunk & huffmanCountMask)
if n > huffmanChunkBits {
chunk = h.links[chunk>>huffmanValueShift][(f.b>>huffmanChunkBits)&h.linkMask]
n = uint(chunk & huffmanCountMask)
}
if n <= f.nb {
if n == 0 {
f.err = CorruptInputError(f.roffset)
return 0, f.err
}
f.b >>= n
f.nb -= n
return int(chunk >> huffmanValueShift), nil
}
}
}
func makeReader(r io.Reader) Reader {
if rr, ok := r.(Reader); ok {
return rr
}
return bufio.NewReader(r)
}
func fixedHuffmanDecoderInit() {
fixedOnce.Do(func() {
// These come from the RFC section 3.2.6.
var bits [288]int
for i := 0; i < 144; i++ {
bits[i] = 8
}
for i := 144; i < 256; i++ {
bits[i] = 9
}
for i := 256; i < 280; i++ {
bits[i] = 7
}
for i := 280; i < 288; i++ {
bits[i] = 8
}
fixedHuffmanDecoder.init(bits[:])
})
}
func (f *decompressor) Reset(r io.Reader, dict []byte) error {
*f = decompressor{
r: makeReader(r),
bits: f.bits,
codebits: f.codebits,
h1: f.h1,
h2: f.h2,
dict: f.dict,
step: (*decompressor).nextBlock,
}
f.dict.init(maxMatchOffset, dict)
return nil
}
// NewReader returns a new ReadCloser that can be used
// to read the uncompressed version of r.
// If r does not also implement io.ByteReader,
// the decompressor may read more data than necessary from r.
// It is the caller's responsibility to call Close on the ReadCloser
// when finished reading.
//
// The ReadCloser returned by NewReader also implements Resetter.
func NewReader(r io.Reader) io.ReadCloser {
fixedHuffmanDecoderInit()
var f decompressor
f.r = makeReader(r)
f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int)
f.step = (*decompressor).nextBlock
f.dict.init(maxMatchOffset, nil)
return &f
}
// NewReaderDict is like NewReader but initializes the reader
// with a preset dictionary. The returned Reader behaves as if
// the uncompressed data stream started with the given dictionary,
// which has already been read. NewReaderDict is typically used
// to read data compressed by NewWriterDict.
//
// The ReadCloser returned by NewReader also implements Resetter.
func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
fixedHuffmanDecoderInit()
var f decompressor
f.r = makeReader(r)
f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int)
f.step = (*decompressor).nextBlock
f.dict.init(maxMatchOffset, dict)
return &f
}
vendor/github.com/klauspost/compress/flate/reverse_bits.go
Generated Vendored
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
var reverseByte = [256]byte{
0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0,
0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4,
0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec,
0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea,
0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6,
0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1,
0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9,
0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed,
0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3,
0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7,
0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef,
0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff,
}
func reverseUint16(v uint16) uint16 {
return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8
}
func reverseBits(number uint16, bitLength byte) uint16 {
return reverseUint16(number << uint8(16-bitLength))
}
vendor/github.com/klauspost/compress/flate/snappy.go
Generated Vendored
+900
View File
@@ -0,0 +1,900 @@
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
// Modified for deflate by Klaus Post (c) 2015.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// emitLiteral writes a literal chunk and returns the number of bytes written.
func emitLiteral(dst *tokens, lit []byte) {
ol := int(dst.n)
for i, v := range lit {
dst.tokens[(i+ol)&maxStoreBlockSize] = token(v)
}
dst.n += uint16(len(lit))
}
// emitCopy writes a copy chunk and returns the number of bytes written.
func emitCopy(dst *tokens, offset, length int) {
dst.tokens[dst.n] = matchToken(uint32(length-3), uint32(offset-minOffsetSize))
dst.n++
}
type snappyEnc interface {
Encode(dst *tokens, src []byte)
Reset()
}
func newSnappy(level int) snappyEnc {
switch level {
case 1:
return &snappyL1{}
case 2:
return &snappyL2{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}
case 3:
return &snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}
case 4:
return &snappyL4{snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}}
default:
panic("invalid level specified")
}
}
const (
tableBits = 14 // Bits used in the table
tableSize = 1 << tableBits // Size of the table
tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
baseMatchOffset = 1 // The smallest match offset
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
maxMatchOffset = 1 << 15 // The largest match offset
)
func load32(b []byte, i int) uint32 {
b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}
func load64(b []byte, i int) uint64 {
b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}
func hash(u uint32) uint32 {
return (u * 0x1e35a7bd) >> tableShift
}
// snappyL1 encapsulates level 1 compression
type snappyL1 struct{}
func (e *snappyL1) Reset() {}
func (e *snappyL1) Encode(dst *tokens, src []byte) {
const (
inputMargin = 16 - 1
minNonLiteralBlockSize = 1 + 1 + inputMargin
)
// This check isn't in the Snappy implementation, but there, the caller
// instead of the callee handles this case.
if len(src) < minNonLiteralBlockSize {
// We do not fill the token table.
// This will be picked up by caller.
dst.n = uint16(len(src))
return
}
// Initialize the hash table.
//
// The table element type is uint16, as s < sLimit and sLimit < len(src)
// and len(src) <= maxStoreBlockSize and maxStoreBlockSize == 65535.
var table [tableSize]uint16
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := len(src) - inputMargin
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := 0
// The encoded form must start with a literal, as there are no previous
// bytes to copy, so we start looking for hash matches at s == 1.
s := 1
nextHash := hash(load32(src, s))
for {
// Copied from the C++ snappy implementation:
//
// Heuristic match skipping: If 32 bytes are scanned with no matches
// found, start looking only at every other byte. If 32 more bytes are
// scanned (or skipped), look at every third byte, etc.. When a match
// is found, immediately go back to looking at every byte. This is a
// small loss (~5% performance, ~0.1% density) for compressible data
// due to more bookkeeping, but for non-compressible data (such as
// JPEG) it's a huge win since the compressor quickly "realizes" the
// data is incompressible and doesn't bother looking for matches
// everywhere.
//
// The "skip" variable keeps track of how many bytes there are since
// the last match; dividing it by 32 (ie. right-shifting by five) gives
// the number of bytes to move ahead for each iteration.
skip := 32
nextS := s
candidate := 0
for {
s = nextS
bytesBetweenHashLookups := skip >> 5
nextS = s + bytesBetweenHashLookups
skip += bytesBetweenHashLookups
if nextS > sLimit {
goto emitRemainder
}
candidate = int(table[nextHash&tableMask])
table[nextHash&tableMask] = uint16(s)
nextHash = hash(load32(src, nextS))
if s-candidate <= maxMatchOffset && load32(src, s) == load32(src, candidate) {
break
}
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
emitLiteral(dst, src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
base := s
// Extend the 4-byte match as long as possible.
//
// This is an inlined version of Snappy's:
// s = extendMatch(src, candidate+4, s+4)
s += 4
s1 := base + maxMatchLength
if s1 > len(src) {
s1 = len(src)
}
a := src[s:s1]
b := src[candidate+4:]
b = b[:len(a)]
l := len(a)
for i := range a {
if a[i] != b[i] {
l = i
break
}
}
s += l
// matchToken is flate's equivalent of Snappy's emitCopy.
dst.tokens[dst.n] = matchToken(uint32(s-base-baseMatchLength), uint32(base-candidate-baseMatchOffset))
dst.n++
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-1 and at s. If
// another emitCopy is not our next move, also calculate nextHash
// at s+1. At least on GOARCH=amd64, these three hash calculations
// are faster as one load64 call (with some shifts) instead of
// three load32 calls.
x := load64(src, s-1)
prevHash := hash(uint32(x >> 0))
table[prevHash&tableMask] = uint16(s - 1)
currHash := hash(uint32(x >> 8))
candidate = int(table[currHash&tableMask])
table[currHash&tableMask] = uint16(s)
if s-candidate > maxMatchOffset || uint32(x>>8) != load32(src, candidate) {
nextHash = hash(uint32(x >> 16))
s++
break
}
}
}
emitRemainder:
if nextEmit < len(src) {
emitLiteral(dst, src[nextEmit:])
}
}
type tableEntry struct {
val uint32
offset int32
}
func load3232(b []byte, i int32) uint32 {
b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}
func load6432(b []byte, i int32) uint64 {
b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}
// snappyGen maintains the table for matches,
// and the previous byte block for level 2.
// This is the generic implementation.
type snappyGen struct {
prev []byte
cur int32
}
// snappyGen maintains the table for matches,
// and the previous byte block for level 2.
// This is the generic implementation.
type snappyL2 struct {
snappyGen
table [tableSize]tableEntry
}
// EncodeL2 uses a similar algorithm to level 1, but is capable
// of matching across blocks giving better compression at a small slowdown.
func (e *snappyL2) Encode(dst *tokens, src []byte) {
const (
inputMargin = 8 - 1
minNonLiteralBlockSize = 1 + 1 + inputMargin
)
// Protect against e.cur wraparound.
if e.cur > 1<<30 {
for i := range e.table[:] {
e.table[i] = tableEntry{}
}
e.cur = maxStoreBlockSize
}
// This check isn't in the Snappy implementation, but there, the caller
// instead of the callee handles this case.
if len(src) < minNonLiteralBlockSize {
// We do not fill the token table.
// This will be picked up by caller.
dst.n = uint16(len(src))
e.cur += maxStoreBlockSize
e.prev = e.prev[:0]
return
}
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := int32(len(src) - inputMargin)
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := int32(0)
s := int32(0)
cv := load3232(src, s)
nextHash := hash(cv)
for {
// Copied from the C++ snappy implementation:
//
// Heuristic match skipping: If 32 bytes are scanned with no matches
// found, start looking only at every other byte. If 32 more bytes are
// scanned (or skipped), look at every third byte, etc.. When a match
// is found, immediately go back to looking at every byte. This is a
// small loss (~5% performance, ~0.1% density) for compressible data
// due to more bookkeeping, but for non-compressible data (such as
// JPEG) it's a huge win since the compressor quickly "realizes" the
// data is incompressible and doesn't bother looking for matches
// everywhere.
//
// The "skip" variable keeps track of how many bytes there are since
// the last match; dividing it by 32 (ie. right-shifting by five) gives
// the number of bytes to move ahead for each iteration.
skip := int32(32)
nextS := s
var candidate tableEntry
for {
s = nextS
bytesBetweenHashLookups := skip >> 5
nextS = s + bytesBetweenHashLookups
skip += bytesBetweenHashLookups
if nextS > sLimit {
goto emitRemainder
}
candidate = e.table[nextHash&tableMask]
now := load3232(src, nextS)
e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv}
nextHash = hash(now)
offset := s - (candidate.offset - e.cur)
if offset > maxMatchOffset || cv != candidate.val {
// Out of range or not matched.
cv = now
continue
}
break
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
emitLiteral(dst, src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
// Extend the 4-byte match as long as possible.
//
s += 4
t := candidate.offset - e.cur + 4
l := e.matchlen(s, t, src)
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
dst.n++
s += l
nextEmit = s
if s >= sLimit {
t += l
// Index first pair after match end.
if int(t+4) < len(src) && t > 0 {
cv := load3232(src, t)
e.table[hash(cv)&tableMask] = tableEntry{offset: t + e.cur, val: cv}
}
goto emitRemainder
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-1 and at s. If
// another emitCopy is not our next move, also calculate nextHash
// at s+1. At least on GOARCH=amd64, these three hash calculations
// are faster as one load64 call (with some shifts) instead of
// three load32 calls.
x := load6432(src, s-1)
prevHash := hash(uint32(x))
e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)}
x >>= 8
currHash := hash(uint32(x))
candidate = e.table[currHash&tableMask]
e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)}
offset := s - (candidate.offset - e.cur)
if offset > maxMatchOffset || uint32(x) != candidate.val {
cv = uint32(x >> 8)
nextHash = hash(cv)
s++
break
}
}
}
emitRemainder:
if int(nextEmit) < len(src) {
emitLiteral(dst, src[nextEmit:])
}
e.cur += int32(len(src))
e.prev = e.prev[:len(src)]
copy(e.prev, src)
}
type tableEntryPrev struct {
Cur tableEntry
Prev tableEntry
}
// snappyL3
type snappyL3 struct {
snappyGen
table [tableSize]tableEntryPrev
}
// Encode uses a similar algorithm to level 2, will check up to two candidates.
func (e *snappyL3) Encode(dst *tokens, src []byte) {
const (
inputMargin = 8 - 1
minNonLiteralBlockSize = 1 + 1 + inputMargin
)
// Protect against e.cur wraparound.
if e.cur > 1<<30 {
for i := range e.table[:] {
e.table[i] = tableEntryPrev{}
}
e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]}
}
// This check isn't in the Snappy implementation, but there, the caller
// instead of the callee handles this case.
if len(src) < minNonLiteralBlockSize {
// We do not fill the token table.
// This will be picked up by caller.
dst.n = uint16(len(src))
e.cur += maxStoreBlockSize
e.prev = e.prev[:0]
return
}
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := int32(len(src) - inputMargin)
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := int32(0)
s := int32(0)
cv := load3232(src, s)
nextHash := hash(cv)
for {
// Copied from the C++ snappy implementation:
//
// Heuristic match skipping: If 32 bytes are scanned with no matches
// found, start looking only at every other byte. If 32 more bytes are
// scanned (or skipped), look at every third byte, etc.. When a match
// is found, immediately go back to looking at every byte. This is a
// small loss (~5% performance, ~0.1% density) for compressible data
// due to more bookkeeping, but for non-compressible data (such as
// JPEG) it's a huge win since the compressor quickly "realizes" the
// data is incompressible and doesn't bother looking for matches
// everywhere.
//
// The "skip" variable keeps track of how many bytes there are since
// the last match; dividing it by 32 (ie. right-shifting by five) gives
// the number of bytes to move ahead for each iteration.
skip := int32(32)
nextS := s
var candidate tableEntry
for {
s = nextS
bytesBetweenHashLookups := skip >> 5
nextS = s + bytesBetweenHashLookups
skip += bytesBetweenHashLookups
if nextS > sLimit {
goto emitRemainder
}
candidates := e.table[nextHash&tableMask]
now := load3232(src, nextS)
e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}}
nextHash = hash(now)
// Check both candidates
candidate = candidates.Cur
if cv == candidate.val {
offset := s - (candidate.offset - e.cur)
if offset <= maxMatchOffset {
break
}
} else {
// We only check if value mismatches.
// Offset will always be invalid in other cases.
candidate = candidates.Prev
if cv == candidate.val {
offset := s - (candidate.offset - e.cur)
if offset <= maxMatchOffset {
break
}
}
}
cv = now
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
emitLiteral(dst, src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
// Extend the 4-byte match as long as possible.
//
s += 4
t := candidate.offset - e.cur + 4
l := e.matchlen(s, t, src)
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
dst.n++
s += l
nextEmit = s
if s >= sLimit {
t += l
// Index first pair after match end.
if int(t+4) < len(src) && t > 0 {
cv := load3232(src, t)
nextHash = hash(cv)
e.table[nextHash&tableMask] = tableEntryPrev{
Prev: e.table[nextHash&tableMask].Cur,
Cur: tableEntry{offset: e.cur + t, val: cv},
}
}
goto emitRemainder
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-3 to s. If
// another emitCopy is not our next move, also calculate nextHash
// at s+1. At least on GOARCH=amd64, these three hash calculations
// are faster as one load64 call (with some shifts) instead of
// three load32 calls.
x := load6432(src, s-3)
prevHash := hash(uint32(x))
e.table[prevHash&tableMask] = tableEntryPrev{
Prev: e.table[prevHash&tableMask].Cur,
Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)},
}
x >>= 8
prevHash = hash(uint32(x))
e.table[prevHash&tableMask] = tableEntryPrev{
Prev: e.table[prevHash&tableMask].Cur,
Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)},
}
x >>= 8
prevHash = hash(uint32(x))
e.table[prevHash&tableMask] = tableEntryPrev{
Prev: e.table[prevHash&tableMask].Cur,
Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)},
}
x >>= 8
currHash := hash(uint32(x))
candidates := e.table[currHash&tableMask]
cv = uint32(x)
e.table[currHash&tableMask] = tableEntryPrev{
Prev: candidates.Cur,
Cur: tableEntry{offset: s + e.cur, val: cv},
}
// Check both candidates
candidate = candidates.Cur
if cv == candidate.val {
offset := s - (candidate.offset - e.cur)
if offset <= maxMatchOffset {
continue
}
} else {
// We only check if value mismatches.
// Offset will always be invalid in other cases.
candidate = candidates.Prev
if cv == candidate.val {
offset := s - (candidate.offset - e.cur)
if offset <= maxMatchOffset {
continue
}
}
}
cv = uint32(x >> 8)
nextHash = hash(cv)
s++
break
}
}
emitRemainder:
if int(nextEmit) < len(src) {
emitLiteral(dst, src[nextEmit:])
}
e.cur += int32(len(src))
e.prev = e.prev[:len(src)]
copy(e.prev, src)
}
// snappyL4
type snappyL4 struct {
snappyL3
}
// Encode uses a similar algorithm to level 3,
// but will check up to two candidates if first isn't long enough.
func (e *snappyL4) Encode(dst *tokens, src []byte) {
const (
inputMargin = 8 - 3
minNonLiteralBlockSize = 1 + 1 + inputMargin
matchLenGood = 12
)
// Protect against e.cur wraparound.
if e.cur > 1<<30 {
for i := range e.table[:] {
e.table[i] = tableEntryPrev{}
}
e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]}
}
// This check isn't in the Snappy implementation, but there, the caller
// instead of the callee handles this case.
if len(src) < minNonLiteralBlockSize {
// We do not fill the token table.
// This will be picked up by caller.
dst.n = uint16(len(src))
e.cur += maxStoreBlockSize
e.prev = e.prev[:0]
return
}
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := int32(len(src) - inputMargin)
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := int32(0)
s := int32(0)
cv := load3232(src, s)
nextHash := hash(cv)
for {
// Copied from the C++ snappy implementation:
//
// Heuristic match skipping: If 32 bytes are scanned with no matches
// found, start looking only at every other byte. If 32 more bytes are
// scanned (or skipped), look at every third byte, etc.. When a match
// is found, immediately go back to looking at every byte. This is a
// small loss (~5% performance, ~0.1% density) for compressible data
// due to more bookkeeping, but for non-compressible data (such as
// JPEG) it's a huge win since the compressor quickly "realizes" the
// data is incompressible and doesn't bother looking for matches
// everywhere.
//
// The "skip" variable keeps track of how many bytes there are since
// the last match; dividing it by 32 (ie. right-shifting by five) gives
// the number of bytes to move ahead for each iteration.
skip := int32(32)
nextS := s
var candidate tableEntry
var candidateAlt tableEntry
for {
s = nextS
bytesBetweenHashLookups := skip >> 5
nextS = s + bytesBetweenHashLookups
skip += bytesBetweenHashLookups
if nextS > sLimit {
goto emitRemainder
}
candidates := e.table[nextHash&tableMask]
now := load3232(src, nextS)
e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}}
nextHash = hash(now)
// Check both candidates
candidate = candidates.Cur
if cv == candidate.val {
offset := s - (candidate.offset - e.cur)
if offset < maxMatchOffset {
offset = s - (candidates.Prev.offset - e.cur)
if cv == candidates.Prev.val && offset < maxMatchOffset {
candidateAlt = candidates.Prev
}
break
}
} else {
// We only check if value mismatches.
// Offset will always be invalid in other cases.
candidate = candidates.Prev
if cv == candidate.val {
offset := s - (candidate.offset - e.cur)
if offset < maxMatchOffset {
break
}
}
}
cv = now
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
emitLiteral(dst, src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
// Extend the 4-byte match as long as possible.
//
s += 4
t := candidate.offset - e.cur + 4
l := e.matchlen(s, t, src)
// Try alternative candidate if match length < matchLenGood.
if l < matchLenGood-4 && candidateAlt.offset != 0 {
t2 := candidateAlt.offset - e.cur + 4
l2 := e.matchlen(s, t2, src)
if l2 > l {
l = l2
t = t2
}
}
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
dst.n++
s += l
nextEmit = s
if s >= sLimit {
t += l
// Index first pair after match end.
if int(t+4) < len(src) && t > 0 {
cv := load3232(src, t)
nextHash = hash(cv)
e.table[nextHash&tableMask] = tableEntryPrev{
Prev: e.table[nextHash&tableMask].Cur,
Cur: tableEntry{offset: e.cur + t, val: cv},
}
}
goto emitRemainder
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-3 to s. If
// another emitCopy is not our next move, also calculate nextHash
// at s+1. At least on GOARCH=amd64, these three hash calculations
// are faster as one load64 call (with some shifts) instead of
// three load32 calls.
x := load6432(src, s-3)
prevHash := hash(uint32(x))
e.table[prevHash&tableMask] = tableEntryPrev{
Prev: e.table[prevHash&tableMask].Cur,
Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)},
}
x >>= 8
prevHash = hash(uint32(x))
e.table[prevHash&tableMask] = tableEntryPrev{
Prev: e.table[prevHash&tableMask].Cur,
Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)},
}
x >>= 8
prevHash = hash(uint32(x))
e.table[prevHash&tableMask] = tableEntryPrev{
Prev: e.table[prevHash&tableMask].Cur,
Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)},
}
x >>= 8
currHash := hash(uint32(x))
candidates := e.table[currHash&tableMask]
cv = uint32(x)
e.table[currHash&tableMask] = tableEntryPrev{
Prev: candidates.Cur,
Cur: tableEntry{offset: s + e.cur, val: cv},
}
// Check both candidates
candidate = candidates.Cur
candidateAlt = tableEntry{}
if cv == candidate.val {
offset := s - (candidate.offset - e.cur)
if offset <= maxMatchOffset {
offset = s - (candidates.Prev.offset - e.cur)
if cv == candidates.Prev.val && offset <= maxMatchOffset {
candidateAlt = candidates.Prev
}
continue
}
} else {
// We only check if value mismatches.
// Offset will always be invalid in other cases.
candidate = candidates.Prev
if cv == candidate.val {
offset := s - (candidate.offset - e.cur)
if offset <= maxMatchOffset {
continue
}
}
}
cv = uint32(x >> 8)
nextHash = hash(cv)
s++
break
}
}
emitRemainder:
if int(nextEmit) < len(src) {
emitLiteral(dst, src[nextEmit:])
}
e.cur += int32(len(src))
e.prev = e.prev[:len(src)]
copy(e.prev, src)
}
func (e *snappyGen) matchlen(s, t int32, src []byte) int32 {
s1 := int(s) + maxMatchLength - 4
if s1 > len(src) {
s1 = len(src)
}
// If we are inside the current block
if t >= 0 {
b := src[t:]
a := src[s:s1]
b = b[:len(a)]
// Extend the match to be as long as possible.
for i := range a {
if a[i] != b[i] {
return int32(i)
}
}
return int32(len(a))
}
// We found a match in the previous block.
tp := int32(len(e.prev)) + t
if tp < 0 {
return 0
}
// Extend the match to be as long as possible.
a := src[s:s1]
b := e.prev[tp:]
if len(b) > len(a) {
b = b[:len(a)]
}
a = a[:len(b)]
for i := range b {
if a[i] != b[i] {
return int32(i)
}
}
// If we reached our limit, we matched everything we are
// allowed to in the previous block and we return.
n := int32(len(b))
if int(s+n) == s1 {
return n
}
// Continue looking for more matches in the current block.
a = src[s+n : s1]
b = src[:len(a)]
for i := range a {
if a[i] != b[i] {
return int32(i) + n
}
}
return int32(len(a)) + n
}
// Reset the encoding table.
func (e *snappyGen) Reset() {
e.prev = e.prev[:0]
e.cur += maxMatchOffset
}
vendor/github.com/klauspost/compress/flate/token.go
Generated Vendored
+115
View File
@@ -0,0 +1,115 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import "fmt"
const (
// 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused
// 8 bits: xlength = length - MIN_MATCH_LENGTH
// 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal
lengthShift = 22
offsetMask = 1<<lengthShift - 1
typeMask = 3 << 30
literalType = 0 << 30
matchType = 1 << 30
)
// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
// is lengthCodes[length - MIN_MATCH_LENGTH]
var lengthCodes = [...]uint32{
0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 28,
}
var offsetCodes = [...]uint32{
0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
}
type token uint32
type tokens struct {
tokens [maxStoreBlockSize + 1]token
n uint16 // Must be able to contain maxStoreBlockSize
}
// Convert a literal into a literal token.
func literalToken(literal uint32) token { return token(literalType + literal) }
// Convert a < xlength, xoffset > pair into a match token.
func matchToken(xlength uint32, xoffset uint32) token {
return token(matchType + xlength<<lengthShift + xoffset)
}
func matchTokend(xlength uint32, xoffset uint32) token {
if xlength > maxMatchLength || xoffset > maxMatchOffset {
panic(fmt.Sprintf("Invalid match: len: %d, offset: %d\n", xlength, xoffset))
return token(matchType)
}
return token(matchType + xlength<<lengthShift + xoffset)
}
// Returns the type of a token
func (t token) typ() uint32 { return uint32(t) & typeMask }
// Returns the literal of a literal token
func (t token) literal() uint32 { return uint32(t - literalType) }
// Returns the extra offset of a match token
func (t token) offset() uint32 { return uint32(t) & offsetMask }
func (t token) length() uint32 { return uint32((t - matchType) >> lengthShift) }
func lengthCode(len uint32) uint32 { return lengthCodes[len] }
// Returns the offset code corresponding to a specific offset
func offsetCode(off uint32) uint32 {
if off < uint32(len(offsetCodes)) {
return offsetCodes[off]
} else if off>>7 < uint32(len(offsetCodes)) {
return offsetCodes[off>>7] + 14
} else {
return offsetCodes[off>>14] + 28
}
}
vendor/github.com/klauspost/compress/gzip/gunzip.go
Generated Vendored
+344
View File
@@ -0,0 +1,344 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package gzip implements reading and writing of gzip format compressed files,
// as specified in RFC 1952.
package gzip
import (
"bufio"
"encoding/binary"
"errors"
"hash/crc32"
"io"
"time"
"github.com/klauspost/compress/flate"
)
const (
gzipID1 = 0x1f
gzipID2 = 0x8b
gzipDeflate = 8
flagText = 1 << 0
flagHdrCrc = 1 << 1
flagExtra = 1 << 2
flagName = 1 << 3
flagComment = 1 << 4
)
var (
// ErrChecksum is returned when reading GZIP data that has an invalid checksum.
ErrChecksum = errors.New("gzip: invalid checksum")
// ErrHeader is returned when reading GZIP data that has an invalid header.
ErrHeader = errors.New("gzip: invalid header")
)
var le = binary.LittleEndian
// noEOF converts io.EOF to io.ErrUnexpectedEOF.
func noEOF(err error) error {
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}
// The gzip file stores a header giving metadata about the compressed file.
// That header is exposed as the fields of the Writer and Reader structs.
//
// Strings must be UTF-8 encoded and may only contain Unicode code points
// U+0001 through U+00FF, due to limitations of the GZIP file format.
type Header struct {
Comment string // comment
Extra []byte // "extra data"
ModTime time.Time // modification time
Name string // file name
OS byte // operating system type
}
// A Reader is an io.Reader that can be read to retrieve
// uncompressed data from a gzip-format compressed file.
//
// In general, a gzip file can be a concatenation of gzip files,
// each with its own header. Reads from the Reader
// return the concatenation of the uncompressed data of each.
// Only the first header is recorded in the Reader fields.
//
// Gzip files store a length and checksum of the uncompressed data.
// The Reader will return a ErrChecksum when Read
// reaches the end of the uncompressed data if it does not
// have the expected length or checksum. Clients should treat data
// returned by Read as tentative until they receive the io.EOF
// marking the end of the data.
type Reader struct {
Header // valid after NewReader or Reader.Reset
r flate.Reader
decompressor io.ReadCloser
digest uint32 // CRC-32, IEEE polynomial (section 8)
size uint32 // Uncompressed size (section 2.3.1)
buf [512]byte
err error
multistream bool
}
// NewReader creates a new Reader reading the given reader.
// If r does not also implement io.ByteReader,
// the decompressor may read more data than necessary from r.
//
// It is the caller's responsibility to call Close on the Reader when done.
//
// The Reader.Header fields will be valid in the Reader returned.
func NewReader(r io.Reader) (*Reader, error) {
z := new(Reader)
if err := z.Reset(r); err != nil {
return nil, err
}
return z, nil
}
// Reset discards the Reader z's state and makes it equivalent to the
// result of its original state from NewReader, but reading from r instead.
// This permits reusing a Reader rather than allocating a new one.
func (z *Reader) Reset(r io.Reader) error {
*z = Reader{
decompressor: z.decompressor,
multistream: true,
}
if rr, ok := r.(flate.Reader); ok {
z.r = rr
} else {
z.r = bufio.NewReader(r)
}
z.Header, z.err = z.readHeader()
return z.err
}
// Multistream controls whether the reader supports multistream files.
//
// If enabled (the default), the Reader expects the input to be a sequence
// of individually gzipped data streams, each with its own header and
// trailer, ending at EOF. The effect is that the concatenation of a sequence
// of gzipped files is treated as equivalent to the gzip of the concatenation
// of the sequence. This is standard behavior for gzip readers.
//
// Calling Multistream(false) disables this behavior; disabling the behavior
// can be useful when reading file formats that distinguish individual gzip
// data streams or mix gzip data streams with other data streams.
// In this mode, when the Reader reaches the end of the data stream,
// Read returns io.EOF. If the underlying reader implements io.ByteReader,
// it will be left positioned just after the gzip stream.
// To start the next stream, call z.Reset(r) followed by z.Multistream(false).
// If there is no next stream, z.Reset(r) will return io.EOF.
func (z *Reader) Multistream(ok bool) {
z.multistream = ok
}
// readString reads a NUL-terminated string from z.r.
// It treats the bytes read as being encoded as ISO 8859-1 (Latin-1) and
// will output a string encoded using UTF-8.
// This method always updates z.digest with the data read.
func (z *Reader) readString() (string, error) {
var err error
needConv := false
for i := 0; ; i++ {
if i >= len(z.buf) {
return "", ErrHeader
}
z.buf[i], err = z.r.ReadByte()
if err != nil {
return "", err
}
if z.buf[i] > 0x7f {
needConv = true
}
if z.buf[i] == 0 {
// Digest covers the NUL terminator.
z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:i+1])
// Strings are ISO 8859-1, Latin-1 (RFC 1952, section 2.3.1).
if needConv {
s := make([]rune, 0, i)
for _, v := range z.buf[:i] {
s = append(s, rune(v))
}
return string(s), nil
}
return string(z.buf[:i]), nil
}
}
}
// readHeader reads the GZIP header according to section 2.3.1.
// This method does not set z.err.
func (z *Reader) readHeader() (hdr Header, err error) {
if _, err = io.ReadFull(z.r, z.buf[:10]); err != nil {
// RFC 1952, section 2.2, says the following:
// A gzip file consists of a series of "members" (compressed data sets).
//
// Other than this, the specification does not clarify whether a
// "series" is defined as "one or more" or "zero or more". To err on the
// side of caution, Go interprets this to mean "zero or more".
// Thus, it is okay to return io.EOF here.
return hdr, err
}
if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate {
return hdr, ErrHeader
}
flg := z.buf[3]
hdr.ModTime = time.Unix(int64(le.Uint32(z.buf[4:8])), 0)
// z.buf[8] is XFL and is currently ignored.
hdr.OS = z.buf[9]
z.digest = crc32.ChecksumIEEE(z.buf[:10])
if flg&flagExtra != 0 {
if _, err = io.ReadFull(z.r, z.buf[:2]); err != nil {
return hdr, noEOF(err)
}
z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:2])
data := make([]byte, le.Uint16(z.buf[:2]))
if _, err = io.ReadFull(z.r, data); err != nil {
return hdr, noEOF(err)
}
z.digest = crc32.Update(z.digest, crc32.IEEETable, data)
hdr.Extra = data
}
var s string
if flg&flagName != 0 {
if s, err = z.readString(); err != nil {
return hdr, err
}
hdr.Name = s
}
if flg&flagComment != 0 {
if s, err = z.readString(); err != nil {
return hdr, err
}
hdr.Comment = s
}
if flg&flagHdrCrc != 0 {
if _, err = io.ReadFull(z.r, z.buf[:2]); err != nil {
return hdr, noEOF(err)
}
digest := le.Uint16(z.buf[:2])
if digest != uint16(z.digest) {
return hdr, ErrHeader
}
}
z.digest = 0
if z.decompressor == nil {
z.decompressor = flate.NewReader(z.r)
} else {
z.decompressor.(flate.Resetter).Reset(z.r, nil)
}
return hdr, nil
}
// Read implements io.Reader, reading uncompressed bytes from its underlying Reader.
func (z *Reader) Read(p []byte) (n int, err error) {
if z.err != nil {
return 0, z.err
}
n, z.err = z.decompressor.Read(p)
z.digest = crc32.Update(z.digest, crc32.IEEETable, p[:n])
z.size += uint32(n)
if z.err != io.EOF {
// In the normal case we return here.
return n, z.err
}
// Finished file; check checksum and size.
if _, err := io.ReadFull(z.r, z.buf[:8]); err != nil {
z.err = noEOF(err)
return n, z.err
}
digest := le.Uint32(z.buf[:4])
size := le.Uint32(z.buf[4:8])
if digest != z.digest || size != z.size {
z.err = ErrChecksum
return n, z.err
}
z.digest, z.size = 0, 0
// File is ok; check if there is another.
if !z.multistream {
return n, io.EOF
}
z.err = nil // Remove io.EOF
if _, z.err = z.readHeader(); z.err != nil {
return n, z.err
}
// Read from next file, if necessary.
if n > 0 {
return n, nil
}
return z.Read(p)
}
// Support the io.WriteTo interface for io.Copy and friends.
func (z *Reader) WriteTo(w io.Writer) (int64, error) {
total := int64(0)
crcWriter := crc32.NewIEEE()
for {
if z.err != nil {
if z.err == io.EOF {
return total, nil
}
return total, z.err
}
// We write both to output and digest.
mw := io.MultiWriter(w, crcWriter)
n, err := z.decompressor.(io.WriterTo).WriteTo(mw)
total += n
z.size += uint32(n)
if err != nil {
z.err = err
return total, z.err
}
// Finished file; check checksum + size.
if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
z.err = err
return total, err
}
z.digest = crcWriter.Sum32()
digest := le.Uint32(z.buf[:4])
size := le.Uint32(z.buf[4:8])
if digest != z.digest || size != z.size {
z.err = ErrChecksum
return total, z.err
}
z.digest, z.size = 0, 0
// File is ok; check if there is another.
if !z.multistream {
return total, nil
}
crcWriter.Reset()
z.err = nil // Remove io.EOF
if _, z.err = z.readHeader(); z.err != nil {
if z.err == io.EOF {
return total, nil
}
return total, z.err
}
}
}
// Close closes the Reader. It does not close the underlying io.Reader.
// In order for the GZIP checksum to be verified, the reader must be
// fully consumed until the io.EOF.
func (z *Reader) Close() error { return z.decompressor.Close() }
vendor/github.com/klauspost/compress/gzip/gzip.go
Generated Vendored
+251
View File
@@ -0,0 +1,251 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gzip
import (
"errors"
"fmt"
"hash/crc32"
"io"
"github.com/klauspost/compress/flate"
)
// These constants are copied from the flate package, so that code that imports
// "compress/gzip" does not also have to import "compress/flate".
const (
NoCompression = flate.NoCompression
BestSpeed = flate.BestSpeed
BestCompression = flate.BestCompression
DefaultCompression = flate.DefaultCompression
ConstantCompression = flate.ConstantCompression
HuffmanOnly = flate.HuffmanOnly
)
// A Writer is an io.WriteCloser.
// Writes to a Writer are compressed and written to w.
type Writer struct {
Header // written at first call to Write, Flush, or Close
w io.Writer
level int
wroteHeader bool
compressor *flate.Writer
digest uint32 // CRC-32, IEEE polynomial (section 8)
size uint32 // Uncompressed size (section 2.3.1)
closed bool
buf [10]byte
err error
}
// NewWriter returns a new Writer.
// Writes to the returned writer are compressed and written to w.
//
// It is the caller's responsibility to call Close on the WriteCloser when done.
// Writes may be buffered and not flushed until Close.
//
// Callers that wish to set the fields in Writer.Header must do so before
// the first call to Write, Flush, or Close.
func NewWriter(w io.Writer) *Writer {
z, _ := NewWriterLevel(w, DefaultCompression)
return z
}
// NewWriterLevel is like NewWriter but specifies the compression level instead
// of assuming DefaultCompression.
//
// The compression level can be DefaultCompression, NoCompression, or any
// integer value between BestSpeed and BestCompression inclusive. The error
// returned will be nil if the level is valid.
func NewWriterLevel(w io.Writer, level int) (*Writer, error) {
if level < HuffmanOnly || level > BestCompression {
return nil, fmt.Errorf("gzip: invalid compression level: %d", level)
}
z := new(Writer)
z.init(w, level)
return z, nil
}
func (z *Writer) init(w io.Writer, level int) {
compressor := z.compressor
if compressor != nil {
compressor.Reset(w)
}
*z = Writer{
Header: Header{
OS: 255, // unknown
},
w: w,
level: level,
compressor: compressor,
}
}
// Reset discards the Writer z's state and makes it equivalent to the
// result of its original state from NewWriter or NewWriterLevel, but
// writing to w instead. This permits reusing a Writer rather than
// allocating a new one.
func (z *Writer) Reset(w io.Writer) {
z.init(w, z.level)
}
// writeBytes writes a length-prefixed byte slice to z.w.
func (z *Writer) writeBytes(b []byte) error {
if len(b) > 0xffff {
return errors.New("gzip.Write: Extra data is too large")
}
le.PutUint16(z.buf[:2], uint16(len(b)))
_, err := z.w.Write(z.buf[:2])
if err != nil {
return err
}
_, err = z.w.Write(b)
return err
}
// writeString writes a UTF-8 string s in GZIP's format to z.w.
// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
func (z *Writer) writeString(s string) (err error) {
// GZIP stores Latin-1 strings; error if non-Latin-1; convert if non-ASCII.
needconv := false
for _, v := range s {
if v == 0 || v > 0xff {
return errors.New("gzip.Write: non-Latin-1 header string")
}
if v > 0x7f {
needconv = true
}
}
if needconv {
b := make([]byte, 0, len(s))
for _, v := range s {
b = append(b, byte(v))
}
_, err = z.w.Write(b)
} else {
_, err = io.WriteString(z.w, s)
}
if err != nil {
return err
}
// GZIP strings are NUL-terminated.
z.buf[0] = 0
_, err = z.w.Write(z.buf[:1])
return err
}
// Write writes a compressed form of p to the underlying io.Writer. The
// compressed bytes are not necessarily flushed until the Writer is closed.
func (z *Writer) Write(p []byte) (int, error) {
if z.err != nil {
return 0, z.err
}
var n int
// Write the GZIP header lazily.
if !z.wroteHeader {
z.wroteHeader = true
z.buf[0] = gzipID1
z.buf[1] = gzipID2
z.buf[2] = gzipDeflate
z.buf[3] = 0
if z.Extra != nil {
z.buf[3] |= 0x04
}
if z.Name != "" {
z.buf[3] |= 0x08
}
if z.Comment != "" {
z.buf[3] |= 0x10
}
le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix()))
if z.level == BestCompression {
z.buf[8] = 2
} else if z.level == BestSpeed {
z.buf[8] = 4
} else {
z.buf[8] = 0
}
z.buf[9] = z.OS
n, z.err = z.w.Write(z.buf[:10])
if z.err != nil {
return n, z.err
}
if z.Extra != nil {
z.err = z.writeBytes(z.Extra)
if z.err != nil {
return n, z.err
}
}
if z.Name != "" {
z.err = z.writeString(z.Name)
if z.err != nil {
return n, z.err
}
}
if z.Comment != "" {
z.err = z.writeString(z.Comment)
if z.err != nil {
return n, z.err
}
}
if z.compressor == nil {
z.compressor, _ = flate.NewWriter(z.w, z.level)
}
}
z.size += uint32(len(p))
z.digest = crc32.Update(z.digest, crc32.IEEETable, p)
n, z.err = z.compressor.Write(p)
return n, z.err
}
// Flush flushes any pending compressed data to the underlying writer.
//
// It is useful mainly in compressed network protocols, to ensure that
// a remote reader has enough data to reconstruct a packet. Flush does
// not return until the data has been written. If the underlying
// writer returns an error, Flush returns that error.
//
// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
func (z *Writer) Flush() error {
if z.err != nil {
return z.err
}
if z.closed {
return nil
}
if !z.wroteHeader {
z.Write(nil)
if z.err != nil {
return z.err
}
}
z.err = z.compressor.Flush()
return z.err
}
// Close closes the Writer, flushing any unwritten data to the underlying
// io.Writer, but does not close the underlying io.Writer.
func (z *Writer) Close() error {
if z.err != nil {
return z.err
}
if z.closed {
return nil
}
z.closed = true
if !z.wroteHeader {
z.Write(nil)
if z.err != nil {
return z.err
}
}
z.err = z.compressor.Close()
if z.err != nil {
return z.err
}
le.PutUint32(z.buf[:4], z.digest)
le.PutUint32(z.buf[4:8], z.size)
_, z.err = z.w.Write(z.buf[:8])
return z.err
}
vendor/github.com/klauspost/compress/zlib/reader.go
Generated Vendored
+178
View File
@@ -0,0 +1,178 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package zlib implements reading and writing of zlib format compressed data,
as specified in RFC 1950.
The implementation provides filters that uncompress during reading
and compress during writing. For example, to write compressed data
to a buffer:
var b bytes.Buffer
w := zlib.NewWriter(&b)
w.Write([]byte("hello, world\n"))
w.Close()
and to read that data back:
r, err := zlib.NewReader(&b)
io.Copy(os.Stdout, r)
r.Close()
*/
package zlib
import (
"bufio"
"errors"
"hash"
"hash/adler32"
"io"
"github.com/klauspost/compress/flate"
)
const zlibDeflate = 8
var (
// ErrChecksum is returned when reading ZLIB data that has an invalid checksum.
ErrChecksum = errors.New("zlib: invalid checksum")
// ErrDictionary is returned when reading ZLIB data that has an invalid dictionary.
ErrDictionary = errors.New("zlib: invalid dictionary")
// ErrHeader is returned when reading ZLIB data that has an invalid header.
ErrHeader = errors.New("zlib: invalid header")
)
type reader struct {
r flate.Reader
decompressor io.ReadCloser
digest hash.Hash32
err error
scratch [4]byte
}
// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to
// to switch to a new underlying Reader. This permits reusing a ReadCloser
// instead of allocating a new one.
type Resetter interface {
// Reset discards any buffered data and resets the Resetter as if it was
// newly initialized with the given reader.
Reset(r io.Reader, dict []byte) error
}
// NewReader creates a new ReadCloser.
// Reads from the returned ReadCloser read and decompress data from r.
// If r does not implement io.ByteReader, the decompressor may read more
// data than necessary from r.
// It is the caller's responsibility to call Close on the ReadCloser when done.
//
// The ReadCloser returned by NewReader also implements Resetter.
func NewReader(r io.Reader) (io.ReadCloser, error) {
return NewReaderDict(r, nil)
}
// NewReaderDict is like NewReader but uses a preset dictionary.
// NewReaderDict ignores the dictionary if the compressed data does not refer to it.
// If the compressed data refers to a different dictionary, NewReaderDict returns ErrDictionary.
//
// The ReadCloser returned by NewReaderDict also implements Resetter.
func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, error) {
z := new(reader)
err := z.Reset(r, dict)
if err != nil {
return nil, err
}
return z, nil
}
func (z *reader) Read(p []byte) (int, error) {
if z.err != nil {
return 0, z.err
}
var n int
n, z.err = z.decompressor.Read(p)
z.digest.Write(p[0:n])
if z.err != io.EOF {
// In the normal case we return here.
return n, z.err
}
// Finished file; check checksum.
if _, err := io.ReadFull(z.r, z.scratch[0:4]); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
z.err = err
return n, z.err
}
// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
if checksum != z.digest.Sum32() {
z.err = ErrChecksum
return n, z.err
}
return n, io.EOF
}
// Calling Close does not close the wrapped io.Reader originally passed to NewReader.
// In order for the ZLIB checksum to be verified, the reader must be
// fully consumed until the io.EOF.
func (z *reader) Close() error {
if z.err != nil && z.err != io.EOF {
return z.err
}
z.err = z.decompressor.Close()
return z.err
}
func (z *reader) Reset(r io.Reader, dict []byte) error {
*z = reader{decompressor: z.decompressor}
if fr, ok := r.(flate.Reader); ok {
z.r = fr
} else {
z.r = bufio.NewReader(r)
}
// Read the header (RFC 1950 section 2.2.).
_, z.err = io.ReadFull(z.r, z.scratch[0:2])
if z.err != nil {
if z.err == io.EOF {
z.err = io.ErrUnexpectedEOF
}
return z.err
}
h := uint(z.scratch[0])<<8 | uint(z.scratch[1])
if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) {
z.err = ErrHeader
return z.err
}
haveDict := z.scratch[1]&0x20 != 0
if haveDict {
_, z.err = io.ReadFull(z.r, z.scratch[0:4])
if z.err != nil {
if z.err == io.EOF {
z.err = io.ErrUnexpectedEOF
}
return z.err
}
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
if checksum != adler32.Checksum(dict) {
z.err = ErrDictionary
return z.err
}
}
if z.decompressor == nil {
if haveDict {
z.decompressor = flate.NewReaderDict(z.r, dict)
} else {
z.decompressor = flate.NewReader(z.r)
}
} else {
z.decompressor.(flate.Resetter).Reset(z.r, dict)
}
z.digest = adler32.New()
return nil
}
vendor/github.com/klauspost/compress/zlib/writer.go
Generated Vendored
+201
View File
@@ -0,0 +1,201 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package zlib
import (
"fmt"
"hash"
"hash/adler32"
"io"
"github.com/klauspost/compress/flate"
)
// These constants are copied from the flate package, so that code that imports
// "compress/zlib" does not also have to import "compress/flate".
const (
NoCompression = flate.NoCompression
BestSpeed = flate.BestSpeed
BestCompression = flate.BestCompression
DefaultCompression = flate.DefaultCompression
ConstantCompression = flate.ConstantCompression
HuffmanOnly = flate.HuffmanOnly
)
// A Writer takes data written to it and writes the compressed
// form of that data to an underlying writer (see NewWriter).
type Writer struct {
w io.Writer
level int
dict []byte
compressor *flate.Writer
digest hash.Hash32
err error
scratch [4]byte
wroteHeader bool
}
// NewWriter creates a new Writer.
// Writes to the returned Writer are compressed and written to w.
//
// It is the caller's responsibility to call Close on the WriteCloser when done.
// Writes may be buffered and not flushed until Close.
func NewWriter(w io.Writer) *Writer {
z, _ := NewWriterLevelDict(w, DefaultCompression, nil)
return z
}
// NewWriterLevel is like NewWriter but specifies the compression level instead
// of assuming DefaultCompression.
//
// The compression level can be DefaultCompression, NoCompression, HuffmanOnly
// or any integer value between BestSpeed and BestCompression inclusive.
// The error returned will be nil if the level is valid.
func NewWriterLevel(w io.Writer, level int) (*Writer, error) {
return NewWriterLevelDict(w, level, nil)
}
// NewWriterLevelDict is like NewWriterLevel but specifies a dictionary to
// compress with.
//
// The dictionary may be nil. If not, its contents should not be modified until
// the Writer is closed.
func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) {
if level < HuffmanOnly || level > BestCompression {
return nil, fmt.Errorf("zlib: invalid compression level: %d", level)
}
return &Writer{
w: w,
level: level,
dict: dict,
}, nil
}
// Reset clears the state of the Writer z such that it is equivalent to its
// initial state from NewWriterLevel or NewWriterLevelDict, but instead writing
// to w.
func (z *Writer) Reset(w io.Writer) {
z.w = w
// z.level and z.dict left unchanged.
if z.compressor != nil {
z.compressor.Reset(w)
}
if z.digest != nil {
z.digest.Reset()
}
z.err = nil
z.scratch = [4]byte{}
z.wroteHeader = false
}
// writeHeader writes the ZLIB header.
func (z *Writer) writeHeader() (err error) {
z.wroteHeader = true
// ZLIB has a two-byte header (as documented in RFC 1950).
// The first four bits is the CINFO (compression info), which is 7 for the default deflate window size.
// The next four bits is the CM (compression method), which is 8 for deflate.
z.scratch[0] = 0x78
// The next two bits is the FLEVEL (compression level). The four values are:
// 0=fastest, 1=fast, 2=default, 3=best.
// The next bit, FDICT, is set if a dictionary is given.
// The final five FCHECK bits form a mod-31 checksum.
switch z.level {
case -2, 0, 1:
z.scratch[1] = 0 << 6
case 2, 3, 4, 5:
z.scratch[1] = 1 << 6
case 6, -1:
z.scratch[1] = 2 << 6
case 7, 8, 9:
z.scratch[1] = 3 << 6
default:
panic("unreachable")
}
if z.dict != nil {
z.scratch[1] |= 1 << 5
}
z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31)
if _, err = z.w.Write(z.scratch[0:2]); err != nil {
return err
}
if z.dict != nil {
// The next four bytes are the Adler-32 checksum of the dictionary.
checksum := adler32.Checksum(z.dict)
z.scratch[0] = uint8(checksum >> 24)
z.scratch[1] = uint8(checksum >> 16)
z.scratch[2] = uint8(checksum >> 8)
z.scratch[3] = uint8(checksum >> 0)
if _, err = z.w.Write(z.scratch[0:4]); err != nil {
return err
}
}
if z.compressor == nil {
// Initialize deflater unless the Writer is being reused
// after a Reset call.
z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict)
if err != nil {
return err
}
z.digest = adler32.New()
}
return nil
}
// Write writes a compressed form of p to the underlying io.Writer. The
// compressed bytes are not necessarily flushed until the Writer is closed or
// explicitly flushed.
func (z *Writer) Write(p []byte) (n int, err error) {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil {
return 0, z.err
}
if len(p) == 0 {
return 0, nil
}
n, err = z.compressor.Write(p)
if err != nil {
z.err = err
return
}
z.digest.Write(p)
return
}
// Flush flushes the Writer to its underlying io.Writer.
func (z *Writer) Flush() error {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil {
return z.err
}
z.err = z.compressor.Flush()
return z.err
}
// Close closes the Writer, flushing any unwritten data to the underlying
// io.Writer, but does not close the underlying io.Writer.
func (z *Writer) Close() error {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil {
return z.err
}
z.err = z.compressor.Close()
if z.err != nil {
return z.err
}
checksum := z.digest.Sum32()
// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
z.scratch[0] = uint8(checksum >> 24)
z.scratch[1] = uint8(checksum >> 16)
z.scratch[2] = uint8(checksum >> 8)
z.scratch[3] = uint8(checksum >> 0)
_, z.err = z.w.Write(z.scratch[0:4])
return z.err
}
vendor/github.com/klauspost/cpuid/LICENSE
Generated Vendored
+22
View File
@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Klaus Post
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
vendor/github.com/klauspost/cpuid/README.md
Generated Vendored
+145
View File
@@ -0,0 +1,145 @@
# cpuid
Package cpuid provides information about the CPU running the current program.
CPU features are detected on startup, and kept for fast access through the life of the application.
Currently x86 / x64 (AMD64) is supported, and no external C (cgo) code is used, which should make the library very easy to use.
You can access the CPU information by accessing the shared CPU variable of the cpuid library.
Package home: https://github.com/klauspost/cpuid
[![GoDoc][1]][2] [![Build Status][3]][4]
[1]: https://godoc.org/github.com/klauspost/cpuid?status.svg
[2]: https://godoc.org/github.com/klauspost/cpuid
[3]: https://travis-ci.org/klauspost/cpuid.svg
[4]: https://travis-ci.org/klauspost/cpuid
# features
## CPU Instructions
* **CMOV** (i686 CMOV)
* **NX** (NX (No-Execute) bit)
* **AMD3DNOW** (AMD 3DNOW)
* **AMD3DNOWEXT** (AMD 3DNowExt)
* **MMX** (standard MMX)
* **MMXEXT** (SSE integer functions or AMD MMX ext)
* **SSE** (SSE functions)
* **SSE2** (P4 SSE functions)
* **SSE3** (Prescott SSE3 functions)
* **SSSE3** (Conroe SSSE3 functions)
* **SSE4** (Penryn SSE4.1 functions)
* **SSE4A** (AMD Barcelona microarchitecture SSE4a instructions)
* **SSE42** (Nehalem SSE4.2 functions)
* **AVX** (AVX functions)
* **AVX2** (AVX2 functions)
* **FMA3** (Intel FMA 3)
* **FMA4** (Bulldozer FMA4 functions)
* **XOP** (Bulldozer XOP functions)
* **F16C** (Half-precision floating-point conversion)
* **BMI1** (Bit Manipulation Instruction Set 1)
* **BMI2** (Bit Manipulation Instruction Set 2)
* **TBM** (AMD Trailing Bit Manipulation)
* **LZCNT** (LZCNT instruction)
* **POPCNT** (POPCNT instruction)
* **AESNI** (Advanced Encryption Standard New Instructions)
* **CLMUL** (Carry-less Multiplication)
* **HTT** (Hyperthreading (enabled))
* **HLE** (Hardware Lock Elision)
* **RTM** (Restricted Transactional Memory)
* **RDRAND** (RDRAND instruction is available)
* **RDSEED** (RDSEED instruction is available)
* **ADX** (Intel ADX (Multi-Precision Add-Carry Instruction Extensions))
* **SHA** (Intel SHA Extensions)
* **AVX512F** (AVX-512 Foundation)
* **AVX512DQ** (AVX-512 Doubleword and Quadword Instructions)
* **AVX512IFMA** (AVX-512 Integer Fused Multiply-Add Instructions)
* **AVX512PF** (AVX-512 Prefetch Instructions)
* **AVX512ER** (AVX-512 Exponential and Reciprocal Instructions)
* **AVX512CD** (AVX-512 Conflict Detection Instructions)
* **AVX512BW** (AVX-512 Byte and Word Instructions)
* **AVX512VL** (AVX-512 Vector Length Extensions)
* **AVX512VBMI** (AVX-512 Vector Bit Manipulation Instructions)
* **MPX** (Intel MPX (Memory Protection Extensions))
* **ERMS** (Enhanced REP MOVSB/STOSB)
* **RDTSCP** (RDTSCP Instruction)
* **CX16** (CMPXCHG16B Instruction)
* **SGX** (Software Guard Extensions, with activation details)
## Performance
* **RDTSCP()** Returns current cycle count. Can be used for benchmarking.
* **SSE2SLOW** (SSE2 is supported, but usually not faster)
* **SSE3SLOW** (SSE3 is supported, but usually not faster)
* **ATOM** (Atom processor, some SSSE3 instructions are slower)
* **Cache line** (Probable size of a cache line).
* **L1, L2, L3 Cache size** on newer Intel/AMD CPUs.
## Cpu Vendor/VM
* **Intel**
* **AMD**
* **VIA**
* **Transmeta**
* **NSC**
* **KVM** (Kernel-based Virtual Machine)
* **MSVM** (Microsoft Hyper-V or Windows Virtual PC)
* **VMware**
* **XenHVM**
# installing
```go get github.com/klauspost/cpuid```
# example
```Go
package main
import (
"fmt"
"github.com/klauspost/cpuid"
)
func main() {
// Print basic CPU information:
fmt.Println("Name:", cpuid.CPU.BrandName)
fmt.Println("PhysicalCores:", cpuid.CPU.PhysicalCores)
fmt.Println("ThreadsPerCore:", cpuid.CPU.ThreadsPerCore)
fmt.Println("LogicalCores:", cpuid.CPU.LogicalCores)
fmt.Println("Family", cpuid.CPU.Family, "Model:", cpuid.CPU.Model)
fmt.Println("Features:", cpuid.CPU.Features)
fmt.Println("Cacheline bytes:", cpuid.CPU.CacheLine)
fmt.Println("L1 Data Cache:", cpuid.CPU.Cache.L1D, "bytes")
fmt.Println("L1 Instruction Cache:", cpuid.CPU.Cache.L1D, "bytes")
fmt.Println("L2 Cache:", cpuid.CPU.Cache.L2, "bytes")
fmt.Println("L3 Cache:", cpuid.CPU.Cache.L3, "bytes")
// Test if we have a specific feature:
if cpuid.CPU.SSE() {
fmt.Println("We have Streaming SIMD Extensions")
}
}
```
Sample output:
```
>go run main.go
Name: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
PhysicalCores: 2
ThreadsPerCore: 2
LogicalCores: 4
Family 6 Model: 42
Features: CMOV,MMX,MMXEXT,SSE,SSE2,SSE3,SSSE3,SSE4.1,SSE4.2,AVX,AESNI,CLMUL
Cacheline bytes: 64
We have Streaming SIMD Extensions
```
# private package
In the "private" folder you can find an autogenerated version of the library you can include in your own packages.
For this purpose all exports are removed, and functions and constants are lowercased.
This is not a recommended way of using the library, but provided for convenience, if it is difficult for you to use external packages.
# license
This code is published under an MIT license. See LICENSE file for more information.
vendor/github.com/klauspost/cpuid/cpuid.go
Generated Vendored
+1022
View File
File diff suppressed because it is too large. Load diff
vendor/github.com/klauspost/cpuid/cpuid_386.s
Generated Vendored
+42
View File
@@ -0,0 +1,42 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
// +build 386,!gccgo
// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
TEXT ·asmCpuid(SB), 7, $0
XORL CX, CX
MOVL op+0(FP), AX
CPUID
MOVL AX, eax+4(FP)
MOVL BX, ebx+8(FP)
MOVL CX, ecx+12(FP)
MOVL DX, edx+16(FP)
RET
// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
TEXT ·asmCpuidex(SB), 7, $0
MOVL op+0(FP), AX
MOVL op2+4(FP), CX
CPUID
MOVL AX, eax+8(FP)
MOVL BX, ebx+12(FP)
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET
// func xgetbv(index uint32) (eax, edx uint32)
TEXT ·asmXgetbv(SB), 7, $0
MOVL index+0(FP), CX
BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV
MOVL AX, eax+4(FP)
MOVL DX, edx+8(FP)
RET
// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
TEXT ·asmRdtscpAsm(SB), 7, $0
BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP
MOVL AX, eax+0(FP)
MOVL BX, ebx+4(FP)
MOVL CX, ecx+8(FP)
MOVL DX, edx+12(FP)
RET
vendor/github.com/klauspost/cpuid/cpuid_amd64.s
Generated Vendored
+42
View File
@@ -0,0 +1,42 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
//+build amd64,!gccgo
// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
TEXT ·asmCpuid(SB), 7, $0
XORQ CX, CX
MOVL op+0(FP), AX
CPUID
MOVL AX, eax+8(FP)
MOVL BX, ebx+12(FP)
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET
// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
TEXT ·asmCpuidex(SB), 7, $0
MOVL op+0(FP), AX
MOVL op2+4(FP), CX
CPUID
MOVL AX, eax+8(FP)
MOVL BX, ebx+12(FP)
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET
// func asmXgetbv(index uint32) (eax, edx uint32)
TEXT ·asmXgetbv(SB), 7, $0
MOVL index+0(FP), CX
BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV
MOVL AX, eax+8(FP)
MOVL DX, edx+12(FP)
RET
// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
TEXT ·asmRdtscpAsm(SB), 7, $0
BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP
MOVL AX, eax+0(FP)
MOVL BX, ebx+4(FP)
MOVL CX, ecx+8(FP)
MOVL DX, edx+12(FP)
RET
vendor/github.com/klauspost/cpuid/detect_intel.go
Generated Vendored
+17
View File
@@ -0,0 +1,17 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
// +build 386,!gccgo amd64,!gccgo
package cpuid
func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
func asmXgetbv(index uint32) (eax, edx uint32)
func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
func initCPU() {
cpuid = asmCpuid
cpuidex = asmCpuidex
xgetbv = asmXgetbv
rdtscpAsm = asmRdtscpAsm
}
vendor/github.com/klauspost/cpuid/detect_ref.go
Generated Vendored
+23
View File
@@ -0,0 +1,23 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
// +build !amd64,!386 gccgo
package cpuid
func initCPU() {
cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) {
return 0, 0, 0, 0
}
cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) {
return 0, 0, 0, 0
}
xgetbv = func(index uint32) (eax, edx uint32) {
return 0, 0
}
rdtscpAsm = func() (eax, ebx, ecx, edx uint32) {
return 0, 0, 0, 0
}
}
vendor/github.com/klauspost/cpuid/generate.go
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
package cpuid
//go:generate go run private-gen.go
vendor/github.com/klauspost/cpuid/private-gen.go
Generated Vendored
+476
View File
@@ -0,0 +1,476 @@
// +build ignore
package main
import (
"bytes"
"fmt"
"go/ast"
"go/parser"
"go/printer"
"go/token"
"io"
"io/ioutil"
"log"
"os"
"reflect"
"strings"
"unicode"
"unicode/utf8"
)
var inFiles = []string{"cpuid.go", "cpuid_test.go"}
var copyFiles = []string{"cpuid_amd64.s", "cpuid_386.s", "detect_ref.go", "detect_intel.go"}
var fileSet = token.NewFileSet()
var reWrites = []rewrite{
initRewrite("CPUInfo -> cpuInfo"),
initRewrite("Vendor -> vendor"),
initRewrite("Flags -> flags"),
initRewrite("Detect -> detect"),
initRewrite("CPU -> cpu"),
}
var excludeNames = map[string]bool{"string": true, "join": true, "trim": true,
// cpuid_test.go
"t": true, "println": true, "logf": true, "log": true, "fatalf": true, "fatal": true,
}
var excludePrefixes = []string{"test", "benchmark"}
func main() {
Package := "private"
parserMode := parser.ParseComments
exported := make(map[string]rewrite)
for _, file := range inFiles {
in, err := os.Open(file)
if err != nil {
log.Fatalf("opening input", err)
}
src, err := ioutil.ReadAll(in)
if err != nil {
log.Fatalf("reading input", err)
}
astfile, err := parser.ParseFile(fileSet, file, src, parserMode)
if err != nil {
log.Fatalf("parsing input", err)
}
for _, rw := range reWrites {
astfile = rw(astfile)
}
// Inspect the AST and print all identifiers and literals.
var startDecl token.Pos
var endDecl token.Pos
ast.Inspect(astfile, func(n ast.Node) bool {
var s string
switch x := n.(type) {
case *ast.Ident:
if x.IsExported() {
t := strings.ToLower(x.Name)
for _, pre := range excludePrefixes {
if strings.HasPrefix(t, pre) {
return true
}
}
if excludeNames[t] != true {
//if x.Pos() > startDecl && x.Pos() < endDecl {
exported[x.Name] = initRewrite(x.Name + " -> " + t)
}
}
case *ast.GenDecl:
if x.Tok == token.CONST && x.Lparen > 0 {
startDecl = x.Lparen
endDecl = x.Rparen
// fmt.Printf("Decl:%s -> %s\n", fileSet.Position(startDecl), fileSet.Position(endDecl))
}
}
if s != "" {
fmt.Printf("%s:\t%s\n", fileSet.Position(n.Pos()), s)
}
return true
})
for _, rw := range exported {
astfile = rw(astfile)
}
var buf bytes.Buffer
printer.Fprint(&buf, fileSet, astfile)
// Remove package documentation and insert information
s := buf.String()
ind := strings.Index(buf.String(), "\npackage cpuid")
s = s[ind:]
s = "// Generated, DO NOT EDIT,\n" +
"// but copy it to your own project and rename the package.\n" +
"// See more at http://github.com/klauspost/cpuid\n" +
s
outputName := Package + string(os.PathSeparator) + file
err = ioutil.WriteFile(outputName, []byte(s), 0644)
if err != nil {
log.Fatalf("writing output: %s", err)
}
log.Println("Generated", outputName)
}
for _, file := range copyFiles {
dst := ""
if strings.HasPrefix(file, "cpuid") {
dst = Package + string(os.PathSeparator) + file
} else {
dst = Package + string(os.PathSeparator) + "cpuid_" + file
}
err := copyFile(file, dst)
if err != nil {
log.Fatalf("copying file: %s", err)
}
log.Println("Copied", dst)
}
}
// CopyFile copies a file from src to dst. If src and dst files exist, and are
// the same, then return success. Copy the file contents from src to dst.
func copyFile(src, dst string) (err error) {
sfi, err := os.Stat(src)
if err != nil {
return
}
if !sfi.Mode().IsRegular() {
// cannot copy non-regular files (e.g., directories,
// symlinks, devices, etc.)
return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
}
dfi, err := os.Stat(dst)
if err != nil {
if !os.IsNotExist(err) {
return
}
} else {
if !(dfi.Mode().IsRegular()) {
return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
}
if os.SameFile(sfi, dfi) {
return
}
}
err = copyFileContents(src, dst)
return
}
// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
err = out.Sync()
return
}
type rewrite func(*ast.File) *ast.File
// Mostly copied from gofmt
func initRewrite(rewriteRule string) rewrite {
f := strings.Split(rewriteRule, "->")
if len(f) != 2 {
fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n")
os.Exit(2)
}
pattern := parseExpr(f[0], "pattern")
replace := parseExpr(f[1], "replacement")
return func(p *ast.File) *ast.File { return rewriteFile(pattern, replace, p) }
}
// parseExpr parses s as an expression.
// It might make sense to expand this to allow statement patterns,
// but there are problems with preserving formatting and also
// with what a wildcard for a statement looks like.
func parseExpr(s, what string) ast.Expr {
x, err := parser.ParseExpr(s)
if err != nil {
fmt.Fprintf(os.Stderr, "parsing %s %s at %s\n", what, s, err)
os.Exit(2)
}
return x
}
// Keep this function for debugging.
/*
func dump(msg string, val reflect.Value) {
fmt.Printf("%s:\n", msg)
ast.Print(fileSet, val.Interface())
fmt.Println()
}
*/
// rewriteFile applies the rewrite rule 'pattern -> replace' to an entire file.
func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File {
cmap := ast.NewCommentMap(fileSet, p, p.Comments)
m := make(map[string]reflect.Value)
pat := reflect.ValueOf(pattern)
repl := reflect.ValueOf(replace)
var rewriteVal func(val reflect.Value) reflect.Value
rewriteVal = func(val reflect.Value) reflect.Value {
// don't bother if val is invalid to start with
if !val.IsValid() {
return reflect.Value{}
}
for k := range m {
delete(m, k)
}
val = apply(rewriteVal, val)
if match(m, pat, val) {
val = subst(m, repl, reflect.ValueOf(val.Interface().(ast.Node).Pos()))
}
return val
}
r := apply(rewriteVal, reflect.ValueOf(p)).Interface().(*ast.File)
r.Comments = cmap.Filter(r).Comments() // recreate comments list
return r
}
// set is a wrapper for x.Set(y); it protects the caller from panics if x cannot be changed to y.
func set(x, y reflect.Value) {
// don't bother if x cannot be set or y is invalid
if !x.CanSet() || !y.IsValid() {
return
}
defer func() {
if x := recover(); x != nil {
if s, ok := x.(string); ok &&
(strings.Contains(s, "type mismatch") || strings.Contains(s, "not assignable")) {
// x cannot be set to y - ignore this rewrite
return
}
panic(x)
}
}()
x.Set(y)
}
// Values/types for special cases.
var (
objectPtrNil = reflect.ValueOf((*ast.Object)(nil))
scopePtrNil = reflect.ValueOf((*ast.Scope)(nil))
identType = reflect.TypeOf((*ast.Ident)(nil))
objectPtrType = reflect.TypeOf((*ast.Object)(nil))
positionType = reflect.TypeOf(token.NoPos)
callExprType = reflect.TypeOf((*ast.CallExpr)(nil))
scopePtrType = reflect.TypeOf((*ast.Scope)(nil))
)
// apply replaces each AST field x in val with f(x), returning val.
// To avoid extra conversions, f operates on the reflect.Value form.
func apply(f func(reflect.Value) reflect.Value, val reflect.Value) reflect.Value {
if !val.IsValid() {
return reflect.Value{}
}
// *ast.Objects introduce cycles and are likely incorrect after
// rewrite; don't follow them but replace with nil instead
if val.Type() == objectPtrType {
return objectPtrNil
}
// similarly for scopes: they are likely incorrect after a rewrite;
// replace them with nil
if val.Type() == scopePtrType {
return scopePtrNil
}
switch v := reflect.Indirect(val); v.Kind() {
case reflect.Slice:
for i := 0; i < v.Len(); i++ {
e := v.Index(i)
set(e, f(e))
}
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
e := v.Field(i)
set(e, f(e))
}
case reflect.Interface:
e := v.Elem()
set(v, f(e))
}
return val
}
func isWildcard(s string) bool {
rune, size := utf8.DecodeRuneInString(s)
return size == len(s) && unicode.IsLower(rune)
}
// match returns true if pattern matches val,
// recording wildcard submatches in m.
// If m == nil, match checks whether pattern == val.
func match(m map[string]reflect.Value, pattern, val reflect.Value) bool {
// Wildcard matches any expression. If it appears multiple
// times in the pattern, it must match the same expression
// each time.
if m != nil && pattern.IsValid() && pattern.Type() == identType {
name := pattern.Interface().(*ast.Ident).Name
if isWildcard(name) && val.IsValid() {
// wildcards only match valid (non-nil) expressions.
if _, ok := val.Interface().(ast.Expr); ok && !val.IsNil() {
if old, ok := m[name]; ok {
return match(nil, old, val)
}
m[name] = val
return true
}
}
}
// Otherwise, pattern and val must match recursively.
if !pattern.IsValid() || !val.IsValid() {
return !pattern.IsValid() && !val.IsValid()
}
if pattern.Type() != val.Type() {
return false
}
// Special cases.
switch pattern.Type() {
case identType:
// For identifiers, only the names need to match
// (and none of the other *ast.Object information).
// This is a common case, handle it all here instead
// of recursing down any further via reflection.
p := pattern.Interface().(*ast.Ident)
v := val.Interface().(*ast.Ident)
return p == nil && v == nil || p != nil && v != nil && p.Name == v.Name
case objectPtrType, positionType:
// object pointers and token positions always match
return true
case callExprType:
// For calls, the Ellipsis fields (token.Position) must
// match since that is how f(x) and f(x...) are different.
// Check them here but fall through for the remaining fields.
p := pattern.Interface().(*ast.CallExpr)
v := val.Interface().(*ast.CallExpr)
if p.Ellipsis.IsValid() != v.Ellipsis.IsValid() {
return false
}
}
p := reflect.Indirect(pattern)
v := reflect.Indirect(val)
if !p.IsValid() || !v.IsValid() {
return !p.IsValid() && !v.IsValid()
}
switch p.Kind() {
case reflect.Slice:
if p.Len() != v.Len() {
return false
}
for i := 0; i < p.Len(); i++ {
if !match(m, p.Index(i), v.Index(i)) {
return false
}
}
return true
case reflect.Struct:
for i := 0; i < p.NumField(); i++ {
if !match(m, p.Field(i), v.Field(i)) {
return false
}
}
return true
case reflect.Interface:
return match(m, p.Elem(), v.Elem())
}
// Handle token integers, etc.
return p.Interface() == v.Interface()
}
// subst returns a copy of pattern with values from m substituted in place
// of wildcards and pos used as the position of tokens from the pattern.
// if m == nil, subst returns a copy of pattern and doesn't change the line
// number information.
func subst(m map[string]reflect.Value, pattern reflect.Value, pos reflect.Value) reflect.Value {
if !pattern.IsValid() {
return reflect.Value{}
}
// Wildcard gets replaced with map value.
if m != nil && pattern.Type() == identType {
name := pattern.Interface().(*ast.Ident).Name
if isWildcard(name) {
if old, ok := m[name]; ok {
return subst(nil, old, reflect.Value{})
}
}
}
if pos.IsValid() && pattern.Type() == positionType {
// use new position only if old position was valid in the first place
if old := pattern.Interface().(token.Pos); !old.IsValid() {
return pattern
}
return pos
}
// Otherwise copy.
switch p := pattern; p.Kind() {
case reflect.Slice:
v := reflect.MakeSlice(p.Type(), p.Len(), p.Len())
for i := 0; i < p.Len(); i++ {
v.Index(i).Set(subst(m, p.Index(i), pos))
}
return v
case reflect.Struct:
v := reflect.New(p.Type()).Elem()
for i := 0; i < p.NumField(); i++ {
v.Field(i).Set(subst(m, p.Field(i), pos))
}
return v
case reflect.Ptr:
v := reflect.New(p.Type()).Elem()
if elem := p.Elem(); elem.IsValid() {
v.Set(subst(m, elem, pos).Addr())
}
return v
case reflect.Interface:
v := reflect.New(p.Type()).Elem()
if elem := p.Elem(); elem.IsValid() {
v.Set(subst(m, elem, pos))
}
return v
}
return pattern
}
vendor/github.com/revel/config/README.md
Generated Vendored
+90
View File
@@ -0,0 +1,90 @@
config
======
This package implements a basic configuration file parser language which
provides a structure similar to what you would find on Microsoft Windows INI
files.
The configuration file consists of sections, led by a "*[section]*" header and
followed by "*name: value*" entries; "*name=value*" is also accepted. Note that
leading whitespace is removed from values. The optional values can contain
format strings which refer to other values in the same section, or values in a
special *DEFAULT* section. Additional defaults can be provided on initialization
and retrieval. Comments are indicated by ";" or "#"; a comment may begin
anywhere on a line, including on the same line after parameters or section
declarations.
For example:
[My Section]
foodir: %(dir)s/whatever
dir=foo
would resolve the "*%(dir)s*" to the value of "*dir*" (*foo* in this case). All
reference expansions are done on demand.
The functionality and workflow is loosely based on the *configparser* package of
the Python Standard Library.
## Installation
go get github.com/revel/config
## Operating instructions
Given a sample configuration file:
[DEFAULT]
host: www.example.com
protocol: http://
base-url: %(protocol)s%(host)s
[service-1]
url: %(base-url)s/some/path
delegation: on
maxclients: 200 # do not set this higher
comments: This is a multi-line
entry # And this is a comment
To read this configuration file, do:
c, _ := config.ReadDefault("config.cfg")
c.String("service-1", "url")
// result is string "http://www.example.com/some/path"
c.Int("service-1", "maxclients")
// result is int 200
c.Bool("service-1", "delegation")
// result is bool true
c.String("service-1", "comments")
// result is string "This is a multi-line\nentry"
Note the support for unfolding variables (such as *%(base-url)s*), which are read
from the special (reserved) section name *[DEFAULT]*.
A new configuration file can also be created with:
c := config.NewDefault()
c.AddSection("Section")
c.AddOption("Section", "option", "value")
c.WriteFile("config.cfg", 0644, "A header for this file")
This results in the file:
# A header for this file
[Section]
option: value
Note that sections, options and values are all case-sensitive.
## License
The source files are distributed under the [Mozilla Public License, version 2.0](http://mozilla.org/MPL/2.0/),
unless otherwise noted.
Please read the [FAQ](http://www.mozilla.org/MPL/2.0/FAQ.html)
if you have further questions regarding the license.
vendor/github.com/revel/config/config.go
Generated Vendored
+152
View File
@@ -0,0 +1,152 @@
// Copyright 2009 The "config" Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"regexp"
"strings"
)
// config constants
const (
// Default section name.
DefaultSection = "DEFAULT"
// Maximum allowed depth when recursively substituing variable names.
DepthValues = 200
DefaultComment = "# "
AlternativeComment = "; "
DefaultSeparator = ":"
AlternativeSeparator = "="
)
var (
// Strings accepted as boolean.
boolString = map[string]bool{
"t": true,
"true": true,
"y": true,
"yes": true,
"on": true,
"1": true,
"f": false,
"false": false,
"n": false,
"no": false,
"off": false,
"0": false,
}
varRegExp = regexp.MustCompile(`%\(([a-zA-Z0-9_.\-]+)\)s`) // %(variable)s
envVarRegExp = regexp.MustCompile(`\${([a-zA-Z0-9_.\-]+)}`) // ${envvar}
)
// Config is the representation of configuration settings.
type Config struct {
comment string
separator string
// Sections order
lastIDSection int // Last section identifier
idSection map[string]int // Section : position
// The last option identifier used for each section.
lastIDOption map[string]int // Section : last identifier
// Section -> option : value
data map[string]map[string]*tValue
}
// tValue holds the input position for a value.
type tValue struct {
position int // Option order
v string // value
}
// New creates an empty configuration representation.
// This representation can be filled with AddSection and AddOption and then
// saved to a file using WriteFile.
//
// == Arguments
//
// comment: has to be `DefaultComment` or `AlternativeComment`
// separator: has to be `DefaultSeparator` or `AlternativeSeparator`
// preSpace: indicate if is inserted a space before of the separator
// postSpace: indicate if is added a space after of the separator
func New(comment, separator string, preSpace, postSpace bool) *Config {
if comment != DefaultComment && comment != AlternativeComment {
panic("comment character not valid")
}
if separator != DefaultSeparator && separator != AlternativeSeparator {
panic("separator character not valid")
}
// == Get spaces around separator
if preSpace {
separator = " " + separator
}
if postSpace {
separator += " "
}
//==
c := new(Config)
c.comment = comment
c.separator = separator
c.idSection = make(map[string]int)
c.lastIDOption = make(map[string]int)
c.data = make(map[string]map[string]*tValue)
c.AddSection(DefaultSection) // Default section always exists.
return c
}
// NewDefault creates a configuration representation with values by default.
func NewDefault() *Config {
return New(DefaultComment, DefaultSeparator, false, true)
}
// Merge merges the given configuration "source" with this one ("target").
//
// Merging means that any option (under any section) from source that is not in
// target will be copied into target. When the target already has an option with
// the same name and section then it is overwritten (i.o.w. the source wins).
func (target *Config) Merge(source *Config) {
if source == nil || source.data == nil || len(source.data) == 0 {
return
}
for section, option := range source.data {
for optionName, optionValue := range option {
target.AddOption(section, optionName, optionValue.v)
}
}
}
// == Utility
func stripComments(l string) string {
// Comments are preceded by space or TAB
for _, c := range []string{" ;", "\t;", " #", "\t#"} {
if i := strings.Index(l, c); i != -1 {
l = l[0:i]
}
}
return l
}
vendor/github.com/revel/config/context.go
Generated Vendored
+169
View File
@@ -0,0 +1,169 @@
// Copyright 2016 The "config" Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
)
// Context structure handles the parsing of app.conf
// It has a "preferred" section that is checked first for option queries.
// If the preferred section does not have the option, the DEFAULT section is
// checked fallback.
type Context struct {
config *Config
section string // Check this section first, then fall back to DEFAULT
}
// NewContext creates a default section and returns config context
func NewContext() *Context {
return &Context{config: NewDefault()}
}
// LoadContext loads the ini config from gives multiple conf paths
func LoadContext(confName string, confPaths []string) (*Context, error) {
ctx := NewContext()
for _, confPath := range confPaths {
path := filepath.Join(confPath, confName)
conf, err := ReadDefault(path)
if err != nil {
if _, isPathErr := err.(*os.PathError); !isPathErr {
return nil, fmt.Errorf("%v: %v", path, err)
}
continue
}
ctx.config.Merge(conf)
}
return ctx, nil
}
// Raw returns raw config instance
func (c *Context) Raw() *Config {
return c.config
}
// SetSection the section scope of ini config
// For e.g.: dev or prod
func (c *Context) SetSection(section string) {
c.section = section
}
// SetOption sets the value for the given key
func (c *Context) SetOption(name, value string) {
c.config.AddOption(c.section, name, value)
}
// Int returns `int` config value and if found returns true
// otherwise false
func (c *Context) Int(option string) (result int, found bool) {
result, err := c.config.Int(c.section, option)
if err == nil {
return result, true
}
if _, ok := err.(OptionError); ok {
return 0, false
}
// If it wasn't an OptionError, it must have failed to parse.
return 0, false
}
// IntDefault returns `int` config value if found otherwise
// returns given default int value
func (c *Context) IntDefault(option string, dfault int) int {
if r, found := c.Int(option); found {
return r
}
return dfault
}
// Bool returns `bool` config value and if found returns true
// otherwise false
func (c *Context) Bool(option string) (result, found bool) {
result, err := c.config.Bool(c.section, option)
if err == nil {
return result, true
}
if _, ok := err.(OptionError); ok {
return false, false
}
// If it wasn't an OptionError, it must have failed to parse.
return false, false
}
// BoolDefault returns `bool` config value if found otherwise
// returns given default bool value
func (c *Context) BoolDefault(option string, dfault bool) bool {
if r, found := c.Bool(option); found {
return r
}
return dfault
}
// String returns `string` config value and if found returns true
// otherwise false
func (c *Context) String(option string) (result string, found bool) {
if r, err := c.config.String(c.section, option); err == nil {
return stripQuotes(r), true
}
return "", false
}
// StringDefault returns `string` config value if found otherwise
// returns given default string value
func (c *Context) StringDefault(option, dfault string) string {
if r, found := c.String(option); found {
return r
}
return dfault
}
// HasSection checks if the configuration has the given section.
// (The default section always exists.)
func (c *Context) HasSection(section string) bool {
return c.config.HasSection(section)
}
// Options returns all configuration option keys.
// If a prefix is provided, then that is applied as a filter.
func (c *Context) Options(prefix string) []string {
var options []string
keys, _ := c.config.Options(c.section)
for _, key := range keys {
if strings.HasPrefix(key, prefix) {
options = append(options, key)
}
}
return options
}
// Helpers
func stripQuotes(s string) string {
if s == "" {
return s
}
if s[0] == '"' && s[len(s)-1] == '"' {
return s[1 : len(s)-1]
}
return s
}
vendor/github.com/revel/config/error.go
Generated Vendored
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2009 The "config" Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
// SectionError type string
type SectionError string
func (e SectionError) Error() string {
return "section not found: " + string(e)
}
// OptionError type string
type OptionError string
func (e OptionError) Error() string {
return "option not found: " + string(e)
}
vendor/github.com/revel/config/option.go
Generated Vendored
+113
View File
@@ -0,0 +1,113 @@
// Copyright 2009 The "config" Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import "errors"
// AddOption adds a new option and value to the configuration.
//
// If the section is nil then uses the section by default; if it does not exist,
// it is created in advance.
//
// It returns true if the option and value were inserted, and false if the value
// was overwritten.
func (c *Config) AddOption(section string, option string, value string) bool {
c.AddSection(section) // Make sure section exists
if section == "" {
section = DefaultSection
}
_, ok := c.data[section][option]
c.data[section][option] = &tValue{c.lastIDOption[section], value}
c.lastIDOption[section]++
return !ok
}
// RemoveOption removes a option and value from the configuration.
// It returns true if the option and value were removed, and false otherwise,
// including if the section did not exist.
func (c *Config) RemoveOption(section string, option string) bool {
if _, ok := c.data[section]; !ok {
return false
}
_, ok := c.data[section][option]
delete(c.data[section], option)
return ok
}
// HasOption checks if the configuration has the given option in the section.
// It returns false if either the option or section do not exist.
func (c *Config) HasOption(section string, option string) bool {
if _, ok := c.data[section]; !ok {
return false
}
_, okd := c.data[DefaultSection][option]
_, oknd := c.data[section][option]
return okd || oknd
}
// Options returns the list of options available in the given section.
// It returns an error if the section does not exist and an empty list if the
// section is empty. Options within the default section are also included.
func (c *Config) Options(section string) (options []string, err error) {
if _, ok := c.data[section]; !ok {
return nil, errors.New(SectionError(section).Error())
}
// Keep a map of option names we've seen to deduplicate.
optionMap := make(map[string]struct{},
len(c.data[DefaultSection])+len(c.data[section]))
for s := range c.data[DefaultSection] {
optionMap[s] = struct{}{}
}
for s := range c.data[section] {
optionMap[s] = struct{}{}
}
// Get the keys.
i := 0
options = make([]string, len(optionMap))
for k := range optionMap {
options[i] = k
i++
}
return options, nil
}
// SectionOptions returns only the list of options available in the given section.
// Unlike Options, SectionOptions doesn't return options in default section.
// It returns an error if the section doesn't exist.
func (c *Config) SectionOptions(section string) (options []string, err error) {
if _, ok := c.data[section]; !ok {
return nil, errors.New(SectionError(section).Error())
}
options = make([]string, len(c.data[section]))
i := 0
for s := range c.data[section] {
options[i] = s
i++
}
return options, nil
}
vendor/github.com/revel/config/read.go
Generated Vendored
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2009 The "config" Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"bufio"
"fmt"
"os"
"strings"
"unicode"
)
// _read is the base to read a file and get the configuration representation.
// That representation can be queried with GetString, etc.
func _read(fname string, c *Config) (*Config, error) {
file, err := os.Open(fname)
if err != nil {
return nil, err
}
if err = c.read(bufio.NewReader(file)); err != nil {
return nil, err
}
if err = file.Close(); err != nil {
return nil, err
}
return c, nil
}
// Read reads a configuration file and returns its representation.
// All arguments, except `fname`, are related to `New()`
func Read(fname string, comment, separator string, preSpace, postSpace bool) (*Config, error) {
return _read(fname, New(comment, separator, preSpace, postSpace))
}
// ReadDefault reads a configuration file and returns its representation.
// It uses values by default.
func ReadDefault(fname string) (*Config, error) {
return _read(fname, NewDefault())
}
// * * *
func (c *Config) read(buf *bufio.Reader) (err error) {
var section, option string
var scanner = bufio.NewScanner(buf)
lineNo := 0
for scanner.Scan() {
l := strings.TrimRightFunc(stripComments(scanner.Text()), unicode.IsSpace)
lineNo++
// Switch written for readability (not performance)
switch {
// Empty line and comments
case len(l) == 0, l[0] == '#', l[0] == ';':
continue
// New section. The [ must be at the start of the line
case l[0] == '[' && l[len(l)-1] == ']':
option = "" // reset multi-line value
section = strings.TrimSpace(l[1 : len(l)-1])
c.AddSection(section)
// Continuation of multi-line value
// starts with whitespace, we're in a section and working on an option
case section != "" && option != "" && (l[0] == ' ' || l[0] == '\t'):
prev, _ := c.RawString(section, option)
value := strings.TrimSpace(l)
c.AddOption(section, option, prev+"\n"+value)
// Other alternatives
default:
i := strings.IndexAny(l, "=:")
switch {
// Option and value
case i > 0 && l[0] != ' ' && l[0] != '\t': // found an =: and it's not a multiline continuation
option = strings.TrimSpace(l[0:i])
value := strings.TrimSpace(l[i+1:])
c.AddOption(section, option, value)
default:
return fmt.Errorf("could not parse line #%v: %v", lineNo, l)
}
}
}
return scanner.Err()
}
vendor/github.com/revel/config/section.go
Generated Vendored
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2009 The "config" Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
// AddSection adds a new section to the configuration.
//
// If the section is nil then uses the section by default which it's already
// created.
//
// It returns true if the new section was inserted, and false if the section
// already existed.
func (c *Config) AddSection(section string) bool {
// DefaultSection
if section == "" {
return false
}
if _, ok := c.data[section]; ok {
return false
}
c.data[section] = make(map[string]*tValue)
// Section order
c.idSection[section] = c.lastIDSection
c.lastIDSection++
return true
}
// RemoveSection removes a section from the configuration.
// It returns true if the section was removed, and false if section did not exist.
func (c *Config) RemoveSection(section string) bool {
_, ok := c.data[section]
// Default section cannot be removed.
if !ok || section == DefaultSection {
return false
}
for o := range c.data[section] {
delete(c.data[section], o) // *value
}
delete(c.data, section)
delete(c.lastIDOption, section)
delete(c.idSection, section)
return true
}
// HasSection checks if the configuration has the given section.
// (The default section always exists.)
func (c *Config) HasSection(section string) bool {
_, ok := c.data[section]
return ok
}
// Sections returns the list of sections in the configuration.
// (The default section always exists).
func (c *Config) Sections() (sections []string) {
sections = make([]string, len(c.idSection))
pos := 0 // Position in sections
for i := 0; i < c.lastIDSection; i++ {
for section, id := range c.idSection {
if id == i {
sections[pos] = section
pos++
}
}
}
return sections
}
vendor/github.com/revel/config/type.go
Generated Vendored
+155
View File
@@ -0,0 +1,155 @@
// Copyright 2009 The "config" Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"errors"
"fmt"
"os"
"regexp"
"strconv"
"strings"
)
// Substitutes values, calculated by callback, on matching regex
func (c *Config) computeVar(beforeValue *string, regx *regexp.Regexp, headsz, tailsz int, withVar func(*string) string) (*string, error) {
var i int
computedVal := beforeValue
for i = 0; i < DepthValues; i++ { // keep a sane depth
vr := regx.FindStringSubmatchIndex(*computedVal)
if len(vr) == 0 {
break
}
varname := (*computedVal)[vr[headsz]:vr[headsz+1]]
varVal := withVar(&varname)
if varVal == "" {
return &varVal, fmt.Errorf("Option not found: %s", varname)
}
// substitute by new value and take off leading '%(' and trailing ')s'
// %(foo)s => headsz=2, tailsz=2
// ${foo} => headsz=2, tailsz=1
newVal := (*computedVal)[0:vr[headsz]-headsz] + varVal + (*computedVal)[vr[headsz+1]+tailsz:]
computedVal = &newVal
}
if i == DepthValues {
retVal := ""
return &retVal,
fmt.Errorf("Possible cycle while unfolding variables: max depth of %d reached", DepthValues)
}
return computedVal, nil
}
// Bool has the same behaviour as String but converts the response to bool.
// See "boolString" for string values converted to bool.
func (c *Config) Bool(section string, option string) (value bool, err error) {
sv, err := c.String(section, option)
if err != nil {
return false, err
}
value, ok := boolString[strings.ToLower(sv)]
if !ok {
return false, errors.New("could not parse bool value: " + sv)
}
return value, nil
}
// Float has the same behaviour as String but converts the response to float.
func (c *Config) Float(section string, option string) (value float64, err error) {
sv, err := c.String(section, option)
if err == nil {
value, err = strconv.ParseFloat(sv, 64)
}
return value, err
}
// Int has the same behaviour as String but converts the response to int.
func (c *Config) Int(section string, option string) (value int, err error) {
sv, err := c.String(section, option)
if err == nil {
value, err = strconv.Atoi(sv)
}
return value, err
}
// RawString gets the (raw) string value for the given option in the section.
// The raw string value is not subjected to unfolding, which was illustrated in
// the beginning of this documentation.
//
// It returns an error if either the section or the option do not exist.
func (c *Config) RawString(section string, option string) (value string, err error) {
if _, ok := c.data[section]; ok {
if tValue, ok := c.data[section][option]; ok {
return tValue.v, nil
}
}
return c.RawStringDefault(option)
}
// RawStringDefault gets the (raw) string value for the given option from the
// DEFAULT section.
//
// It returns an error if the option does not exist in the DEFAULT section.
func (c *Config) RawStringDefault(option string) (value string, err error) {
if tValue, ok := c.data[DefaultSection][option]; ok {
return tValue.v, nil
}
return "", OptionError(option)
}
// String gets the string value for the given option in the section.
// If the value needs to be unfolded (see e.g. %(host)s example in the beginning
// of this documentation), then String does this unfolding automatically, up to
// `DepthValues` number of iterations.
//
// It returns an error if either the section or the option do not exist, or the
// unfolding cycled.
func (c *Config) String(section string, option string) (value string, err error) {
value, err = c.RawString(section, option)
if err != nil {
return "", err
}
// % variables
computedVal, err := c.computeVar(&value, varRegExp, 2, 2, func(varName *string) string {
lowerVar := *varName
// search variable in default section as well as current section
varVal, _ := c.data[DefaultSection][lowerVar]
if _, ok := c.data[section][lowerVar]; ok {
varVal = c.data[section][lowerVar]
}
return varVal.v
})
value = *computedVal
if err != nil {
return value, err
}
// $ environment variables
computedVal, err = c.computeVar(&value, envVarRegExp, 2, 1, func(varName *string) string {
return os.Getenv(*varName)
})
value = *computedVal
return value, err
}
vendor/github.com/revel/config/write.go
Generated Vendored
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2009 The "config" Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"bufio"
"fmt"
"os"
"strings"
)
// WriteFile saves the configuration representation to a file.
// The desired file permissions must be passed as in os.Open. The header is a
// string that is saved as a comment in the first line of the file.
func (c *Config) WriteFile(fname string, perm os.FileMode, header string) error {
file, err := os.OpenFile(fname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
if err != nil {
return err
}
buf := bufio.NewWriter(file)
if err = c.write(buf, header); err != nil {
return err
}
_ = buf.Flush()
return file.Close()
}
func (c *Config) write(buf *bufio.Writer, header string) (err error) {
if header != "" {
// Add comment character after of each new line.
if i := strings.Index(header, "\n"); i != -1 {
header = strings.Replace(header, "\n", "\n"+c.comment, -1)
}
if _, err = buf.WriteString(c.comment + header + "\n"); err != nil {
return err
}
}
for _, orderedSection := range c.Sections() {
for section, sectionMap := range c.data {
if section == orderedSection {
// Skip default section if empty.
if section == DefaultSection && len(sectionMap) == 0 {
continue
}
if _, err = buf.WriteString("\n[" + section + "]\n"); err != nil {
return err
}
// Follow the input order in options.
for i := 0; i < c.lastIDOption[section]; i++ {
for option, tValue := range sectionMap {
if tValue.position == i {
if _, err = buf.WriteString(fmt.Sprint(
option, c.separator, tValue.v, "\n")); err != nil {
return err
}
c.RemoveOption(section, option)
break
}
}
}
}
}
}
_, err = buf.WriteString("\n")
return err
}
vendor/github.com/revel/modules/README.md
Generated Vendored
+29
View File
@@ -0,0 +1,29 @@
---
title: Modules Overview
section: modules
---
Revel Modules
==================================
## Core Modules
These modules are part of the core revel and a dependancy
- [Static](static/) = Serves static files
- [Jobs](jobs/) = The Cron Jobs runner
- [TestRunner](testrunner/) = Test sweet
## Upcoming
- [csrf](csrf/)
- [pprof](/pprof/)
## Xperimental
- [Auth](/auth/)
- [Db](/testrunner/)
### Caution
this is a work in progress
vendor/github.com/revel/modules/auth/README.md
Generated Vendored
+6
View File
@@ -0,0 +1,6 @@
modules/auth
===============
Basic user/auth module
This should be modeled after [flask-security](https://github.com/mattupstate/flask-security)
vendor/github.com/revel/modules/auth/auth.go
Generated Vendored
+78
View File
@@ -0,0 +1,78 @@
package auth
import (
// "errors"
)
var (
Store StorageDriver
)
// Store = gormauth.NewGormAuthDriver()
type UserAuth interface {
// getters/setters implemented by the app-level model
UserId() string
Secret() string
HashedSecret() string
SetHashedSecret(string)
SecretDriver
// // implemented by secret driver
// Authenticate() (bool, error)
}
type SecretDriver interface {
Authenticate() (bool, error)
HashSecret(args ...interface{}) (string, error)
// stuff for documentation
// UserContext is expected in these?
// Secret expects 0 or non-0 arguments
// When no parameter is passed, it acts as a getter
// When one or more parameters are passed, it acts as a setter
// A driver should specify the expected arguments and their meanings
// Register()
// Login()
// Logout()
}
type StorageDriver interface {
Save(user interface{}) error
// Load should take a partially filled struct
// (with values needed to look up)
// and fills in the rest
Load(user interface{}) error
}
// func init() {
// // auth.Store = gorm...
// }
// func (c App) Login(email, password string) {
// u := User {
// Email ...
// }
// good, err := auth.Authenticate(email, password)
// user, err := user_info.GetUserByEmail(email)
// }
// Bycrypt Authenticate() expects a single string argument of the plaintext password
// It returns true on success and false if error or password mismatch
// func Authenticate(attemptedUser UserAuth) (bool, error) {
// // check user in Store
// loadedUser, err := Store.Load(attemptedUser.UserId())
// if err != nil {
// return false, errors.New("User Not Found")
// }
// loadedUser.Authenticate(attemptedUser.Secret())
// // successfully authenticated
// return true, nil
// }
vendor/github.com/revel/modules/auth/auth_test.go
Generated Vendored
+134
View File
@@ -0,0 +1,134 @@
package auth_test
import (
"errors"
"testing"
"github.com/revel/modules/auth"
"github.com/revel/modules/auth/driver/secret"
)
type User struct {
email string
password string
hashpass string
secret.BcryptAuth // SecurityDriver for testing
}
func NewUser(email, pass string) *User {
u := &User{
email: email,
password: pass,
}
u.UserContext = u
return u
}
func (self *User) UserId() string {
return self.email
}
func (self *User) Secret() string {
return self.password
}
func (self *User) HashedSecret() string {
return self.hashpass
}
func (self *User) SetHashedSecret(hpass string) {
self.hashpass = hpass
}
// func (self *User) Load() string
type TestStore struct {
data map[string]string
}
func (self *TestStore) Save(user interface{}) error {
u, ok := user.(*User)
if !ok {
return errors.New("TestStore.Save() expected arg of type User")
}
hPass, err := u.HashSecret(u.Secret())
if err != nil {
return err
}
self.data[u.UserId()] = hPass
return nil
}
func (self *TestStore) Load(user interface{}) error {
u, ok := user.(*User)
if !ok {
return errors.New("TestStore.Load() expected arg of type User")
}
hpass, ok := self.data[u.UserId()]
if !ok {
return errors.New("Record Not Found")
}
u.SetHashedSecret(hpass)
return nil
}
func TestPasswordHash(t *testing.T) {
auth.Store = &TestStore{
data: make(map[string]string),
}
u := NewUser("demo@domain.com", "demopass")
fail := NewUser("demo@domain.com", "")
var err error
u.hashpass, err = u.HashSecret(u.password)
if err != nil {
t.Errorf("Should have hashed password, get error: %v\n", err)
}
fail.hashpass, err = fail.HashSecret(fail.password)
if err == nil {
t.Errorf("Should have failed hashing\n")
}
}
func TestAuthenticate(t *testing.T) {
auth.Store = &TestStore{
data: make(map[string]string),
}
// user registered a long time ago
u := NewUser("demo@domain.com", "demopass")
err := auth.Store.Save(u)
if err != nil {
t.Errorf("Should have saved user: %v", err)
}
// users now logging in
pass := NewUser("demo@domain.com", "demopass")
fail := NewUser("demo@domain.com", "invalid")
// valid user is now trying to login
// check user in DB
err = auth.Store.Load(pass)
if err != nil {
t.Errorf("Should have loaded pass user: %v\n", err)
}
// check credentials
ok, err := pass.Authenticate()
if !ok || err != nil {
t.Errorf("Should have authenticated user")
}
// invalid user is now trying to login
err = auth.Store.Load(fail)
if err != nil {
t.Errorf("Should have loaded fail user")
}
// this should fail
ok, err = fail.Authenticate()
if ok || err != nil {
t.Errorf("Should have failed to authenticate user: %v\n", err)
}
}
vendor/github.com/revel/modules/auth/doc.go
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
/* A basic user authentication module for Revel
list of concerns:
- Separating out the interface and driver
- Removing DB/Storage dependency
- UUID as default identifier?
- how to deal with password/secret or generally, method of authorization
- default {views,controllers,routes} for register/login/logut ?
- reset password in most basic ?
- activation (and other features) in a second / more sophisticated driver
- filter for checking that user is authenticated
I think a driver is made up of 2 parts
data prep and data storage
register and password reset are part of data prep
as is the auth hash method
they don't care how the data is stored
then there is the data store
perhaps each auth user model should instantiate 2 drivers instead of 1?
one for data prep components and one for storage
so the security driver and the storage driver
*/
package auth
vendor/github.com/revel/modules/auth/driver/secret/bcrypt.go
Generated Vendored
+73
View File
@@ -0,0 +1,73 @@
package secret
import (
"errors"
"github.com/revel/modules/auth"
"golang.org/x/crypto/bcrypt"
)
// example implementation of a Revel auth security driver
// This driver should be embedded into your app-level User model
// It expects your User model to have `Password` and `HashedPassword` string fields
//
// Your User model also needs to set itself as the UserContext for the BcryptAuth driver
//
// func NewUser(email, pass string) *User {
// u := &User{
// email: email,
// password: pass,
// }
// u.UserContext = u
// }
//
type BcryptAuth struct {
UserContext auth.UserAuth
}
// Bcrypt Secret() returns the hashed version of the password.
// It expects an argument of type string, which is the plain text password
func (self *BcryptAuth) HashSecret(args ...interface{}) (string, error) {
if auth.Store == nil {
return "", errors.New("Auth module StorageDriver not set")
}
argLen := len(args)
if argLen == 0 {
// we are getting
return string(self.UserContext.HashedSecret()), nil
}
if argLen == 1 {
// we are setting
password, ok := args[0].(string)
if !ok {
return "", errors.New("Wrong argument type provided, expected plaintext password as string")
}
hPass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return "", err
}
self.UserContext.SetHashedSecret(string(hPass))
return self.UserContext.HashedSecret(), nil
}
// bad argument count
return "", errors.New("Too many arguments provided, expected one")
}
// Bycrypt Authenticate() expects a single string argument of the plaintext password
// It returns true on success and false if error or password mismatch
func (self *BcryptAuth) Authenticate() (bool, error) {
// check password
err := bcrypt.CompareHashAndPassword([]byte(self.UserContext.HashedSecret()), []byte(self.UserContext.Secret()))
if err == bcrypt.ErrMismatchedHashAndPassword {
return false, nil
}
if err != nil {
return false, err
}
// successfully authenticated
return true, nil
}
vendor/github.com/revel/modules/auth/driver/storage/mysql/README.md
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
Mysql Auth driver
==================
vendor/github.com/revel/modules/auth/driver/storage/postgres/README.md
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
Postgresql Auth driver
==================
vendor/github.com/revel/modules/auth/driver/storage/sqlite/README.md
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
Sqlite Auth driver
==================
vendor/github.com/revel/modules/auth/user.go
Generated Vendored
+17
View File
@@ -0,0 +1,17 @@
package auth
// var storageDriver auth.StorageDriver // postgres in example
// type AuthUserModel struct {
// userId string
// security *SecurityDriver // bcrypt in example
// }
// func (self *AuthUserModel) UserId() string {
// return self.userId
// }
// func (self *AuthUserModel) Secret() string {
// return self.security.Secret()
// }
vendor/github.com/revel/modules/csrf/app/csrf.go
Generated Vendored
+121
View File
@@ -0,0 +1,121 @@
package csrf
import (
"crypto/rand"
"crypto/subtle"
"encoding/hex"
"html/template"
"io"
"math"
"net/url"
"github.com/revel/revel"
)
// allowMethods are HTTP methods that do NOT require a token
var allowedMethods = map[string]bool{
"GET": true,
"HEAD": true,
"OPTIONS": true,
"TRACE": true,
}
func RandomString(length int) (string, error) {
buffer := make([]byte, int(math.Ceil(float64(length)/2)))
if _, err := io.ReadFull(rand.Reader, buffer); err != nil {
return "", nil
}
str := hex.EncodeToString(buffer)
return str[:length], nil
}
func RefreshToken(c *revel.Controller) {
token, err := RandomString(64)
if err != nil {
panic(err)
}
c.Session["csrf_token"] = token
}
// CsrfFilter enables CSRF request token creation and verification.
//
// Usage:
// 1) Add `csrf.CsrfFilter` to the app's filters (it must come after the revel.SessionFilter).
// 2) Add CSRF fields to a form with the template tag `{{ csrftoken . }}`. The filter adds a function closure to the `ViewArgs` that can pull out the secret and make the token as-needed, caching the value in the request. Ajax support provided through the `X-CSRFToken` header.
func CsrfFilter(c *revel.Controller, fc []revel.Filter) {
token, foundToken := c.Session["csrf_token"]
if !foundToken {
RefreshToken(c)
}
referer, refErr := url.Parse(c.Request.Header.Get("Referer"))
isSameOrigin := sameOrigin(c.Request.URL, referer)
// If the Request method isn't in the white listed methods
if !allowedMethods[c.Request.Method] && !IsExempt(c) {
// Token wasn't present at all
if !foundToken {
c.Result = c.Forbidden("REVEL CSRF: Session token missing.")
return
}
// Referer header is invalid
if refErr != nil {
c.Result = c.Forbidden("REVEL CSRF: HTTP Referer malformed.")
return
}
// Same origin
if !isSameOrigin {
c.Result = c.Forbidden("REVEL CSRF: Same origin mismatch.")
return
}
var requestToken string
// First check for token in post data
if c.Request.Method == "POST" {
requestToken = c.Request.FormValue("csrftoken")
}
// Then check for token in custom headers, as with AJAX
if requestToken == "" {
requestToken = c.Request.Header.Get("X-CSRFToken")
}
if requestToken == "" || !compareToken(requestToken, token) {
c.Result = c.Forbidden("REVEL CSRF: Invalid token.")
return
}
}
fc[0](c, fc[1:])
// Only add token to ViewArgs if the request is: not AJAX, not missing referer header, and is same origin.
if c.Request.Header.Get("X-CSRFToken") == "" && isSameOrigin {
c.ViewArgs["_csrftoken"] = token
}
}
func compareToken(requestToken, token string) bool {
// ConstantTimeCompare will panic if the []byte aren't the same length
if len(requestToken) != len(token) {
return false
}
return subtle.ConstantTimeCompare([]byte(requestToken), []byte(token)) == 1
}
// Validates same origin policy
func sameOrigin(u1, u2 *url.URL) bool {
return u1.Scheme == u2.Scheme && u1.Host == u2.Host
}
func init() {
revel.TemplateFuncs["csrftoken"] = func(viewArgs map[string]interface{}) template.HTML {
if tokenFunc, ok := viewArgs["_csrftoken"]; !ok {
panic("REVEL CSRF: _csrftoken missing from ViewArgs.")
} else {
return template.HTML(tokenFunc.(func() string)())
}
}
}
vendor/github.com/revel/modules/csrf/app/csrf_test.go
Generated Vendored
+169
View File
@@ -0,0 +1,169 @@
package csrf
import (
"bytes"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"testing"
"github.com/revel/revel"
)
var testFilters = []revel.Filter{
CsrfFilter,
func(c *revel.Controller, fc []revel.Filter) {
c.RenderHtml("{{ csrftoken . }}")
},
}
func TestTokenInSession(t *testing.T) {
resp := httptest.NewRecorder()
getRequest, _ := http.NewRequest("GET", "http://www.example.com/", nil)
c := revel.NewController(revel.NewRequest(getRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
testFilters[0](c, testFilters)
if _, ok := c.Session["csrf_token"]; !ok {
t.Fatal("token should be present in session")
}
}
func TestPostWithoutToken(t *testing.T) {
resp := httptest.NewRecorder()
postRequest, _ := http.NewRequest("POST", "http://www.example.com/", nil)
c := revel.NewController(revel.NewRequest(postRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
testFilters[0](c, testFilters)
if c.Response.Status != 403 {
t.Fatal("post without token should be forbidden")
}
}
func TestNoReferrer(t *testing.T) {
resp := httptest.NewRecorder()
postRequest, _ := http.NewRequest("POST", "http://www.example.com/", nil)
c := revel.NewController(revel.NewRequest(postRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
RefreshToken(c)
token := c.Session["csrf_token"]
// make a new request with the token
data := url.Values{}
data.Set("csrftoken", token)
formPostRequest, _ := http.NewRequest("POST", "http://www.example.com/", bytes.NewBufferString(data.Encode()))
formPostRequest.Header.Add("Content-Type", "application/x-www-form-urlencoded")
formPostRequest.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
// and replace the old request
c.Request = revel.NewRequest(formPostRequest)
testFilters[0](c, testFilters)
if c.Response.Status != 403 {
t.Fatal("post without referer should be forbidden")
}
}
func TestRefererHttps(t *testing.T) {
resp := httptest.NewRecorder()
postRequest, _ := http.NewRequest("POST", "http://www.example.com/", nil)
c := revel.NewController(revel.NewRequest(postRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
RefreshToken(c)
token := c.Session["csrf_token"]
// make a new request with the token
data := url.Values{}
data.Set("csrftoken", token)
formPostRequest, _ := http.NewRequest("POST", "https://www.example.com/", bytes.NewBufferString(data.Encode()))
formPostRequest.Header.Add("Content-Type", "application/x-www-form-urlencoded")
formPostRequest.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
formPostRequest.Header.Add("Referer", "http://www.example.com/")
// and replace the old request
c.Request = revel.NewRequest(formPostRequest)
testFilters[0](c, testFilters)
if c.Response.Status != 403 {
t.Fatal("posts to https should have an https referer")
}
}
func TestHeaderWithToken(t *testing.T) {
resp := httptest.NewRecorder()
postRequest, _ := http.NewRequest("POST", "http://www.example.com/", nil)
c := revel.NewController(revel.NewRequest(postRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
RefreshToken(c)
token := c.Session["csrf_token"]
// make a new request with the token
formPostRequest, _ := http.NewRequest("POST", "http://www.example.com/", nil)
formPostRequest.Header.Add("X-CSRFToken", token)
formPostRequest.Header.Add("Referer", "http://www.example.com/")
// and replace the old request
c.Request = revel.NewRequest(formPostRequest)
testFilters[0](c, testFilters)
if c.Response.Status == 403 {
t.Fatal("post with http header token should be allowed")
}
}
func TestFormPostWithToken(t *testing.T) {
resp := httptest.NewRecorder()
postRequest, _ := http.NewRequest("POST", "http://www.example.com/", nil)
c := revel.NewController(revel.NewRequest(postRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
RefreshToken(c)
token := c.Session["csrf_token"]
// make a new request with the token
data := url.Values{}
data.Set("csrftoken", token)
formPostRequest, _ := http.NewRequest("POST", "http://www.example.com/", bytes.NewBufferString(data.Encode()))
formPostRequest.Header.Add("Content-Type", "application/x-www-form-urlencoded")
formPostRequest.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
formPostRequest.Header.Add("Referer", "http://www.example.com/")
// and replace the old request
c.Request = revel.NewRequest(formPostRequest)
testFilters[0](c, testFilters)
if c.Response.Status == 403 {
t.Fatal("form post with token should be allowed")
}
}
func TestNoTokenInArgsWhenCORs(t *testing.T) {
resp := httptest.NewRecorder()
getRequest, _ := http.NewRequest("GET", "http://www.example1.com/", nil)
getRequest.Header.Add("Referer", "http://www.example2.com/")
c := revel.NewController(revel.NewRequest(getRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
testFilters[0](c, testFilters)
if _, ok := c.ViewArgs["_csrftoken"]; ok {
t.Fatal("ViewArgs should not contain token when not same origin")
}
}
vendor/github.com/revel/modules/csrf/app/exempt.go
Generated Vendored
+36
View File
@@ -0,0 +1,36 @@
package csrf
import (
"fmt"
"strings"
"github.com/revel/revel"
)
var (
exemptPath = make(map[string]bool)
exemptAction = make(map[string]bool)
)
func MarkExempt(route string) {
if strings.HasPrefix(route, "/") {
// e.g. "/controller/action"
exemptPath[strings.ToLower(route)] = true
} else if routeParts := strings.Split(route, "."); len(routeParts) == 2 {
// e.g. "ControllerName.ActionName"
exemptAction[route] = true
} else {
err := fmt.Sprintf("csrf.MarkExempt() received invalid argument \"%v\". Either provide a path prefixed with \"/\" or controller action in the form of \"ControllerName.ActionName\".", route)
panic(err)
}
}
func IsExempt(c *revel.Controller) bool {
if _, ok := exemptPath[strings.ToLower(c.Request.Request.URL.Path)]; ok {
return true
} else if _, ok := exemptAction[c.Action]; ok {
return true
}
return false
}
vendor/github.com/revel/modules/csrf/app/exempt_test.go
Generated Vendored
+55
View File
@@ -0,0 +1,55 @@
package csrf
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/revel/revel"
)
func TestExemptPath(t *testing.T) {
MarkExempt("/Controller/Action")
resp := httptest.NewRecorder()
postRequest, _ := http.NewRequest("POST", "http://www.example.com/Controller/Action", nil)
c := revel.NewController(revel.NewRequest(postRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
testFilters[0](c, testFilters)
if c.Response.Status == 403 {
t.Fatal("post to csrf exempt action should pass")
}
}
func TestExemptPathCaseInsensitive(t *testing.T) {
MarkExempt("/Controller/Action")
resp := httptest.NewRecorder()
postRequest, _ := http.NewRequest("POST", "http://www.example.com/controller/action", nil)
c := revel.NewController(revel.NewRequest(postRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
testFilters[0](c, testFilters)
if c.Response.Status == 403 {
t.Fatal("post to csrf exempt action should pass")
}
}
func TestExemptAction(t *testing.T) {
MarkExempt("Controller.Action")
resp := httptest.NewRecorder()
postRequest, _ := http.NewRequest("POST", "http://www.example.com/Controller/Action", nil)
c := revel.NewController(revel.NewRequest(postRequest), revel.NewResponse(resp))
c.Session = make(revel.Session)
c.Action = "Controller.Action"
testFilters[0](c, testFilters)
if c.Response.Status == 403 {
t.Fatal("post to csrf exempt action should pass")
}
}
vendor/github.com/revel/modules/db/app/db.go
Generated Vendored
+94
View File
@@ -0,0 +1,94 @@
// Copyright (c) 2012-2016 The Revel Framework Authors, All rights reserved.
// Revel Framework source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.
// Package db module configures a database connection for the application.
//
// Developers use this module by importing and calling db.Init().
// A "Transactional" controller type is provided as a way to import interceptors
// that manage the transaction
//
// In particular, a transaction is begun before each request and committed on
// success. If a panic occurred during the request, the transaction is rolled
// back. (The application may also roll the transaction back itself.)
package db
import (
"database/sql"
"github.com/revel/revel"
)
// Database connection variables
var (
Db *sql.DB
Driver string
Spec string
)
// Init method used to initialize DB module on `OnAppStart`
func Init() {
// Read configuration.
var found bool
if Driver, found = revel.Config.String("db.driver"); !found {
revel.ERROR.Fatal("db.driver not configured")
}
if Spec, found = revel.Config.String("db.spec"); !found {
revel.ERROR.Fatal("db.spec not configured")
}
// Open a connection.
var err error
Db, err = sql.Open(Driver, Spec)
if err != nil {
revel.ERROR.Fatal(err)
}
}
// Transactional definition for database transaction
type Transactional struct {
*revel.Controller
Txn *sql.Tx
}
// Begin a transaction
func (c *Transactional) Begin() revel.Result {
txn, err := Db.Begin()
if err != nil {
panic(err)
}
c.Txn = txn
return nil
}
// Rollback if it's still going (must have panicked).
func (c *Transactional) Rollback() revel.Result {
if c.Txn != nil {
if err := c.Txn.Rollback(); err != nil {
if err != sql.ErrTxDone {
panic(err)
}
}
c.Txn = nil
}
return nil
}
// Commit the transaction.
func (c *Transactional) Commit() revel.Result {
if c.Txn != nil {
if err := c.Txn.Commit(); err != nil {
if err != sql.ErrTxDone {
panic(err)
}
}
c.Txn = nil
}
return nil
}
func init() {
revel.InterceptMethod((*Transactional).Begin, revel.BEFORE)
revel.InterceptMethod((*Transactional).Commit, revel.AFTER)
revel.InterceptMethod((*Transactional).Rollback, revel.FINALLY)
}
vendor/github.com/revel/modules/jobs/README.md
Generated Vendored
+220
View File
@@ -0,0 +1,220 @@
---
title: Jobs
github:
labels:
- topic-jobs
- topic-runtime
---
The [`Jobs`](https://godoc.org/github.com/revel/modules/jobs/app/jobs) framework for performing work asynchronously, outside of the
request flow. This may take the form of [recurring tasks](#jobs) that updates cached data
or [one-off tasks](#OneOff) such as sending emails.
## Activation
The [`Jobs`](https://godoc.org/github.com/revel/modules/jobs/app/jobs) framework is included as an optional [module](index.html), and is not enabled by default.
To activate it, add `module.jobs` to the [app.conf](../manual/appconf.html) file:
```ini
module.jobs = github.com/revel/modules/jobs
```
Additionally, in order to access the job monitoring page, you will need to add
this line to the `conf/routes` file, which will insert the `/@jobs` url:
module:jobs
## Options
There are some [configuration settings](../manual/appconf.html#jobs) that tell the framework what sort of limitations
to place on the jobs that it runs. These are listed below with their default values;
- [`jobs.pool = 10`](appconf.html#jobspool) - The number of jobs allowed to run simultaneously
- [`jobs.selfconcurrent = false`](appconf.html#jobsselfconcurrent) - Allow a job to run only if previous instances are done
- [`jobs.acceptproxyaddress = false`](appconf#jobsacceptproxyaddress) - Accept `X-Forwarded-For` header value (which is spoofable) to allow or deny status page access
## Implementing Jobs
To create a Job, implement the [`cron.Job`](https://github.com/robfig/cron/) interface. The
[`Job`](https://godoc.org/github.com/revel/modules/jobs/app/jobs#Job) interface has the following signature:
{% highlight go %}
// https://github.com/robfig/cron/blob/master/cron.go
type Job interface {
Run()
}
{% endhighlight %}
For example:
{% highlight go %}
type MyJob struct {}
func (j MyJob) Run() {
// Do something
}
{% endhighlight %}
## Startup jobs
To run a task on application startup, use
[`revel.OnAppStart()`](https://godoc.org/github.com/revel/revel#OnAppStart) to register a function.
Revel runs these tasks serially, before starting the server. Note that this
functionality does not actually use the jobs module, but it can be used to
submit a job for execution that doesn't block server startup.
{% highlight go %}
func init() {
revel.OnAppStart(func() { jobs.Now(populateCache{}) })
}
{% endhighlight %}
<a name="RecurringJobs"></a>
## Recurring Jobs
Jobs may be scheduled to run on any schedule. There are two options for expressing the schedule:
1. A cron specification
2. A fixed interval
Revel uses the [`cron library`](https://godoc.org/github.com/revel/cron) to parse the
schedule and run the jobs. The library's
[README](https://github.com/revel/cron/blob/master/README.md) provides a detailed
description of the format accepted.
Jobs are generally registered using the
[`revel.OnAppStart()`](https://godoc.org/github.com/revel/revel#OnAppStart) hook, but they may be
registered at any later time as well.
Here are some examples:
{% highlight go %}
import (
"github.com/revel/revel"
"github.com/revel/modules/jobs/app/jobs"
"time"
)
type ReminderEmails struct {
// filtered
}
func (e ReminderEmails) Run() {
// Queries the DB
// Sends some email
}
func init() {
revel.OnAppStart(func() {
jobs.Schedule("0 0 0 * * ?", ReminderEmails{})
jobs.Schedule("@midnight", ReminderEmails{})
jobs.Schedule("@every 24h", ReminderEmails{})
jobs.Every(24 * time.Hour, ReminderEmails{})
})
}
{% endhighlight %}
<a name="NamedSchedules"></a>
## Named schedules
You can [configure schedules ](appconf.html#jobs) in the [`app.conf`](appconf.html) file and reference them anywhere.
This provides an easy way to reuse, and a useful description for crontab specs.
Here is an example **named cron schedule**, in an [`app.conf`](appconf.html) file:
cron.workhours_15m = 0 */15 9-17 ? * MON-FRI
Use the named schedule by referencing it anywhere you would have used a cron spec.
{% highlight go %}
func init() {
revel.OnAppStart(func() {
jobs.Schedule("cron.workhours_15m", ReminderEmails{})
})
}
{% endhighlight %}
<div class="alert alert-warning">
<b>IMPORTANT</b>: The cron schedule's name must begin with <b>cron</b>.
</div>
<a name="OneOff"></a>
## One-off Jobs
Sometimes it is necessary to do something in response to a user action. In these
cases, the jobs module allows you to submit a job to be run a single time.
The only control offered is how long to wait until the job should be run.
{% highlight go %}
type AppController struct { *revel.Controller }
func (c AppController) Action() revel.Result {
// Handle the request.
...
// Send them email asynchronously, right now.
jobs.Now(SendConfirmationEmail{})
// Or, send them email asynchronously after a minute.
jobs.In(time.Minute, SendConfirmationEmail{})
}
{% endhighlight %}
## Registering functions
It is possible to register a `func()` as a job by wrapping it in the [`jobs.Func`](https://godoc.org/github.com/revel/modules/jobs/app/jobs#Func)
type. For example:
{% highlight go %}
func sendReminderEmails() {
// Query the DB
// Send some email
}
func init() {
revel.OnAppStart(func() {
jobs.Schedule("@midnight", jobs.Func(sendReminderEmails))
})
}
{% endhighlight %}
## Job Status
The jobs module provides a status page (`/@jobs` url) that shows:
- a list of the scheduled jobs it knows about
- the current status; **IDLE** or **RUNNING**
- the previous and next run times
<div class="alert alert-info">For security purposes, the status page is restricted to requests that originate
from 127.0.0.1.</div>
![Job Status Page](../img/jobs-status.png)
## Constrained pool size
It is possible to configure the job module to limit the number of jobs that are
allowed to run at the same time. This allows the developer to restrict the
resources that could be potentially in use by asynchronous jobs -- typically
interactive responsiveness is valued above asynchronous processing. When a pool
is full of running jobs, new jobs block to wait for running jobs to complete.
**Implementation Note**: The implementation blocks on a channel receive, which is
implemented to be [FIFO](http://en.wikipedia.org/wiki/FIFO) for waiting goroutines (but not specified/required to be
so). [See here for discussion](https://groups.google.com/forum/?fromgroups=#!topic/golang-nuts/CPwv8WlqKag).
## Future areas for development
* Allow access to the job status page with HTTP Basic Authentication credentials
* Allow administrators to run scheduled jobs interactively from the status page
* Provide more visibility into the job runner, e.g. the pool size, the job queue length, etc.
vendor/github.com/revel/modules/jobs/app/controllers/status.go
Generated Vendored
+35
View File
@@ -0,0 +1,35 @@
package controllers
import (
"strings"
"github.com/revel/cron"
"github.com/revel/modules/jobs/app/jobs"
"github.com/revel/revel"
)
type Jobs struct {
*revel.Controller
}
func (c Jobs) Status() revel.Result {
remoteAddress := c.Request.RemoteAddr
if revel.Config.BoolDefault("jobs.acceptproxyaddress", false) {
if proxiedAddress, isProxied := c.Request.Header["X-Forwarded-For"]; isProxied {
remoteAddress = proxiedAddress[0]
}
}
if !strings.HasPrefix(remoteAddress, "127.0.0.1") &&
!strings.HasPrefix(remoteAddress, "::1") &&
!strings.HasPrefix(remoteAddress, "[::1]") {
return c.Forbidden("%s is not local", remoteAddress)
}
entries := jobs.MainCron.Entries()
return c.Render(entries)
}
func init() {
revel.TemplateFuncs["castjob"] = func(job cron.Job) *jobs.Job {
return job.(*jobs.Job)
}
}
vendor/github.com/revel/modules/jobs/app/jobs/job.go
Generated Vendored
+67
View File
@@ -0,0 +1,67 @@
package jobs
import (
"reflect"
"runtime/debug"
"sync"
"sync/atomic"
"github.com/revel/cron"
"github.com/revel/revel"
)
type Job struct {
Name string
inner cron.Job
status uint32
running sync.Mutex
}
const UnNamed = "(unnamed)"
func New(job cron.Job) *Job {
name := reflect.TypeOf(job).Name()
if name == "Func" {
name = UnNamed
}
return &Job{
Name: name,
inner: job,
}
}
func (j *Job) Status() string {
if atomic.LoadUint32(&j.status) > 0 {
return "RUNNING"
}
return "IDLE"
}
func (j *Job) Run() {
// If the job panics, just print a stack trace.
// Don't let the whole process die.
defer func() {
if err := recover(); err != nil {
if revelError := revel.NewErrorFromPanic(err); revelError != nil {
revel.ERROR.Print(err, "\n", revelError.Stack)
} else {
revel.ERROR.Print(err, "\n", string(debug.Stack()))
}
}
}()
if !selfConcurrent {
j.running.Lock()
defer j.running.Unlock()
}
if workPermits != nil {
workPermits <- struct{}{}
defer func() { <-workPermits }()
}
atomic.StoreUint32(&j.status, 1)
defer atomic.StoreUint32(&j.status, 0)
j.inner.Run()
}
vendor/github.com/revel/modules/jobs/app/jobs/jobrunner.go
Generated Vendored
+70
View File
@@ -0,0 +1,70 @@
// A job runner for executing scheduled or ad-hoc tasks asynchronously from HTTP requests.
//
// It adds a couple of features on top of the cron package to make it play nicely with Revel:
//
// 1. Protection against job panics. (They print to ERROR instead of take down the process)
//
// 2. (Optional) Limit on the number of jobs that may run simulatenously, to
// limit resource consumption.
//
// 3. (Optional) Protection against multiple instances of a single job running
// concurrently. If one execution runs into the next, the next will be queued.
//
// 4. Cron expressions may be defined in app.conf and are reusable across jobs.
//
// 5. Job status reporting.
package jobs
import (
"strings"
"time"
"github.com/revel/cron"
"github.com/revel/revel"
)
// Callers can use jobs.Func to wrap a raw func.
// (Copying the type to this package makes it more visible)
//
// For example:
// jobs.Schedule("cron.frequent", jobs.Func(myFunc))
type Func func()
func (r Func) Run() { r() }
func Schedule(spec string, job cron.Job) error {
// Look to see if given spec is a key from the Config.
if strings.HasPrefix(spec, "cron.") {
confSpec, found := revel.Config.String(spec)
if !found {
panic("Cron spec not found: " + spec)
}
spec = confSpec
}
sched, err := cron.Parse(spec)
if err != nil {
return err
}
MainCron.Schedule(sched, New(job))
return nil
}
// Run the given job at a fixed interval.
// The interval provided is the time between the job ending and the job being run again.
// The time that the job takes to run is not included in the interval.
func Every(duration time.Duration, job cron.Job) {
MainCron.Schedule(cron.Every(duration), New(job))
}
// Run the given job right now.
func Now(job cron.Job) {
go New(job).Run()
}
// Run the given job once, after the given delay.
func In(duration time.Duration, job cron.Job) {
go func() {
time.Sleep(duration)
New(job).Run()
}()
}
vendor/github.com/revel/modules/jobs/app/jobs/plugin.go
Generated Vendored
+33
View File
@@ -0,0 +1,33 @@
package jobs
import (
"fmt"
"github.com/revel/cron"
"github.com/revel/revel"
)
const DefaultJobPoolSize = 10
var (
// Singleton instance of the underlying job scheduler.
MainCron *cron.Cron
// This limits the number of jobs allowed to run concurrently.
workPermits chan struct{}
// Is a single job allowed to run concurrently with itself?
selfConcurrent bool
)
func init() {
MainCron = cron.New()
revel.OnAppStart(func() {
if size := revel.Config.IntDefault("jobs.pool", DefaultJobPoolSize); size > 0 {
workPermits = make(chan struct{}, size)
}
selfConcurrent = revel.Config.BoolDefault("jobs.selfconcurrent", false)
MainCron.Start()
fmt.Println("Go to /@jobs to see job status.")
})
}
vendor/github.com/revel/modules/jobs/app/views/Jobs/Status.html
Generated Vendored
+39
View File
@@ -0,0 +1,39 @@
<html>
<head>
<style>
body {
font-size: 12px;
font-family: sans-serif;
}
table {
border-collapse: collapse;
border: none;
}
table td, table th {
padding: 4 10px;
border: none;
}
table tr:nth-child(odd) {
background-color: #f0f0f0;
}
th {
text-align: left;
}
</style>
</head>
<body>
<h1>Scheduled Jobs</h1>
<table>
<tr><th>Name</th><th>Status</th><th>Last run</th><th>Next run</th></tr>
{{range .entries}}
{{$job := castjob .Job}}
<tr>
<td>{{$job.Name}}</td>
<td>{{$job.Status}}</td>
<td>{{if not .Prev.IsZero}}{{.Prev.Format "2006-01-02 15:04:05"}}{{end}}</td>
<td>{{if not .Next.IsZero}}{{.Next.Format "2006-01-02 15:04:05"}}{{end}}</td>
</tr>
{{end}}
</table>
vendor/github.com/revel/modules/jobs/conf/routes
Generated Vendored
+1
View File
@@ -0,0 +1 @@
GET /@jobs Jobs.Status
vendor/github.com/revel/modules/pprof/README.md
Generated Vendored
+19
View File
@@ -0,0 +1,19 @@
Revel pprof module
============
#### How to use:
1. Open your app.conf file and add the following line:
`module.pprof=github.com/revel/modules/pprof`
This will enable the pprof module.
2. Next, open your routes file and add:
`module:pprof` **Note:** Do not change these routes. The pprof command-line tool by default assumes these routes.
Congrats! You can now profile your application. To use the web interface, visit `http://<host>:<port>/debug/pprof`. You can also use the `go tool pprof` command to profile your application and get a little deeper. Use the command by running `go tool pprof <binary> http://<host>:<port>`. For example, if you modified the booking sample, you would run: `go tool pprof $GOPATH/bin/booking http://localhost:9000` (assuming you used the default `revel run` arguments.
The command-line tool will take a 30-second CPU profile, and save the results to a temporary file in your `$HOME/pprof` directory. You can reference this file at a later time by using the same command as above, but by specifying the filename instead of the server address.
In order to fully utilize the command-line tool, you may need the graphviz utilities. If you're on OS X, you can install these easily using Homebrew: `brew install graphviz`.
To read more about profiling Go programs, here is some reading material: [The Go Blog](http://blog.golang.org/profiling-go-programs) : [net/pprof package documentation](http://golang.org/pkg/net/http/pprof/)
vendor/github.com/revel/modules/pprof/app/controllers/pprof.go
Generated Vendored
+39
View File
@@ -0,0 +1,39 @@
package controllers
import (
"github.com/revel/revel"
"net/http"
"net/http/pprof"
)
type Pprof struct {
*revel.Controller
}
// The PprofHandler type makes it easy to call the net/http/pprof handler methods
// since they all have the same method signature
type PprofHandler func(http.ResponseWriter, *http.Request)
func (r PprofHandler) Apply(req *revel.Request, resp *revel.Response) {
r(resp.Out, req.Request)
}
func (c Pprof) Profile() revel.Result {
return PprofHandler(pprof.Profile)
}
func (c Pprof) Symbol() revel.Result {
return PprofHandler(pprof.Symbol)
}
func (c Pprof) Cmdline() revel.Result {
return PprofHandler(pprof.Cmdline)
}
func (c Pprof) Trace() revel.Result {
return PprofHandler(pprof.Trace)
}
func (c Pprof) Index() revel.Result {
return PprofHandler(pprof.Index)
}
vendor/github.com/revel/modules/pprof/conf/routes
Generated Vendored
+6
View File
@@ -0,0 +1,6 @@
GET /pprof/profile Pprof.Profile
GET /pprof/symbol Pprof.Symbol
GET /debug/pprof/cmdline Pprof.Cmdline
GET /debug/pprof/trace Pprof.Trace
GET /debug/pprof/* Pprof.Index
GET /debug/pprof/ Pprof.Index
vendor/github.com/revel/modules/static/app/controllers/static.go
Generated Vendored
+126
View File
@@ -0,0 +1,126 @@
// Copyright (c) 2012-2016 The Revel Framework Authors, All rights reserved.
// Revel Framework source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.
package controllers
import (
"os"
fpath "path/filepath"
"strings"
"syscall"
"github.com/revel/revel"
)
// Static file serving controller
type Static struct {
*revel.Controller
}
// Serve method handles requests for files. The supplied prefix may be absolute
// or relative. If the prefix is relative it is assumed to be relative to the
// application directory. The filepath may either be just a file or an
// additional filepath to search for the given file. This response may return
// the following responses in the event of an error or invalid request;
// 403(Forbidden): If the prefix filepath combination results in a directory.
// 404(Not found): If the prefix and filepath combination results in a non-existent file.
// 500(Internal Server Error): There are a few edge cases that would likely indicate some configuration error outside of revel.
//
// Note that when defining routes in routes/conf the parameters must not have
// spaces around the comma.
// Bad: Static.Serve("public/img", "favicon.png")
// Good: Static.Serve("public/img","favicon.png")
//
// Examples:
// Serving a directory
// Route (conf/routes):
// GET /public/{<.*>filepath} Static.Serve("public")
// Request:
// public/js/sessvars.js
// Calls
// Static.Serve("public","js/sessvars.js")
//
// Serving a file
// Route (conf/routes):
// GET /favicon.ico Static.Serve("public/img","favicon.png")
// Request:
// favicon.ico
// Calls:
// Static.Serve("public/img", "favicon.png")
func (c Static) Serve(prefix, filepath string) revel.Result {
// Fix for #503.
prefix = c.Params.Fixed.Get("prefix")
if prefix == "" {
return c.NotFound("")
}
return serve(c, prefix, filepath)
}
// ServeModule method allows modules to serve binary files. The parameters are the same
// as Static.Serve with the additional module name pre-pended to the list of
// arguments.
func (c Static) ServeModule(moduleName, prefix, filepath string) revel.Result {
// Fix for #503.
prefix = c.Params.Fixed.Get("prefix")
if prefix == "" {
return c.NotFound("")
}
var basePath string
for _, module := range revel.Modules {
if module.Name == moduleName {
basePath = module.Path
}
}
absPath := fpath.Join(basePath, fpath.FromSlash(prefix))
return serve(c, absPath, filepath)
}
// This method allows static serving of application files in a verified manner.
func serve(c Static, prefix, filepath string) revel.Result {
var basePath string
if !fpath.IsAbs(prefix) {
basePath = revel.BasePath
}
basePathPrefix := fpath.Join(basePath, fpath.FromSlash(prefix))
fname := fpath.Join(basePathPrefix, fpath.FromSlash(filepath))
// Verify the request file path is within the application's scope of access
if !strings.HasPrefix(fname, basePathPrefix) {
revel.WARN.Printf("Attempted to read file outside of base path: %s", fname)
return c.NotFound("")
}
// Verify file path is accessible
finfo, err := os.Stat(fname)
if err != nil {
if os.IsNotExist(err) || err.(*os.PathError).Err == syscall.ENOTDIR {
revel.WARN.Printf("File not found (%s): %s ", fname, err)
return c.NotFound("File not found")
}
revel.ERROR.Printf("Error trying to get fileinfo for '%s': %s", fname, err)
return c.RenderError(err)
}
// Disallow directory listing
if finfo.Mode().IsDir() {
revel.WARN.Printf("Attempted directory listing of %s", fname)
return c.Forbidden("Directory listing not allowed")
}
// Open request file path
file, err := os.Open(fname)
if err != nil {
if os.IsNotExist(err) {
revel.WARN.Printf("File not found (%s): %s ", fname, err)
return c.NotFound("File not found")
}
revel.ERROR.Printf("Error opening '%s': %s", fname, err)
return c.RenderError(err)
}
return c.RenderFile(file, revel.Inline)
}
vendor/github.com/revel/modules/template-engine/ace/README.md
Generated Vendored
+59
View File
@@ -0,0 +1,59 @@
# Ace
The [ACE](https://github.com/yosssi/ace) Templating Plugin
- Ace templates have full access to the revel.TemplateFuncs, any function
defined in there can be used by the this engine
- Ace files must be identified by using a `shebang` on the first line
(preferred method) or changing the file extension to home.ace.html.
By default the operation of revel assumes just a `.html` extension for
controller responses so it would make more sense to use the shebang
- Ace templates can be set to be case sensitive by setting
`ace.tempate=case`, default is not case sensitive. If case sensitivity
is off internal imports must be done using lower case
- All function registered in `revel.TemplateFuncs` are available for use
inside the ace framework
##### Details
Ace is a little different of a templating system, its output is a
standard go template but there is no concept of template sets,
instead you build a composite template using
a *base* template and an *inner* template. The
*inner* template can only contain items like :
```
= content main
h2 Inner Template - Main : {{.Msg}}
= content sub
h3 Inner Template - Sub : {{.Msg}}
```
The base template can contain items like
```
= doctype html
html lang=en
head
meta charset=utf-8
title Ace example
= css
h1 { color: blue; }
body
h1 Base Template : {{.Msg}}
#container.wrapper
= yield main
= yield sub
= include inc .Msg
= javascript
alert('{{.Msg}}');
```
You are allowed to include one *inner* template with the base template,
to do so in revel you can extend your controller from the ace controller
and call `RenderAceTemplate(base ,inner string)` which will insert
the inner template using the outer template.
The ace engine requires that you explicitly set the template type on the
template itself by either using the shebang method on the first line
like `#! ace` or having the file name like `template.ace.html`
either method will work.
vendor/github.com/revel/modules/template-engine/ace/app/controllers/ace.go
Generated Vendored
+13
View File
@@ -0,0 +1,13 @@
package controllers
import "github.com/revel/revel"
type AceController struct {
*revel.Controller
}
// Called to render the ace template inner
func (c *AceController) RenderAceTemplate(base, inner string) revel.Result {
c.ViewArgs["ace_inner"] = inner
return c.RenderTemplate(base)
}
vendor/github.com/revel/modules/template-engine/ace/app/init.go
Generated Vendored
+139
View File
@@ -0,0 +1,139 @@
package app
import (
"fmt"
"github.com/revel/revel"
"github.com/yosssi/ace"
"html/template"
"io"
"strings"
)
const ACE_TEMPLATE = "ace"
// Adapter for Go Templates.
type AceTemplate struct {
*template.Template
engine *AceEngine
*revel.TemplateView
File *ace.File
Inner *ace.File
}
// A bit trick of an implementation
// If the arg contains an ace_inner field then that will be used
// to fetch a new template
func (acetmpl AceTemplate) Render(wr io.Writer, arg interface{}) error {
// We can redirect this render to another template if the arguments contain ace_content in them
if argmap, ok := arg.(map[string]interface{}); ok {
if acecontentraw, ok := argmap["ace-inner"]; ok {
acecontent := acecontentraw.(string)
newtemplatename := acetmpl.TemplateName + "-" + acecontent
// Now lookup the template again
if _, ok := acetmpl.engine.templatesByName[newtemplatename]; !ok {
if inner, ok := acetmpl.engine.templatesByName[acecontent]; !ok {
return fmt.Errorf("Inner content %s not found in ace templates", acecontent)
} else {
acetmpl.engine.templatesByName[newtemplatename] = &AceTemplate{
File: acetmpl.File,
Inner: inner.File,
engine: acetmpl.engine,
TemplateView: acetmpl.TemplateView}
}
}
return acetmpl.engine.templatesByName[newtemplatename].renderInternal(wr, arg)
}
}
return acetmpl.renderInternal(wr, arg)
}
func (acetmpl AceTemplate) renderInternal(wr io.Writer, arg interface{}) error {
if acetmpl.Template == nil {
// Compile the template first
if acetmpl.Inner == nil {
acetmpl.Inner = ace.NewFile("", nil)
}
source := ace.NewSource(acetmpl.File, acetmpl.Inner, acetmpl.engine.files)
result, err := ace.ParseSource(source, acetmpl.engine.Options)
if err != nil {
return err
}
if gtemplate, err := ace.CompileResult(acetmpl.TemplateName, result, acetmpl.engine.Options); err != nil {
return err
} else {
acetmpl.Template = gtemplate
}
}
return acetmpl.Execute(wr, arg)
}
type AceEngine struct {
loader *revel.TemplateLoader
templatesByName map[string]*AceTemplate
files []*ace.File
Options *ace.Options
CaseInsensitiveMode bool
}
func (i *AceEngine) ConvertPath(path string) string {
if i.CaseInsensitiveMode {
return strings.ToLower(path)
}
return path
}
func (i *AceEngine) Handles(templateView *revel.TemplateView) bool {
return revel.EngineHandles(i, templateView)
}
func (engine *AceEngine) ParseAndAdd(baseTemplate *revel.TemplateView) error {
// Ace templates must only render views specified for it (no trial and error)
if baseTemplate.EngineType != ACE_TEMPLATE {
return &revel.Error{
Title: "Template Compilation Error",
Path: baseTemplate.FilePath,
Description: "Not correct template for engine",
Line: 1,
SourceLines: baseTemplate.Content(),
}
}
baseTemplate.TemplateName = engine.ConvertPath(baseTemplate.TemplateName)
file := ace.NewFile(baseTemplate.TemplateName, baseTemplate.FileBytes)
engine.files = append(engine.files, file)
engine.templatesByName[baseTemplate.TemplateName] = &AceTemplate{File: file, engine: engine, TemplateView: baseTemplate}
return nil
}
func (engine *AceEngine) Lookup(templateName string) revel.Template {
if tpl, found := engine.templatesByName[engine.ConvertPath(templateName)]; found {
return tpl
}
return nil
}
func (engine *AceEngine) Name() string {
return ACE_TEMPLATE
}
func (engine *AceEngine) Event(action int, i interface{}) {
if action == revel.TEMPLATE_REFRESH_REQUESTED {
engine.templatesByName = map[string]*AceTemplate{}
engine.CaseInsensitiveMode = revel.Config.StringDefault("ace.template.path", "lower") != "case"
}
}
func init() {
revel.RegisterTemplateLoader(ACE_TEMPLATE, func(loader *revel.TemplateLoader) (revel.TemplateEngine, error) {
return &AceEngine{
loader: loader,
templatesByName: map[string]*AceTemplate{},
Options: &ace.Options{FuncMap: revel.TemplateFuncs},
}, nil
})
}
Loaded 100 of 549 files, more files were not shown because too many files have changed in this diff. Show more