You are amazing
++ Jade/Pug is a terse and simple + templating language with + a focus on performance + and powerful features. +
+Hello ` + hello__1 = `!
` +) +func Jade_hello(word string, wr io.Writer) { + buffer := &WriterAsBuffer{wr} + buffer.WriteString(hello__0) + WriteEscString(word, buffer) + buffer.WriteString(hello__1) +} +``` + +`main.go` +```go +package main +//go:generate jade -pkg=main -writer hello.jade + +import "net/http" + +func main() { + http.HandleFunc("/", func(wr http.ResponseWriter, req *http.Request) { + Jade_hello("jade", wr) + }) + http.ListenAndServe(":8080", nil) +} +``` + +output at localhost:8080 +```html +Hello jade!
+``` + +### github.com/Joker/jade package +generate [`html/template`](https://pkg.go.dev/html/template#hdr-Introduction) at runtime +(This case is slightly slower and doesn't support[^2] all features of Jade.go) + +```go +package main + +import ( + "fmt" + "html/template" + "net/http" + + "github.com/Joker/hpp" // Prettify HTML + "github.com/Joker/jade" +) + +func handler(w http.ResponseWriter, r *http.Request) { + jadeTpl, _ := jade.Parse("jade", []byte("doctype 5\n html: body: p Hello #{.Word} !")) + goTpl, _ := template.New("html").Parse(jadeTpl) + + fmt.Printf("output:%s\n\n", hpp.PrPrint(jadeTpl)) + goTpl.Execute(w, struct{ Word string }{"jade"}) +} + +func main() { + http.HandleFunc("/", handler) + http.ListenAndServe(":8080", nil) +} +``` + +console output +```html + + + +Hello {{.Word}} !
+ + +``` + +output at localhost:8080 +```html +Hello jade !
+``` + +## Performance +The data of chart comes from [SlinSo/goTemplateBenchmark](https://github.com/SlinSo/goTemplateBenchmark). + + +## Custom filter :go +This filter is used as helper for command line tool +(to set imports, function name and parameters). +Filter may be placed at any nesting level. +When Jade used as library :go filter is not needed. + +### Nested filter :func +``` +:go:func + CustomNameForTemplateFunc(any []int, input string, args map[string]int) + +:go:func(name) + OnlyCustomNameForTemplateFunc + +:go:func(args) + (only string, input float32, args uint) +``` + +### Nested filter :import +``` +:go:import + "github.com/Joker/jade" + github.com/Joker/hpp +``` + +#### note +[^1]: + `Usage: ./jade [OPTION]... [FILE]...` + ``` + -basedir string + base directory for templates (default "./") + -d string + directory for generated .go files (default "./") + -fmt + HTML pretty print output for generated functions + -inline + inline HTML in generated functions + -pkg string + package name for generated files (default "jade") + -stdbuf + use bytes.Buffer [default bytebufferpool.ByteBuffer] + -stdlib + use stdlib functions + -writer + use io.Writer for output + ``` +[^2]: + Runtime `html/template` generation doesn't support the following features: + `=>` means it generate the folowing template + ``` + for => "{{/* %s, %s */}}{{ range %s }}" + for if => "{{ if gt len %s 0 }}{{/* %s, %s */}}{{ range %s }}" + + multiline code => "{{/* %s */}}" + inheritance block => "{{/* block */}}" + + case statement => "{{/* switch %s */}}" + when => "{{/* case %s: */}}" + default => "{{/* default: */}}" + ``` + You can change this behaviour in [`config.go`](https://github.com/Joker/jade/blob/master/config.go#L24) file. + Partly this problem can be solved by [custom](https://pkg.go.dev/text/template#example-Template-Func) functions. diff --git a/vendor/github.com/Joker/jade/config.go b/vendor/github.com/Joker/jade/config.go new file mode 100644 index 0000000000..5e63083ae3 --- /dev/null +++ b/vendor/github.com/Joker/jade/config.go @@ -0,0 +1,372 @@ +package jade + +import "io/ioutil" + +//go:generate stringer -type=itemType,NodeType -trimprefix=item -output=config_string.go + +var TabSize = 4 +var ReadFunc = ioutil.ReadFile + +var ( + golang_mode = false + tag__bgn = "<%s%s>" + tag__end = "" + tag__void = "<%s%s/>" + tag__arg_esc = ` %s="{{ print %s }}"` + tag__arg_une = ` %s="{{ print %s }}"` + tag__arg_str = ` %s="%s"` + tag__arg_add = `%s " " %s` + tag__arg_bgn = "" + tag__arg_end = "" + + cond__if = "{{ if %s }}" + cond__unless = "{{ if not %s }}" + cond__case = "{{/* switch %s */}}" + cond__while = "{{ range %s }}" + cond__for = "{{/* %s, %s */}}{{ range %s }}" + cond__end = "{{ end }}" + + cond__for_if = "{{ if gt len %s 0 }}{{/* %s, %s */}}{{ range %s }}" + code__for_else = "{{ end }}{{ else }}" + + code__longcode = "{{/* %s */}}" + code__buffered = "{{ %s }}" + code__unescaped = "{{ %s }}" + code__else = "{{ else }}" + code__else_if = "{{ else if %s }}" + code__case_when = "{{/* case %s: */}}" + code__case_def = "{{/* default: */}}" + code__mix_block = "{{/* block */}}" + + text__str = "%s" + text__comment = "" + + mixin__bgn = "\n%s" + mixin__end = "" + mixin__var_bgn = "" + mixin__var = "{{ $%s := %s }}" + mixin__var_rest = "{{ $%s := %#v }}" + mixin__var_end = "\n" + mixin__var_block_bgn = "" + mixin__var_block = "" + mixin__var_block_end = "" +) + +type ReplaseTokens struct { + GolangMode bool + TagBgn string + TagEnd string + TagVoid string + TagArgEsc string + TagArgUne string + TagArgStr string + TagArgAdd string + TagArgBgn string + TagArgEnd string + + CondIf string + CondUnless string + CondCase string + CondWhile string + CondFor string + CondEnd string + CondForIf string + + CodeForElse string + CodeLongcode string + CodeBuffered string + CodeUnescaped string + CodeElse string + CodeElseIf string + CodeCaseWhen string + CodeCaseDef string + CodeMixBlock string + + TextStr string + TextComment string + + MixinBgn string + MixinEnd string + MixinVarBgn string + MixinVar string + MixinVarRest string + MixinVarEnd string + MixinVarBlockBgn string + MixinVarBlock string + MixinVarBlockEnd string +} + +func Config(c ReplaseTokens) { + golang_mode = c.GolangMode + if c.TagBgn != "" { + tag__bgn = c.TagBgn + } + if c.TagEnd != "" { + tag__end = c.TagEnd + } + if c.TagVoid != "" { + tag__void = c.TagVoid + } + if c.TagArgEsc != "" { + tag__arg_esc = c.TagArgEsc + } + if c.TagArgUne != "" { + tag__arg_une = c.TagArgUne + } + if c.TagArgStr != "" { + tag__arg_str = c.TagArgStr + } + if c.TagArgAdd != "" { + tag__arg_add = c.TagArgAdd + } + if c.TagArgBgn != "" { + tag__arg_bgn = c.TagArgBgn + } + if c.TagArgEnd != "" { + tag__arg_end = c.TagArgEnd + } + if c.CondIf != "" { + cond__if = c.CondIf + } + if c.CondUnless != "" { + cond__unless = c.CondUnless + } + if c.CondCase != "" { + cond__case = c.CondCase + } + if c.CondWhile != "" { + cond__while = c.CondWhile + } + if c.CondFor != "" { + cond__for = c.CondFor + } + if c.CondEnd != "" { + cond__end = c.CondEnd + } + if c.CondForIf != "" { + cond__for_if = c.CondForIf + } + if c.CodeForElse != "" { + code__for_else = c.CodeForElse + } + if c.CodeLongcode != "" { + code__longcode = c.CodeLongcode + } + if c.CodeBuffered != "" { + code__buffered = c.CodeBuffered + } + if c.CodeUnescaped != "" { + code__unescaped = c.CodeUnescaped + } + if c.CodeElse != "" { + code__else = c.CodeElse + } + if c.CodeElseIf != "" { + code__else_if = c.CodeElseIf + } + if c.CodeCaseWhen != "" { + code__case_when = c.CodeCaseWhen + } + if c.CodeCaseDef != "" { + code__case_def = c.CodeCaseDef + } + if c.CodeMixBlock != "" { + code__mix_block = c.CodeMixBlock + } + if c.TextStr != "" { + text__str = c.TextStr + } + if c.TextComment != "" { + text__comment = c.TextComment + } + if c.MixinBgn != "" { + mixin__bgn = c.MixinBgn + } + if c.MixinEnd != "" { + mixin__end = c.MixinEnd + } + if c.MixinVarBgn != "" { + mixin__var_bgn = c.MixinVarBgn + } + if c.MixinVar != "" { + mixin__var = c.MixinVar + } + if c.MixinVarRest != "" { + mixin__var_rest = c.MixinVarRest + } + if c.MixinVarEnd != "" { + mixin__var_end = c.MixinVarEnd + } + if c.MixinVarBlockBgn != "" { + mixin__var_block_bgn = c.MixinVarBlockBgn + } + if c.MixinVarBlock != "" { + mixin__var_block = c.MixinVarBlock + } + if c.MixinVarBlockEnd != "" { + mixin__var_block_end = c.MixinVarBlockEnd + } +} + +// + +type goFilter struct { + Name, Args, Import string +} + +var goFlt goFilter + +// global variable access (goFlt) +func UseGoFilter() *goFilter { return &goFlt } + +// + +type itemType int8 + +const ( + itemError itemType = iota // error occurred; value is text of error + itemEOF + + itemEndL + itemIdent + itemEmptyLine // empty line + + itemText // plain text + + itemComment // html comment + itemHTMLTag // htmlHello jade!
+*/ +func Parse(fname string, text []byte) (string, error) { + outTpl, err := New(fname).Parse(text) + if err != nil { + return "", err + } + bb := new(bytes.Buffer) + outTpl.WriteIn(bb) + return bb.String(), nil +} + +// ParseFile parse the jade template file in given filename +func ParseFile(fname string) (string, error) { + text, err := ReadFunc(fname) + if err != nil { + return "", err + } + return Parse(fname, text) +} + +// ParseWithFileSystem parse in context of a http.FileSystem (supports embedded files) +func ParseWithFileSystem(fname string, text []byte, fs http.FileSystem) (str string, err error) { + outTpl := New(fname) + outTpl.fs = fs + + outTpl, err = outTpl.Parse(text) + if err != nil { + return "", err + } + + bb := new(bytes.Buffer) + outTpl.WriteIn(bb) + return bb.String(), nil +} + +// ParseFileFromFileSystem parse template file in context of a http.FileSystem (supports embedded files) +func ParseFileFromFileSystem(fname string, fs http.FileSystem) (str string, err error) { + text, err := readFile(fname, fs) + if err != nil { + return "", err + } + return ParseWithFileSystem(fname, text, fs) +} + +func (t *tree) WriteIn(b io.Writer) { + t.Root.WriteIn(b) +} diff --git a/vendor/github.com/Shopify/goreferrer/.gitignore b/vendor/github.com/Shopify/goreferrer/.gitignore new file mode 100644 index 0000000000..24a676653d --- /dev/null +++ b/vendor/github.com/Shopify/goreferrer/.gitignore @@ -0,0 +1,4 @@ +vendor/ +.DS_Store +.idea +.vscode/ diff --git a/vendor/github.com/Shopify/goreferrer/LICENSE b/vendor/github.com/Shopify/goreferrer/LICENSE new file mode 100644 index 0000000000..d688cb7ffd --- /dev/null +++ b/vendor/github.com/Shopify/goreferrer/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Steven Normore + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/Shopify/goreferrer/README.md b/vendor/github.com/Shopify/goreferrer/README.md new file mode 100644 index 0000000000..1e7d34d562 --- /dev/null +++ b/vendor/github.com/Shopify/goreferrer/README.md @@ -0,0 +1,42 @@ +goreferrer +========== + +A Go module that analyzes and classifies different kinds of referrer URLs (search, social, ...). + +## Example + +```go +package main + +import ( + "fmt" + + "github.com/Shopify/goreferrer" +) + +var urls = []string{ + "http://ca.search.yahoo.com/search?p=hello", + "https://twitter.com/jdoe/status/391149968360103936", + "http://yoursite.com/links", +} + +func main() { + for _, url := range urls { + r := goreferrer.DefaultRules.Parse(url) + switch r.Type { + case goreferrer.Search: + fmt.Printf("Search %s: %s\n", r.Label, r.Query) + case goreferrer.Social: + fmt.Printf("Social %s\n", r.Label) + case goreferrer.Indirect: + fmt.Printf("Indirect: %s\n", r.URL) + } + } +} +``` +Result: +``` +Search Yahoo: hello +Social Twitter +Indirect: http://yoursite.com/links +``` diff --git a/vendor/github.com/Shopify/goreferrer/default_rules.go b/vendor/github.com/Shopify/goreferrer/default_rules.go new file mode 100644 index 0000000000..a646f91e7b --- /dev/null +++ b/vendor/github.com/Shopify/goreferrer/default_rules.go @@ -0,0 +1,4566 @@ +package goreferrer + +import ( + "strings" +) + +var DefaultRules RuleSet + +func init() { + domainRules, err := LoadJsonDomainRules(strings.NewReader(defaultRules)) + if err != nil { + panic(err) + } + + DefaultRules = RuleSet{ + DomainRules: domainRules, + UaRules: map[string]UaRule{ + "Twitter": { + Url: "twitter://twitter.com", + Domain: "twitter", + Tld: "com", + }, + "Pinterest": { + Url: "pinterest://pinterest.com", + Domain: "pinterest", + Tld: "com", + }, + "Facebook": { + Url: "facebook://facebook.com", + Domain: "facebook", + Tld: "com", + }, + "FBAV": { + Url: "facebook://facebook.com", + Domain: "facebook", + Tld: "com", + }, + }, + } +} + +const defaultRules = ` +{ + "email": { + "126 Mail": { + "domains": [ + "mail.126.com" + ] + }, + "163 Mail": { + "domains": [ + "mail.163.com" + ] + }, + "2degrees": { + "domains": [ + "webmail.2degreesbroadband.co.nz" + ] + }, + "Adam Internet": { + "domains": [ + "webmail.adam.com.au" + ] + }, + "AOL Mail": { + "domains": [ + "mail.aol.com", + "cpw.mail.aol.com" + ] + }, + "Xfinity":{ + "domains": [ + "web.mail.comcast.net" + ] + }, + "Bigpond": { + "domains": [ + "webmail.bigpond.com", + "webmail2.bigpond.com", + "email.telstra.com", + "basic.messaging.bigpond.com" + ] + }, + "Commander": { + "domains": [ + "webmail.commander.net.au" + ] + }, + "Daum Mail": { + "domains": [ + "mail2.daum.net", + "mail.daum.net" + ] + }, + "Dodo": { + "domains": [ + "webmail.dodo.com.au" + ] + }, + "Freenet": { + "domains": [ + "webmail.freenet.de" + ] + }, + "Gmail": { + "domains": [ + "mail.google.com", + "inbox.google.com" + ] + }, + "iiNet": { + "domains": [ + "webmail.iinet.net.au", + "mail.iinet.net.au" + ] + }, + "Inbox.com": { + "domains": [ + "inbox.com" + ] + }, + "iPrimus": { + "domains": [ + "webmail.iprimus.com.au" + ] + }, + "Naver Mail": { + "domains": [ + "mail.naver.com" + ] + }, + "Netspace": { + "domains": [ + "webmail.netspace.net.au" + ] + }, + "Optus Zoo": { + "domains": [ + "webmail.optuszoo.com.au", + "webmail.optusnet.com.au" + ] + }, + "Orange Webmail": { + "domains": [ + "orange.fr/webmail" + ] + }, + "Outlook.com": { + "domains": [ + "mail.live.com", + "outlook.live.com", + "blu180.mail.live.com", + "col130.mail.live.com", + "blu184.mail.live.com", + "bay179.mail.live.com", + "col131.mail.live.com", + "blu179.mail.live.com", + "bay180.mail.live.com", + "blu182.mail.live.com", + "blu181.mail.live.com", + "bay182.mail.live.com", + "snt149.mail.live.com", + "bay181.mail.live.com", + "col129.mail.live.com", + "snt148.mail.live.com", + "snt147.mail.live.com", + "snt146.mail.live.com", + "snt153.mail.live.com", + "snt152.mail.live.com", + "snt150.mail.live.com", + "snt151.mail.live.com", + "col128.mail.live.com", + "blu185.mail.live.com", + "dub125.mail.live.com", + "dub128.mail.live.com", + "dub127.mail.live.com", + "dub131.mail.live.com", + "col125.mail.live.com", + "dub130.mail.live.com", + "blu172.mail.live.com", + "bay169.mail.live.com", + "blu175.mail.live.com", + "blu173.mail.live.com", + "bay176.mail.live.com", + "blu176.mail.live.com", + "col126.mail.live.com", + "col127.mail.live.com", + "blu177.mail.live.com", + "blu174.mail.live.com", + "bay174.mail.live.com", + "bay172.mail.live.com", + "blu169.mail.live.com", + "bay177.mail.live.com", + "blu178.mail.live.com", + "blu171.mail.live.com", + "dub126.mail.live.com", + "blu168.mail.live.com", + "bay173.mail.live.com", + "bay175.mail.live.com", + "bay178.mail.live.com", + "bay168.mail.live.com", + "bay167.mail.live.com", + "blu170.mail.live.com", + "dub124.mail.live.com", + "dub122.mail.live.com", + "dub121.mail.live.com", + "dub129.mail.live.com", + "dub114.mail.live.com", + "dub110.mail.live.com", + "dub111.mail.live.com", + "dub113.mail.live.com", + "dub109.mail.live.com", + "dub120.mail.live.com", + "dub115.mail.live.com", + "dub123.mail.live.com", + "dub119.mail.live.com", + "dub118.mail.live.com", + "dub112.mail.live.com", + "dub117.mail.live.com", + "dub116.mail.live.com", + "blu183.mail.live.com" + ] + }, + "QQ Mail": { + "domains": [ + "mail.qq.com" + ] + }, + "Seznam Mail": { + "domains": [ + "email.seznam.cz" + ] + }, + "Virgin": { + "domains": [ + "webmail.virginbroadband.com.au" + ] + }, + "Vodafone": { + "domains": [ + "webmail.vodafone.co.nz" + ] + }, + "Westnet": { + "domains": [ + "webmail.westnet.com.au" + ] + }, + "Yahoo! Mail": { + "domains": [ + "mail.yahoo.net", + "mail.yahoo.com", + "mail.yahoo.co.uk", + "mail.yahoo.co.jp" + ] + }, + "Mynet Mail": { + "domains": [ + "mail.mynet.com" + ] + }, + "Zoho": { + "domains": [ + "mail.zoho.com" + ] + }, + "MailChimp": { + "domains": [ + "list-manage.com", + "list-manage1.com", + "list-manage2.com", + "list-manage3.com", + "list-manage4.com", + "list-manage5.com", + "list-manage6.com", + "list-manage7.com", + "list-manage8.com", + "list-manage9.com" + ] + } + }, + "search": { + "1.cz": { + "domains": [ + "1.cz" + ], + "parameters": [ + "q" + ] + }, + "1&1": { + "domains": [ + "search.1and1.com" + ], + "parameters": [ + "q" + ] + }, + "1und1": { + "domains": [ + "search.1und1.de" + ], + "parameters": [ + "su" + ] + }, + "360.cn": { + "domains": [ + "so.360.cn", + "www.so.com" + ], + "parameters": [ + "q" + ] + }, + "ABCs\u00f8k": { + "domains": [ + "abcsolk.no", + "verden.abcsok.no" + ], + "parameters": [ + "q" + ] + }, + "AOL": { + "domains": [ + "m.search.aol.com", + "search.aol.ca", + "www.aol.com", + "search.aol.com", + "search.aol.it", + "aolsearch.aol.com", + "aolsearch.com", + "www.aolrecherche.aol.fr", + "www.aolrecherches.aol.fr", + "www.aolimages.aol.fr", + "aim.search.aol.com", + "www.recherche.aol.fr", + "recherche.aol.fr", + "find.web.aol.com", + "recherche.aol.ca", + "aolsearch.aol.co.uk", + "search.aol.co.uk", + "aolrecherche.aol.fr", + "sucheaol.aol.de", + "suche.aol.de", + "suche.aolsvc.de", + "aolbusqueda.aol.com.mx", + "alicesuche.aol.de", + "alicesuchet.aol.de", + "suchet2.aol.de", + "search.hp.my.aol.com.au", + "search.hp.my.aol.de", + "search.hp.my.aol.it", + "search-intl.netscape.com" + ], + "parameters": [ + "q", + "query" + ] + }, + "APOLL07": { + "domains": [ + "apollo7.de" + ], + "parameters": [ + "query" + ] + }, + "Abacho": { + "domains": [ + "www.abacho.de", + "www.abacho.com", + "www.abacho.co.uk", + "www.se.abacho.com", + "www.tr.abacho.com", + "www.abacho.at", + "www.abacho.fr", + "www.abacho.es", + "www.abacho.ch", + "www.abacho.it" + ], + "parameters": [ + "q" + ] + }, + "Acoon": { + "domains": [ + "www.acoon.de" + ], + "parameters": [ + "begriff" + ] + }, + "Alexa": { + "domains": [ + "alexa.com", + "search.toolbars.alexa.com" + ], + "parameters": [ + "q" + ] + }, + "Alice Adsl": { + "domains": [ + "rechercher.aliceadsl.fr" + ], + "parameters": [ + "q" + ] + }, + "AllTheWeb": { + "domains": [ + "www.alltheweb.com" + ], + "parameters": [ + "q" + ] + }, + "Altavista": { + "domains": [ + "www.altavista.com", + "search.altavista.com", + "listings.altavista.com", + "altavista.de", + "altavista.fr", + "be-nl.altavista.com", + "be-fr.altavista.com" + ], + "parameters": [ + "q" + ] + }, + "Apollo Latvia": { + "domains": [ + "apollo.lv/portal/search/" + ], + "parameters": [ + "q" + ] + }, + "Amazon": { + "domains": [ + "amazon.com", + "amazon.co.uk", + "amazon.ca", + "amazon.de", + "amazon.fr", + "amazonaws.com", + "amazon.co.jp", + "amazon.es", + "amazon.it", + "amazon.in", + "www.amazon.com" + ], + "parameters": [ + "keywords", + "field-keywords" + ] + }, + "Apontador": { + "domains": [ + "apontador.com.br", + "www.apontador.com.br" + ], + "parameters": [ + "q" + ] + }, + "Aport": { + "domains": [ + "sm.aport.ru" + ], + "parameters": [ + "r" + ] + }, + "Arcor": { + "domains": [ + "www.arcor.de" + ], + "parameters": [ + "Keywords" + ] + }, + "Arianna": { + "domains": [ + "arianna.libero.it", + "www.arianna.com" + ], + "parameters": [ + "query" + ] + }, + "Ask": { + "domains": [ + "ask.com", + "www.ask.com", + "web.ask.com", + "int.ask.com", + "mws.ask.com", + "uk.ask.com", + "images.ask.com", + "ask.reference.com", + "www.askkids.com", + "iwon.ask.com", + "www.ask.co.uk", + "www.qbyrd.com", + "search-results.com", + "uk.search-results.com", + "www.search-results.com", + "int.search-results.com" + ], + "parameters": [ + "q" + ] + }, + "Ask Toolbar": { + "domains": [ + "search.tb.ask.com" + ], + "parameters": [ + "searchfor" + ] + }, + "Atlas": { + "domains": [ + "searchatlas.centrum.cz" + ], + "parameters": [ + "q" + ] + }, + "Austronaut": { + "domains": [ + "www2.austronaut.at", + "www1.astronaut.at" + ], + "parameters": [ + "q" + ] + }, + "Babylon": { + "domains": [ + "search.babylon.com", + "searchassist.babylon.com" + ], + "parameters": [ + "q" + ] + }, + "Baidu": { + "domains": [ + "www.baidu.com", + "www1.baidu.com", + "zhidao.baidu.com", + "tieba.baidu.com", + "news.baidu.com", + "web.gougou.com", + "m.baidu.com", + "image.baidu.com", + "tieba.baidu.com", + "fanyi.baidu.com", + "zhidao.baidu.com", + "www.baidu.co.th", + "m5.baidu.com", + "m.siteapp.baidu.com" + ], + "parameters": [ + "wd", + "word", + "kw", + "k" + ] + }, + "Biglobe": { + "domains": [ + "cgi.search.biglobe.ne.jp" + ], + "parameters": [ + "q" + ] + }, + "Bing": { + "domains": [ + "bing.com", + "www.bing.com", + "msnbc.msn.com", + "dizionario.it.msn.com", + "cc.bingj.com", + "m.bing.com" + ], + "parameters": [ + "q", + "Q" + ] + }, + "Bing Images": { + "domains": [ + "bing.com/images/search", + "www.bing.com/images/search" + ], + "parameters": [ + "q", + "Q" + ] + }, + "Blogdigger": { + "domains": [ + "www.blogdigger.com" + ], + "parameters": [ + "q" + ] + }, + "Blogpulse": { + "domains": [ + "www.blogpulse.com" + ], + "parameters": [ + "query" + ] + }, + "Bluewin": { + "domains": [ + "search.bluewin.ch" + ], + "parameters": [ + "searchTerm" + ] + }, + "British Telecommunications": { + "domains": [ + "search.bt.com" + ], + "parameters": [ + "q" + ] + }, + "Centrum": { + "domains": [ + "serach.centrum.cz", + "morfeo.centrum.cz" + ], + "parameters": [ + "q" + ] + }, + "Certified-Toolbar": { + "domains": [ + "search.certified-toolbar.com" + ], + "parameters": [ + "q" + ] + }, + "Charter": { + "domains": [ + "www.charter.net" + ], + "parameters": [ + "q" + ] + }, + "Clix": { + "domains": [ + "pesquisa.clix.pt" + ], + "parameters": [ + "question" + ] + }, + "Comcast": { + "domains": [ + "search.comcast.net", + "comcast.net", + "xfinity.com" + ], + "parameters": [ + "q" + ] + }, + "Compuserve": { + "domains": [ + "websearch.cs.com" + ], + "parameters": [ + "query" + ] + }, + "Conduit": { + "domains": [ + "search.conduit.com" + ], + "parameters": [ + "q" + ] + }, + "Crawler": { + "domains": [ + "www.crawler.com" + ], + "parameters": [ + "q" + ] + }, + "Cuil": { + "domains": [ + "www.cuil.com" + ], + "parameters": [ + "q" + ] + }, + "Daemon search": { + "domains": [ + "daemon-search.com", + "my.daemon-search.com" + ], + "parameters": [ + "q" + ] + }, + "Dalesearch": { + "domains": [ + "www.dalesearch.com" + ], + "parameters": [ + "q" + ] + }, + "DasOertliche": { + "domains": [ + "www.dasoertliche.de" + ], + "parameters": [ + "kw" + ] + }, + "DasTelefonbuch": { + "domains": [ + "www1.dastelefonbuch.de" + ], + "parameters": [ + "kw" + ] + }, + "Daum": { + "domains": [ + "search.daum.net" + ], + "parameters": [ + "q" + ] + }, + "Delfi": { + "domains": [ + "otsing.delfi.ee" + ], + "parameters": [ + "q" + ] + }, + "Delfi latvia": { + "domains": [ + "smart.delfi.lv" + ], + "parameters": [ + "q" + ] + }, + "Digg": { + "domains": [ + "digg.com" + ], + "parameters": [ + "s" + ] + }, + "Dodo": { + "domains": [ + "google.dodo.com.au" + ], + "parameters": [ + "q" + ] + }, + "DuckDuckGo": { + "domains": [ + "duckduckgo.com" + ], + "parameters": [ + "q" + ] + }, + "Ecosia": { + "domains": [ + "ecosia.org" + ], + "parameters": [ + "q" + ] + }, + "El Mundo": { + "domains": [ + "ariadna.elmundo.es" + ], + "parameters": [ + "q" + ] + }, + "Eniro": { + "domains": [ + "www.eniro.se" + ], + "parameters": [ + "q", + "search_word" + ] + }, + "Eurip": { + "domains": [ + "www.eurip.com" + ], + "parameters": [ + "q" + ] + }, + "Euroseek": { + "domains": [ + "www.euroseek.com" + ], + "parameters": [ + "string" + ] + }, + "Everyclick": { + "domains": [ + "www.everyclick.com" + ], + "parameters": [ + "keyword" + ] + }, + "Exalead": { + "domains": [ + "www.exalead.fr", + "www.exalead.com" + ], + "parameters": [ + "q" + ] + }, + "Excite": { + "domains": [ + "search.excite.it", + "search.excite.fr", + "search.excite.de", + "search.excite.co.uk", + "serach.excite.es", + "search.excite.nl", + "msxml.excite.com", + "www.excite.co.jp" + ], + "parameters": [ + "q", + "search" + ] + }, + "Fast Browser Search": { + "domains": [ + "www.fastbrowsersearch.com" + ], + "parameters": [ + "q" + ] + }, + "Finderoo": { + "domains": [ + "www.finderoo.com" + ], + "parameters": [ + "q" + ] + }, + "Findwide": { + "domains": [ + "search.findwide.com" + ], + "parameters": [ + "k" + ] + }, + "Fireball": { + "domains": [ + "www.fireball.de" + ], + "parameters": [ + "q" + ] + }, + "Firstfind": { + "domains": [ + "www.firstsfind.com" + ], + "parameters": [ + "qry" + ] + }, + "Fixsuche": { + "domains": [ + "www.fixsuche.de" + ], + "parameters": [ + "q" + ] + }, + "Flix": { + "domains": [ + "www.flix.de" + ], + "parameters": [ + "keyword" + ] + }, + "Flyingbird": { + "domains": [ + "inspsearch.com" + ], + "parameters": [ + "q" + ] + }, + "Forestle": { + "domains": [ + "forestle.org", + "www.forestle.org", + "forestle.mobi" + ], + "parameters": [ + "q" + ] + }, + "Francite": { + "domains": [ + "recherche.francite.com" + ], + "parameters": [ + "name" + ] + }, + "Free": { + "domains": [ + "search.free.fr", + "search1-2.free.fr", + "search1-1.free.fr" + ], + "parameters": [ + "q" + ] + }, + "Freecause": { + "domains": [ + "search.freecause.com" + ], + "parameters": [ + "p" + ] + }, + "Freenet": { + "domains": [ + "suche.freenet.de" + ], + "parameters": [ + "query", + "Keywords" + ] + }, + "Freshweather": { + "domains": [ + "www.fresh-weather.com" + ], + "parameters": [ + "q" + ] + }, + "FriendFeed": { + "domains": [ + "friendfeed.com" + ], + "parameters": [ + "q" + ] + }, + "GAIS": { + "domains": [ + "gais.cs.ccu.edu.tw" + ], + "parameters": [ + "q" + ] + }, + "GMX": { + "domains": [ + "suche.gmx.net" + ], + "parameters": [ + "su" + ] + }, + "Geona": { + "domains": [ + "geona.net" + ], + "parameters": [ + "q" + ] + }, + "Genieo": { + "domains": [ + "search.genieo.com" + ], + "parameters": [ + "q" + ] + }, + "Gigablast": { + "domains": [ + "www.gigablast.com", + "dir.gigablast.com" + ], + "parameters": [ + "q" + ] + }, + "Globososo": { + "domains": [ + "searches.globososo.com", + "search.globososo.com" + ], + "parameters": [ + "q" + ] + }, + "Gnadenmeer": { + "domains": [ + "www.gnadenmeer.de" + ], + "parameters": [ + "keyword" + ] + }, + "Gomeo": { + "domains": [ + "www.gomeo.com" + ], + "parameters": [ + "Keywords" + ] + }, + "Google": { + "domains": [ + "www.google.com", + "www.google.ac", + "www.google.ad", + "www.google.al", + "www.google.com.af", + "www.google.com.ag", + "www.google.com.ai", + "www.google.am", + "www.google.it.ao", + "www.google.com.ar", + "www.google.as", + "www.google.at", + "www.google.com.au", + "www.google.az", + "www.google.ba", + "www.google.com.bd", + "www.google.be", + "www.google.bf", + "www.google.bg", + "www.google.com.bh", + "www.google.bi", + "www.google.bj", + "www.google.com.bn", + "www.google.com.bo", + "www.google.com.br", + "www.google.bs", + "www.google.co.bw", + "www.google.com.by", + "www.google.by", + "www.google.com.bz", + "www.google.ca", + "www.google.com.kh", + "www.google.cc", + "www.google.cd", + "www.google.cf", + "www.google.cat", + "www.google.cg", + "www.google.ch", + "www.google.ci", + "www.google.co.ck", + "www.google.cl", + "www.google.cm", + "www.google.cn", + "www.google.com.co", + "www.google.co.cr", + "www.google.com.cu", + "www.google.cv", + "www.google.com.cy", + "www.google.cz", + "www.google.de", + "www.google.dj", + "www.google.dk", + "www.google.dm", + "www.google.com.do", + "www.google.dz", + "www.google.com.ec", + "www.google.ee", + "www.google.com.eg", + "www.google.es", + "www.google.com.et", + "www.google.fi", + "www.google.com.fj", + "www.google.fm", + "www.google.fr", + "www.google.ga", + "www.google.gd", + "www.google.ge", + "www.google.gf", + "www.google.gg", + "www.google.com.gh", + "www.google.com.gi", + "www.google.gl", + "www.google.gm", + "www.google.gp", + "www.google.gr", + "www.google.com.gt", + "www.google.gy", + "www.google.com.hk", + "www.google.hn", + "www.google.hr", + "www.google.ht", + "www.google.hu", + "www.google.co.id", + "www.google.iq", + "www.google.ie", + "www.google.co.il", + "www.google.im", + "www.google.co.in", + "www.google.io", + "www.google.is", + "www.google.it", + "www.google.je", + "www.google.com.jm", + "www.google.jo", + "www.google.co.jp", + "www.google.co.ke", + "www.google.com.kh", + "www.google.ki", + "www.google.kg", + "www.google.co.kr", + "www.google.com.kw", + "www.google.kz", + "www.google.la", + "www.google.com.lb", + "www.google.com.lc", + "www.google.li", + "www.google.lk", + "www.google.co.ls", + "www.google.lt", + "www.google.lu", + "www.google.lv", + "www.google.com.ly", + "www.google.co.ma", + "www.google.md", + "www.google.me", + "www.google.mg", + "www.google.mk", + "www.google.ml", + "www.google.mn", + "www.google.ms", + "www.google.com.mt", + "www.google.mu", + "www.google.mv", + "www.google.mw", + "www.google.com.mx", + "www.google.com.my", + "www.google.co.mz", + "www.google.com.na", + "www.google.ne", + "www.google.com.nf", + "www.google.com.ng", + "www.google.com.ni", + "www.google.nl", + "www.google.no", + "www.google.com.np", + "www.google.nr", + "www.google.nu", + "www.google.co.nz", + "www.google.com.om", + "www.google.com.pa", + "www.google.com.pe", + "www.google.com.ph", + "www.google.com.pk", + "www.google.pl", + "www.google.pn", + "www.google.com.pr", + "www.google.ps", + "www.google.pt", + "www.google.com.py", + "www.google.com.qa", + "www.google.ro", + "www.google.rs", + "www.google.ru", + "www.google.rw", + "www.google.com.sa", + "www.google.com.sb", + "www.google.sc", + "www.google.se", + "www.google.com.sg", + "www.google.sh", + "www.google.si", + "www.google.sk", + "www.google.com.sl", + "www.google.sn", + "www.google.sm", + "www.google.so", + "www.google.st", + "www.google.com.sv", + "www.google.td", + "www.google.tg", + "www.google.co.th", + "www.google.com.tj", + "www.google.tk", + "www.google.tl", + "www.google.tm", + "www.google.to", + "www.google.com.tn", + "www.google.com.tr", + "www.google.tt", + "www.google.tn", + "www.google.com.tw", + "www.google.co.tz", + "www.google.com.ua", + "www.google.co.ug", + "www.google.ae", + "www.google.co.uk", + "www.google.us", + "www.google.com.uy", + "www.google.co.uz", + "www.google.com.vc", + "www.google.co.ve", + "www.google.vg", + "www.google.co.vi", + "www.google.com.vn", + "www.google.vu", + "www.google.ws", + "www.google.co.za", + "www.google.co.zm", + "www.google.co.zw", + "www.google.com.mm", + "www.google.sr", + "www.google.com.pg", + "www.google.bt", + "www.google.ng", + "www.google.com.iq", + "www.google.co.ao", + "google.com", + "google.ac", + "google.ad", + "google.al", + "google.com.af", + "google.com.ag", + "google.com.ai", + "google.am", + "google.it.ao", + "google.com.ar", + "google.as", + "google.at", + "google.com.au", + "google.az", + "google.ba", + "google.com.bd", + "google.be", + "google.bf", + "google.bg", + "google.com.bh", + "google.bi", + "google.bj", + "google.com.bn", + "google.com.bo", + "google.com.br", + "google.bs", + "google.co.bw", + "google.com.by", + "google.by", + "google.com.bz", + "google.ca", + "google.com.kh", + "google.cc", + "google.cd", + "google.cf", + "google.cat", + "google.cg", + "google.ch", + "google.ci", + "google.co.ck", + "google.cl", + "google.cm", + "google.cn", + "google.com.co", + "google.co.cr", + "google.com.cu", + "google.cv", + "google.com.cy", + "google.cz", + "google.de", + "google.dj", + "google.dk", + "google.dm", + "google.com.do", + "google.dz", + "google.com.ec", + "google.ee", + "google.com.eg", + "google.es", + "google.com.et", + "google.fi", + "google.com.fj", + "google.fm", + "google.fr", + "google.ga", + "google.gd", + "google.ge", + "google.gf", + "google.gg", + "google.com.gh", + "google.com.gi", + "google.gl", + "google.gm", + "google.gp", + "google.gr", + "google.com.gt", + "google.gy", + "google.com.hk", + "google.hn", + "google.hr", + "google.ht", + "google.hu", + "google.co.id", + "google.iq", + "google.ie", + "google.co.il", + "google.im", + "google.co.in", + "google.io", + "google.is", + "google.it", + "google.je", + "google.com.jm", + "google.jo", + "google.co.jp", + "google.co.ke", + "google.com.kh", + "google.ki", + "google.kg", + "google.co.kr", + "google.com.kw", + "google.kz", + "google.la", + "google.com.lb", + "google.com.lc", + "google.li", + "google.lk", + "google.co.ls", + "google.lt", + "google.lu", + "google.lv", + "google.com.ly", + "google.co.ma", + "google.md", + "google.me", + "google.mg", + "google.mk", + "google.ml", + "google.mn", + "google.ms", + "google.com.mt", + "google.mu", + "google.mv", + "google.mw", + "google.com.mx", + "google.com.my", + "google.co.mz", + "google.com.na", + "google.ne", + "google.com.nf", + "google.com.ng", + "google.com.ni", + "google.nl", + "google.no", + "google.com.np", + "google.nr", + "google.nu", + "google.co.nz", + "google.com.om", + "google.com.pa", + "google.com.pe", + "google.com.ph", + "google.com.pk", + "google.pl", + "google.pn", + "google.com.pr", + "google.ps", + "google.pt", + "google.com.py", + "google.com.qa", + "google.ro", + "google.rs", + "google.ru", + "google.rw", + "google.com.sa", + "google.com.sb", + "google.sc", + "google.se", + "google.com.sg", + "google.sh", + "google.si", + "google.sk", + "google.com.sl", + "google.sn", + "google.sm", + "google.so", + "google.st", + "google.com.sv", + "google.td", + "google.tg", + "google.tn", + "google.co.th", + "google.com.tj", + "google.tk", + "google.tl", + "google.tm", + "google.to", + "google.com.tn", + "google.com.tr", + "google.tt", + "google.com.tw", + "google.co.tz", + "google.com.ua", + "google.co.ug", + "google.ae", + "google.co.uk", + "google.us", + "google.com.uy", + "google.co.uz", + "google.com.vc", + "google.co.ve", + "google.vg", + "google.co.vi", + "google.com.vn", + "google.vu", + "google.ws", + "google.co.za", + "google.co.zm", + "google.co.zw", + "search.avg.com", + "isearch.avg.com", + "www.cnn.com", + "darkoogle.com", + "search.darkoogle.com", + "search.foxtab.com", + "www.gooofullsearch.com", + "search.hiyo.com", + "search.incredimail.com", + "search1.incredimail.com", + "search2.incredimail.com", + "search3.incredimail.com", + "search4.incredimail.com", + "search.incredibar.com", + "search.sweetim.com", + "www.fastweb.it", + "search.juno.com", + "find.tdc.dk", + "searchresults.verizon.com", + "search.walla.co.il", + "search.alot.com", + "www.googleearth.de", + "www.googleearth.fr", + "webcache.googleusercontent.com", + "encrypted.google.com", + "googlesyndicatedsearch.com", + "www.googleadservices.com" + ], + "parameters": [ + "q", + "query", + "Keywords", + "*" + ] + }, + "Google Blogsearch": { + "domains": [ + "blogsearch.google.ac", + "blogsearch.google.ad", + "blogsearch.google.ae", + "blogsearch.google.am", + "blogsearch.google.as", + "blogsearch.google.at", + "blogsearch.google.az", + "blogsearch.google.ba", + "blogsearch.google.be", + "blogsearch.google.bf", + "blogsearch.google.bg", + "blogsearch.google.bi", + "blogsearch.google.bj", + "blogsearch.google.bs", + "blogsearch.google.by", + "blogsearch.google.ca", + "blogsearch.google.cat", + "blogsearch.google.cc", + "blogsearch.google.cd", + "blogsearch.google.cf", + "blogsearch.google.cg", + "blogsearch.google.ch", + "blogsearch.google.ci", + "blogsearch.google.cl", + "blogsearch.google.cm", + "blogsearch.google.cn", + "blogsearch.google.co.bw", + "blogsearch.google.co.ck", + "blogsearch.google.co.cr", + "blogsearch.google.co.id", + "blogsearch.google.co.il", + "blogsearch.google.co.in", + "blogsearch.google.co.jp", + "blogsearch.google.co.ke", + "blogsearch.google.co.kr", + "blogsearch.google.co.ls", + "blogsearch.google.co.ma", + "blogsearch.google.co.mz", + "blogsearch.google.co.nz", + "blogsearch.google.co.th", + "blogsearch.google.co.tz", + "blogsearch.google.co.ug", + "blogsearch.google.co.uk", + "blogsearch.google.co.uz", + "blogsearch.google.co.ve", + "blogsearch.google.co.vi", + "blogsearch.google.co.za", + "blogsearch.google.co.zm", + "blogsearch.google.co.zw", + "blogsearch.google.com", + "blogsearch.google.com.af", + "blogsearch.google.com.ag", + "blogsearch.google.com.ai", + "blogsearch.google.com.ar", + "blogsearch.google.com.au", + "blogsearch.google.com.bd", + "blogsearch.google.com.bh", + "blogsearch.google.com.bn", + "blogsearch.google.com.bo", + "blogsearch.google.com.br", + "blogsearch.google.com.by", + "blogsearch.google.com.bz", + "blogsearch.google.com.co", + "blogsearch.google.com.cu", + "blogsearch.google.com.cy", + "blogsearch.google.com.do", + "blogsearch.google.com.ec", + "blogsearch.google.com.eg", + "blogsearch.google.com.et", + "blogsearch.google.com.fj", + "blogsearch.google.com.gh", + "blogsearch.google.com.gi", + "blogsearch.google.com.gt", + "blogsearch.google.com.hk", + "blogsearch.google.com.jm", + "blogsearch.google.com.kh", + "blogsearch.google.com.kh", + "blogsearch.google.com.kw", + "blogsearch.google.com.lb", + "blogsearch.google.com.lc", + "blogsearch.google.com.ly", + "blogsearch.google.com.mt", + "blogsearch.google.com.mx", + "blogsearch.google.com.my", + "blogsearch.google.com.na", + "blogsearch.google.com.nf", + "blogsearch.google.com.ng", + "blogsearch.google.com.ni", + "blogsearch.google.com.np", + "blogsearch.google.com.om", + "blogsearch.google.com.pa", + "blogsearch.google.com.pe", + "blogsearch.google.com.ph", + "blogsearch.google.com.pk", + "blogsearch.google.com.pr", + "blogsearch.google.com.py", + "blogsearch.google.com.qa", + "blogsearch.google.com.sa", + "blogsearch.google.com.sb", + "blogsearch.google.com.sg", + "blogsearch.google.com.sl", + "blogsearch.google.com.sv", + "blogsearch.google.com.tj", + "blogsearch.google.com.tn", + "blogsearch.google.com.tr", + "blogsearch.google.com.tw", + "blogsearch.google.com.ua", + "blogsearch.google.com.uy", + "blogsearch.google.com.vc", + "blogsearch.google.com.vn", + "blogsearch.google.cv", + "blogsearch.google.cz", + "blogsearch.google.de", + "blogsearch.google.dj", + "blogsearch.google.dk", + "blogsearch.google.dm", + "blogsearch.google.dz", + "blogsearch.google.ee", + "blogsearch.google.es", + "blogsearch.google.fi", + "blogsearch.google.fm", + "blogsearch.google.fr", + "blogsearch.google.ga", + "blogsearch.google.gd", + "blogsearch.google.ge", + "blogsearch.google.gf", + "blogsearch.google.gg", + "blogsearch.google.gl", + "blogsearch.google.gm", + "blogsearch.google.gp", + "blogsearch.google.gr", + "blogsearch.google.gy", + "blogsearch.google.hn", + "blogsearch.google.hr", + "blogsearch.google.ht", + "blogsearch.google.hu", + "blogsearch.google.ie", + "blogsearch.google.im", + "blogsearch.google.io", + "blogsearch.google.iq", + "blogsearch.google.is", + "blogsearch.google.it", + "blogsearch.google.it.ao", + "blogsearch.google.je", + "blogsearch.google.jo", + "blogsearch.google.kg", + "blogsearch.google.ki", + "blogsearch.google.kz", + "blogsearch.google.la", + "blogsearch.google.li", + "blogsearch.google.lk", + "blogsearch.google.lt", + "blogsearch.google.lu", + "blogsearch.google.lv", + "blogsearch.google.md", + "blogsearch.google.me", + "blogsearch.google.mg", + "blogsearch.google.mk", + "blogsearch.google.ml", + "blogsearch.google.mn", + "blogsearch.google.ms", + "blogsearch.google.mu", + "blogsearch.google.mv", + "blogsearch.google.mw", + "blogsearch.google.ne", + "blogsearch.google.nl", + "blogsearch.google.no", + "blogsearch.google.nr", + "blogsearch.google.nu", + "blogsearch.google.pl", + "blogsearch.google.pn", + "blogsearch.google.ps", + "blogsearch.google.pt", + "blogsearch.google.ro", + "blogsearch.google.rs", + "blogsearch.google.ru", + "blogsearch.google.rw", + "blogsearch.google.sc", + "blogsearch.google.se", + "blogsearch.google.sh", + "blogsearch.google.si", + "blogsearch.google.sk", + "blogsearch.google.sm", + "blogsearch.google.sn", + "blogsearch.google.so", + "blogsearch.google.st", + "blogsearch.google.td", + "blogsearch.google.tg", + "blogsearch.google.tk", + "blogsearch.google.tl", + "blogsearch.google.tm", + "blogsearch.google.to", + "blogsearch.google.tt", + "blogsearch.google.us", + "blogsearch.google.vg", + "blogsearch.google.vu", + "blogsearch.google.ws" + ], + "parameters": [ + "q" + ] + }, + "Google Images": { + "domains": [ + "google.ac/imgres", + "google.ad/imgres", + "google.ae/imgres", + "google.am/imgres", + "google.as/imgres", + "google.at/imgres", + "google.az/imgres", + "google.ba/imgres", + "google.be/imgres", + "google.bf/imgres", + "google.bg/imgres", + "google.bi/imgres", + "google.bj/imgres", + "google.bs/imgres", + "google.by/imgres", + "google.ca/imgres", + "google.cat/imgres", + "google.cc/imgres", + "google.cd/imgres", + "google.cf/imgres", + "google.cg/imgres", + "google.ch/imgres", + "google.ci/imgres", + "google.cl/imgres", + "google.cm/imgres", + "google.cn/imgres", + "google.co.bw/imgres", + "google.co.ck/imgres", + "google.co.cr/imgres", + "google.co.id/imgres", + "google.co.il/imgres", + "google.co.in/imgres", + "google.co.jp/imgres", + "google.co.ke/imgres", + "google.co.kr/imgres", + "google.co.ls/imgres", + "google.co.ma/imgres", + "google.co.mz/imgres", + "google.co.nz/imgres", + "google.co.th/imgres", + "google.co.tz/imgres", + "google.co.ug/imgres", + "google.co.uk/imgres", + "google.co.uz/imgres", + "google.co.ve/imgres", + "google.co.vi/imgres", + "google.co.za/imgres", + "google.co.zm/imgres", + "google.co.zw/imgres", + "google.com/imgres", + "google.com.af/imgres", + "google.com.ag/imgres", + "google.com.ai/imgres", + "google.com.ar/imgres", + "google.com.au/imgres", + "google.com.bd/imgres", + "google.com.bh/imgres", + "google.com.bn/imgres", + "google.com.bo/imgres", + "google.com.br/imgres", + "google.com.by/imgres", + "google.com.bz/imgres", + "google.com.co/imgres", + "google.com.cu/imgres", + "google.com.cy/imgres", + "google.com.do/imgres", + "google.com.ec/imgres", + "google.com.eg/imgres", + "google.com.et/imgres", + "google.com.fj/imgres", + "google.com.gh/imgres", + "google.com.gi/imgres", + "google.com.gt/imgres", + "google.com.hk/imgres", + "google.com.jm/imgres", + "google.com.kh/imgres", + "google.com.kh/imgres", + "google.com.kw/imgres", + "google.com.lb/imgres", + "google.com.lc/imgres", + "google.com.ly/imgres", + "google.com.mt/imgres", + "google.com.mx/imgres", + "google.com.my/imgres", + "google.com.na/imgres", + "google.com.nf/imgres", + "google.com.ng/imgres", + "google.com.ni/imgres", + "google.com.np/imgres", + "google.com.om/imgres", + "google.com.pa/imgres", + "google.com.pe/imgres", + "google.com.ph/imgres", + "google.com.pk/imgres", + "google.com.pr/imgres", + "google.com.py/imgres", + "google.com.qa/imgres", + "google.com.sa/imgres", + "google.com.sb/imgres", + "google.com.sg/imgres", + "google.com.sl/imgres", + "google.com.sv/imgres", + "google.com.tj/imgres", + "google.com.tn/imgres", + "google.com.tr/imgres", + "google.com.tw/imgres", + "google.com.ua/imgres", + "google.com.uy/imgres", + "google.com.vc/imgres", + "google.com.vn/imgres", + "google.cv/imgres", + "google.cz/imgres", + "google.de/imgres", + "google.dj/imgres", + "google.dk/imgres", + "google.dm/imgres", + "google.dz/imgres", + "google.ee/imgres", + "google.es/imgres", + "google.fi/imgres", + "google.fm/imgres", + "google.fr/imgres", + "google.ga/imgres", + "google.gd/imgres", + "google.ge/imgres", + "google.gf/imgres", + "google.gg/imgres", + "google.gl/imgres", + "google.gm/imgres", + "google.gp/imgres", + "google.gr/imgres", + "google.gy/imgres", + "google.hn/imgres", + "google.hr/imgres", + "google.ht/imgres", + "google.hu/imgres", + "google.ie/imgres", + "google.im/imgres", + "google.io/imgres", + "google.iq/imgres", + "google.is/imgres", + "google.it/imgres", + "google.it.ao/imgres", + "google.je/imgres", + "google.jo/imgres", + "google.kg/imgres", + "google.ki/imgres", + "google.kz/imgres", + "google.la/imgres", + "google.li/imgres", + "google.lk/imgres", + "google.lt/imgres", + "google.lu/imgres", + "google.lv/imgres", + "google.md/imgres", + "google.me/imgres", + "google.mg/imgres", + "google.mk/imgres", + "google.ml/imgres", + "google.mn/imgres", + "google.ms/imgres", + "google.mu/imgres", + "google.mv/imgres", + "google.mw/imgres", + "google.ne/imgres", + "google.nl/imgres", + "google.no/imgres", + "google.nr/imgres", + "google.nu/imgres", + "google.pl/imgres", + "google.pn/imgres", + "google.ps/imgres", + "google.pt/imgres", + "google.ro/imgres", + "google.rs/imgres", + "google.ru/imgres", + "google.rw/imgres", + "google.sc/imgres", + "google.se/imgres", + "google.sh/imgres", + "google.si/imgres", + "google.sk/imgres", + "google.sm/imgres", + "google.sn/imgres", + "google.so/imgres", + "google.st/imgres", + "google.td/imgres", + "google.tg/imgres", + "google.tk/imgres", + "google.tl/imgres", + "google.tm/imgres", + "google.to/imgres", + "google.tt/imgres", + "google.us/imgres", + "google.vg/imgres", + "google.vu/imgres", + "images.google.ws", + "images.google.ac", + "images.google.ad", + "images.google.ae", + "images.google.am", + "images.google.as", + "images.google.at", + "images.google.az", + "images.google.ba", + "images.google.be", + "images.google.bf", + "images.google.bg", + "images.google.bi", + "images.google.bj", + "images.google.bs", + "images.google.by", + "images.google.ca", + "images.google.cat", + "images.google.cc", + "images.google.cd", + "images.google.cf", + "images.google.cg", + "images.google.ch", + "images.google.ci", + "images.google.cl", + "images.google.cm", + "images.google.cn", + "images.google.co.bw", + "images.google.co.ck", + "images.google.co.cr", + "images.google.co.id", + "images.google.co.il", + "images.google.co.in", + "images.google.co.jp", + "images.google.co.ke", + "images.google.co.kr", + "images.google.co.ls", + "images.google.co.ma", + "images.google.co.mz", + "images.google.co.nz", + "images.google.co.th", + "images.google.co.tz", + "images.google.co.ug", + "images.google.co.uk", + "images.google.co.uz", + "images.google.co.ve", + "images.google.co.vi", + "images.google.co.za", + "images.google.co.zm", + "images.google.co.zw", + "images.google.com", + "images.google.com.af", + "images.google.com.ag", + "images.google.com.ai", + "images.google.com.ar", + "images.google.com.au", + "images.google.com.bd", + "images.google.com.bh", + "images.google.com.bn", + "images.google.com.bo", + "images.google.com.br", + "images.google.com.by", + "images.google.com.bz", + "images.google.com.co", + "images.google.com.cu", + "images.google.com.cy", + "images.google.com.do", + "images.google.com.ec", + "images.google.com.eg", + "images.google.com.et", + "images.google.com.fj", + "images.google.com.gh", + "images.google.com.gi", + "images.google.com.gt", + "images.google.com.hk", + "images.google.com.jm", + "images.google.com.kh", + "images.google.com.kh", + "images.google.com.kw", + "images.google.com.lb", + "images.google.com.lc", + "images.google.com.ly", + "images.google.com.mt", + "images.google.com.mx", + "images.google.com.my", + "images.google.com.na", + "images.google.com.nf", + "images.google.com.ng", + "images.google.com.ni", + "images.google.com.np", + "images.google.com.om", + "images.google.com.pa", + "images.google.com.pe", + "images.google.com.ph", + "images.google.com.pk", + "images.google.com.pr", + "images.google.com.py", + "images.google.com.qa", + "images.google.com.sa", + "images.google.com.sb", + "images.google.com.sg", + "images.google.com.sl", + "images.google.com.sv", + "images.google.com.tj", + "images.google.com.tn", + "images.google.com.tr", + "images.google.com.tw", + "images.google.com.ua", + "images.google.com.uy", + "images.google.com.vc", + "images.google.com.vn", + "images.google.cv", + "images.google.cz", + "images.google.de", + "images.google.dj", + "images.google.dk", + "images.google.dm", + "images.google.dz", + "images.google.ee", + "images.google.es", + "images.google.fi", + "images.google.fm", + "images.google.fr", + "images.google.ga", + "images.google.gd", + "images.google.ge", + "images.google.gf", + "images.google.gg", + "images.google.gl", + "images.google.gm", + "images.google.gp", + "images.google.gr", + "images.google.gy", + "images.google.hn", + "images.google.hr", + "images.google.ht", + "images.google.hu", + "images.google.ie", + "images.google.im", + "images.google.io", + "images.google.iq", + "images.google.is", + "images.google.it", + "images.google.it.ao", + "images.google.je", + "images.google.jo", + "images.google.kg", + "images.google.ki", + "images.google.kz", + "images.google.la", + "images.google.li", + "images.google.lk", + "images.google.lt", + "images.google.lu", + "images.google.lv", + "images.google.md", + "images.google.me", + "images.google.mg", + "images.google.mk", + "images.google.ml", + "images.google.mn", + "images.google.ms", + "images.google.mu", + "images.google.mv", + "images.google.mw", + "images.google.ne", + "images.google.nl", + "images.google.no", + "images.google.nr", + "images.google.nu", + "images.google.pl", + "images.google.pn", + "images.google.ps", + "images.google.pt", + "images.google.ro", + "images.google.rs", + "images.google.ru", + "images.google.rw", + "images.google.sc", + "images.google.se", + "images.google.sh", + "images.google.si", + "images.google.sk", + "images.google.sm", + "images.google.sn", + "images.google.so", + "images.google.st", + "images.google.td", + "images.google.tg", + "images.google.tk", + "images.google.tl", + "images.google.tm", + "images.google.to", + "images.google.tt", + "images.google.us", + "images.google.vg", + "images.google.vu", + "images.google.ws" + ], + "parameters": [ + "q" + ] + }, + "Google News": { + "domains": [ + "news.google.ac", + "news.google.ad", + "news.google.ae", + "news.google.am", + "news.google.as", + "news.google.at", + "news.google.az", + "news.google.ba", + "news.google.be", + "news.google.bf", + "news.google.bg", + "news.google.bi", + "news.google.bj", + "news.google.bs", + "news.google.by", + "news.google.ca", + "news.google.cat", + "news.google.cc", + "news.google.cd", + "news.google.cf", + "news.google.cg", + "news.google.ch", + "news.google.ci", + "news.google.cl", + "news.google.cm", + "news.google.cn", + "news.google.co.bw", + "news.google.co.ck", + "news.google.co.cr", + "news.google.co.id", + "news.google.co.il", + "news.google.co.in", + "news.google.co.jp", + "news.google.co.ke", + "news.google.co.kr", + "news.google.co.ls", + "news.google.co.ma", + "news.google.co.mz", + "news.google.co.nz", + "news.google.co.th", + "news.google.co.tz", + "news.google.co.ug", + "news.google.co.uk", + "news.google.co.uz", + "news.google.co.ve", + "news.google.co.vi", + "news.google.co.za", + "news.google.co.zm", + "news.google.co.zw", + "news.google.com", + "news.google.com.af", + "news.google.com.ag", + "news.google.com.ai", + "news.google.com.ar", + "news.google.com.au", + "news.google.com.bd", + "news.google.com.bh", + "news.google.com.bn", + "news.google.com.bo", + "news.google.com.br", + "news.google.com.by", + "news.google.com.bz", + "news.google.com.co", + "news.google.com.cu", + "news.google.com.cy", + "news.google.com.do", + "news.google.com.ec", + "news.google.com.eg", + "news.google.com.et", + "news.google.com.fj", + "news.google.com.gh", + "news.google.com.gi", + "news.google.com.gt", + "news.google.com.hk", + "news.google.com.jm", + "news.google.com.kh", + "news.google.com.kh", + "news.google.com.kw", + "news.google.com.lb", + "news.google.com.lc", + "news.google.com.ly", + "news.google.com.mt", + "news.google.com.mx", + "news.google.com.my", + "news.google.com.na", + "news.google.com.nf", + "news.google.com.ng", + "news.google.com.ni", + "news.google.com.np", + "news.google.com.om", + "news.google.com.pa", + "news.google.com.pe", + "news.google.com.ph", + "news.google.com.pk", + "news.google.com.pr", + "news.google.com.py", + "news.google.com.qa", + "news.google.com.sa", + "news.google.com.sb", + "news.google.com.sg", + "news.google.com.sl", + "news.google.com.sv", + "news.google.com.tj", + "news.google.com.tn", + "news.google.com.tr", + "news.google.com.tw", + "news.google.com.ua", + "news.google.com.uy", + "news.google.com.vc", + "news.google.com.vn", + "news.google.cv", + "news.google.cz", + "news.google.de", + "news.google.dj", + "news.google.dk", + "news.google.dm", + "news.google.dz", + "news.google.ee", + "news.google.es", + "news.google.fi", + "news.google.fm", + "news.google.fr", + "news.google.ga", + "news.google.gd", + "news.google.ge", + "news.google.gf", + "news.google.gg", + "news.google.gl", + "news.google.gm", + "news.google.gp", + "news.google.gr", + "news.google.gy", + "news.google.hn", + "news.google.hr", + "news.google.ht", + "news.google.hu", + "news.google.ie", + "news.google.im", + "news.google.io", + "news.google.iq", + "news.google.is", + "news.google.it", + "news.google.it.ao", + "news.google.je", + "news.google.jo", + "news.google.kg", + "news.google.ki", + "news.google.kz", + "news.google.la", + "news.google.li", + "news.google.lk", + "news.google.lt", + "news.google.lu", + "news.google.lv", + "news.google.md", + "news.google.me", + "news.google.mg", + "news.google.mk", + "news.google.ml", + "news.google.mn", + "news.google.ms", + "news.google.mu", + "news.google.mv", + "news.google.mw", + "news.google.ne", + "news.google.nl", + "news.google.no", + "news.google.nr", + "news.google.nu", + "news.google.pl", + "news.google.pn", + "news.google.ps", + "news.google.pt", + "news.google.ro", + "news.google.rs", + "news.google.ru", + "news.google.rw", + "news.google.sc", + "news.google.se", + "news.google.sh", + "news.google.si", + "news.google.sk", + "news.google.sm", + "news.google.sn", + "news.google.so", + "news.google.st", + "news.google.td", + "news.google.tg", + "news.google.tk", + "news.google.tl", + "news.google.tm", + "news.google.to", + "news.google.tt", + "news.google.us", + "news.google.vg", + "news.google.vu", + "news.google.ws" + ], + "parameters": [ + "q" + ] + }, + "Google Product Search": { + "domains": [ + "google.ac/products", + "google.ad/products", + "google.ae/products", + "google.am/products", + "google.as/products", + "google.at/products", + "google.az/products", + "google.ba/products", + "google.be/products", + "google.bf/products", + "google.bg/products", + "google.bi/products", + "google.bj/products", + "google.bs/products", + "google.by/products", + "google.ca/products", + "google.cat/products", + "google.cc/products", + "google.cd/products", + "google.cf/products", + "google.cg/products", + "google.ch/products", + "google.ci/products", + "google.cl/products", + "google.cm/products", + "google.cn/products", + "google.co.bw/products", + "google.co.ck/products", + "google.co.cr/products", + "google.co.id/products", + "google.co.il/products", + "google.co.in/products", + "google.co.jp/products", + "google.co.ke/products", + "google.co.kr/products", + "google.co.ls/products", + "google.co.ma/products", + "google.co.mz/products", + "google.co.nz/products", + "google.co.th/products", + "google.co.tz/products", + "google.co.ug/products", + "google.co.uk/products", + "google.co.uz/products", + "google.co.ve/products", + "google.co.vi/products", + "google.co.za/products", + "google.co.zm/products", + "google.co.zw/products", + "google.com/products", + "google.com.af/products", + "google.com.ag/products", + "google.com.ai/products", + "google.com.ar/products", + "google.com.au/products", + "google.com.bd/products", + "google.com.bh/products", + "google.com.bn/products", + "google.com.bo/products", + "google.com.br/products", + "google.com.by/products", + "google.com.bz/products", + "google.com.co/products", + "google.com.cu/products", + "google.com.cy/products", + "google.com.do/products", + "google.com.ec/products", + "google.com.eg/products", + "google.com.et/products", + "google.com.fj/products", + "google.com.gh/products", + "google.com.gi/products", + "google.com.gt/products", + "google.com.hk/products", + "google.com.jm/products", + "google.com.kh/products", + "google.com.kh/products", + "google.com.kw/products", + "google.com.lb/products", + "google.com.lc/products", + "google.com.ly/products", + "google.com.mt/products", + "google.com.mx/products", + "google.com.my/products", + "google.com.na/products", + "google.com.nf/products", + "google.com.ng/products", + "google.com.ni/products", + "google.com.np/products", + "google.com.om/products", + "google.com.pa/products", + "google.com.pe/products", + "google.com.ph/products", + "google.com.pk/products", + "google.com.pr/products", + "google.com.py/products", + "google.com.qa/products", + "google.com.sa/products", + "google.com.sb/products", + "google.com.sg/products", + "google.com.sl/products", + "google.com.sv/products", + "google.com.tj/products", + "google.com.tn/products", + "google.com.tr/products", + "google.com.tw/products", + "google.com.ua/products", + "google.com.uy/products", + "google.com.vc/products", + "google.com.vn/products", + "google.cv/products", + "google.cz/products", + "google.de/products", + "google.dj/products", + "google.dk/products", + "google.dm/products", + "google.dz/products", + "google.ee/products", + "google.es/products", + "google.fi/products", + "google.fm/products", + "google.fr/products", + "google.ga/products", + "google.gd/products", + "google.ge/products", + "google.gf/products", + "google.gg/products", + "google.gl/products", + "google.gm/products", + "google.gp/products", + "google.gr/products", + "google.gy/products", + "google.hn/products", + "google.hr/products", + "google.ht/products", + "google.hu/products", + "google.ie/products", + "google.im/products", + "google.io/products", + "google.iq/products", + "google.is/products", + "google.it/products", + "google.it.ao/products", + "google.je/products", + "google.jo/products", + "google.kg/products", + "google.ki/products", + "google.kz/products", + "google.la/products", + "google.li/products", + "google.lk/products", + "google.lt/products", + "google.lu/products", + "google.lv/products", + "google.md/products", + "google.me/products", + "google.mg/products", + "google.mk/products", + "google.ml/products", + "google.mn/products", + "google.ms/products", + "google.mu/products", + "google.mv/products", + "google.mw/products", + "google.ne/products", + "google.nl/products", + "google.no/products", + "google.nr/products", + "google.nu/products", + "google.pl/products", + "google.pn/products", + "google.ps/products", + "google.pt/products", + "google.ro/products", + "google.rs/products", + "google.ru/products", + "google.rw/products", + "google.sc/products", + "google.se/products", + "google.sh/products", + "google.si/products", + "google.sk/products", + "google.sm/products", + "google.sn/products", + "google.so/products", + "google.st/products", + "google.td/products", + "google.tg/products", + "google.tk/products", + "google.tl/products", + "google.tm/products", + "google.to/products", + "google.tt/products", + "google.us/products", + "google.vg/products", + "google.vu/products", + "google.ws/products", + "www.google.ac/products", + "www.google.ad/products", + "www.google.ae/products", + "www.google.am/products", + "www.google.as/products", + "www.google.at/products", + "www.google.az/products", + "www.google.ba/products", + "www.google.be/products", + "www.google.bf/products", + "www.google.bg/products", + "www.google.bi/products", + "www.google.bj/products", + "www.google.bs/products", + "www.google.by/products", + "www.google.ca/products", + "www.google.cat/products", + "www.google.cc/products", + "www.google.cd/products", + "www.google.cf/products", + "www.google.cg/products", + "www.google.ch/products", + "www.google.ci/products", + "www.google.cl/products", + "www.google.cm/products", + "www.google.cn/products", + "www.google.co.bw/products", + "www.google.co.ck/products", + "www.google.co.cr/products", + "www.google.co.id/products", + "www.google.co.il/products", + "www.google.co.in/products", + "www.google.co.jp/products", + "www.google.co.ke/products", + "www.google.co.kr/products", + "www.google.co.ls/products", + "www.google.co.ma/products", + "www.google.co.mz/products", + "www.google.co.nz/products", + "www.google.co.th/products", + "www.google.co.tz/products", + "www.google.co.ug/products", + "www.google.co.uk/products", + "www.google.co.uz/products", + "www.google.co.ve/products", + "www.google.co.vi/products", + "www.google.co.za/products", + "www.google.co.zm/products", + "www.google.co.zw/products", + "www.google.com/products", + "www.google.com.af/products", + "www.google.com.ag/products", + "www.google.com.ai/products", + "www.google.com.ar/products", + "www.google.com.au/products", + "www.google.com.bd/products", + "www.google.com.bh/products", + "www.google.com.bn/products", + "www.google.com.bo/products", + "www.google.com.br/products", + "www.google.com.by/products", + "www.google.com.bz/products", + "www.google.com.co/products", + "www.google.com.cu/products", + "www.google.com.cy/products", + "www.google.com.do/products", + "www.google.com.ec/products", + "www.google.com.eg/products", + "www.google.com.et/products", + "www.google.com.fj/products", + "www.google.com.gh/products", + "www.google.com.gi/products", + "www.google.com.gt/products", + "www.google.com.hk/products", + "www.google.com.jm/products", + "www.google.com.kh/products", + "www.google.com.kh/products", + "www.google.com.kw/products", + "www.google.com.lb/products", + "www.google.com.lc/products", + "www.google.com.ly/products", + "www.google.com.mt/products", + "www.google.com.mx/products", + "www.google.com.my/products", + "www.google.com.na/products", + "www.google.com.nf/products", + "www.google.com.ng/products", + "www.google.com.ni/products", + "www.google.com.np/products", + "www.google.com.om/products", + "www.google.com.pa/products", + "www.google.com.pe/products", + "www.google.com.ph/products", + "www.google.com.pk/products", + "www.google.com.pr/products", + "www.google.com.py/products", + "www.google.com.qa/products", + "www.google.com.sa/products", + "www.google.com.sb/products", + "www.google.com.sg/products", + "www.google.com.sl/products", + "www.google.com.sv/products", + "www.google.com.tj/products", + "www.google.com.tn/products", + "www.google.com.tr/products", + "www.google.com.tw/products", + "www.google.com.ua/products", + "www.google.com.uy/products", + "www.google.com.vc/products", + "www.google.com.vn/products", + "www.google.cv/products", + "www.google.cz/products", + "www.google.de/products", + "www.google.dj/products", + "www.google.dk/products", + "www.google.dm/products", + "www.google.dz/products", + "www.google.ee/products", + "www.google.es/products", + "www.google.fi/products", + "www.google.fm/products", + "www.google.fr/products", + "www.google.ga/products", + "www.google.gd/products", + "www.google.ge/products", + "www.google.gf/products", + "www.google.gg/products", + "www.google.gl/products", + "www.google.gm/products", + "www.google.gp/products", + "www.google.gr/products", + "www.google.gy/products", + "www.google.hn/products", + "www.google.hr/products", + "www.google.ht/products", + "www.google.hu/products", + "www.google.ie/products", + "www.google.im/products", + "www.google.io/products", + "www.google.iq/products", + "www.google.is/products", + "www.google.it/products", + "www.google.it.ao/products", + "www.google.je/products", + "www.google.jo/products", + "www.google.kg/products", + "www.google.ki/products", + "www.google.kz/products", + "www.google.la/products", + "www.google.li/products", + "www.google.lk/products", + "www.google.lt/products", + "www.google.lu/products", + "www.google.lv/products", + "www.google.md/products", + "www.google.me/products", + "www.google.mg/products", + "www.google.mk/products", + "www.google.ml/products", + "www.google.mn/products", + "www.google.ms/products", + "www.google.mu/products", + "www.google.mv/products", + "www.google.mw/products", + "www.google.ne/products", + "www.google.nl/products", + "www.google.no/products", + "www.google.nr/products", + "www.google.nu/products", + "www.google.pl/products", + "www.google.pn/products", + "www.google.ps/products", + "www.google.pt/products", + "www.google.ro/products", + "www.google.rs/products", + "www.google.ru/products", + "www.google.rw/products", + "www.google.sc/products", + "www.google.se/products", + "www.google.sh/products", + "www.google.si/products", + "www.google.sk/products", + "www.google.sm/products", + "www.google.sn/products", + "www.google.so/products", + "www.google.st/products", + "www.google.td/products", + "www.google.tg/products", + "www.google.tk/products", + "www.google.tl/products", + "www.google.tm/products", + "www.google.to/products", + "www.google.tt/products", + "www.google.us/products", + "www.google.vg/products", + "www.google.vu/products", + "www.google.ws/products" + ], + "parameters": [ + "q" + ] + }, + "Google Video": { + "domains": [ + "video.google.com" + ], + "parameters": [ + "q" + ] + }, + "Goyellow.de": { + "domains": [ + "www.goyellow.de" + ], + "parameters": [ + "MDN" + ] + }, + "Gule Sider": { + "domains": [ + "www.gulesider.no" + ], + "parameters": [ + "q" + ] + }, + "HighBeam": { + "domains": [ + "www.highbeam.com" + ], + "parameters": [ + "q" + ] + }, + "Hit-Parade": { + "domains": [ + "req.-hit-parade.com", + "class.hit-parade.com", + "www.hit-parade.com" + ], + "parameters": [ + "p7" + ] + }, + "Holmes": { + "domains": [ + "holmes.ge" + ], + "parameters": [ + "q" + ] + }, + "Hooseek.com": { + "domains": [ + "www.hooseek.com" + ], + "parameters": [ + "recherche" + ] + }, + "Hotbot": { + "domains": [ + "www.hotbot.com" + ], + "parameters": [ + "query" + ] + }, + "Haosou": { + "domains": [ + "www.haosou.com" + ], + "parameters": [ + "q" + ] + }, + "I-play": { + "domains": [ + "start.iplay.com" + ], + "parameters": [ + "q" + ] + }, + "I.ua": { + "domains": [ + "search.i.ua" + ], + "parameters": [ + "q" + ] + }, + "ICQ": { + "domains": [ + "www.icq.com", + "search.icq.com" + ], + "parameters": [ + "q" + ] + }, + "IXquick": { + "domains": [ + "ixquick.com", + "www.eu.ixquick.com", + "ixquick.de", + "www.ixquick.de", + "us.ixquick.com", + "s1.us.ixquick.com", + "s2.us.ixquick.com", + "s3.us.ixquick.com", + "s4.us.ixquick.com", + "s5.us.ixquick.com", + "eu.ixquick.com", + "s8-eu.ixquick.com", + "s1-eu.ixquick.de" + ], + "parameters": [ + "query" + ] + }, + "Icerockeet": { + "domains": [ + "blogs.icerocket.com" + ], + "parameters": [ + "q" + ] + }, + "Ilse": { + "domains": [ + "www.ilse.nl" + ], + "parameters": [ + "search_for" + ] + }, + "InfoSpace": { + "domains": [ + "infospace.com", + "dogpile.com", + "www.dogpile.com", + "metacrawler.com", + "webfetch.com", + "webcrawler.com", + "search.kiwee.com", + "isearch.babylon.com", + "start.facemoods.com", + "search.magnetic.com", + "search.searchcompletion.com", + "clusty.com" + ], + "parameters": [ + "q", + "s" + ] + }, + "Inbox": { + "domains": [ + "inbox.com" + ], + "parameters": [ + "q" + ] + }, + "Inbox.com": { + "domains": [ + "inbox.com/search/" + ], + "parameters": [ + "q" + ] + }, + "Info": { + "domains": [ + "info.com" + ], + "parameters": [ + "qkw" + ] + }, + "Interia": { + "domains": [ + "www.google.interia.pl" + ], + "parameters": [ + "q" + ] + }, + "Jungle Key": { + "domains": [ + "junglekey.com", + "junglekey.fr" + ], + "parameters": [ + "query" + ] + }, + "Jungle Spider": { + "domains": [ + "www.jungle-spider.de" + ], + "parameters": [ + "q" + ] + }, + "Jyxo": { + "domains": [ + "jyxo.1188.cz" + ], + "parameters": [ + "q" + ] + }, + "Kataweb": { + "domains": [ + "www.kataweb.it" + ], + "parameters": [ + "q" + ] + }, + "Kvasir": { + "domains": [ + "www.kvasir.no" + ], + "parameters": [ + "q" + ] + }, + "La Toile Du Quebec Via Google": { + "domains": [ + "www.toile.com", + "web.toile.com" + ], + "parameters": [ + "q" + ] + }, + "Latne": { + "domains": [ + "www.latne.lv" + ], + "parameters": [ + "q" + ] + }, + "Lo.st": { + "domains": [ + "lo.st" + ], + "parameters": [ + "x_query" + ] + }, + "Looksmart": { + "domains": [ + "www.looksmart.com" + ], + "parameters": [ + "key" + ] + }, + "Lycos": { + "domains": [ + "search.lycos.com", + "www.lycos.com", + "lycos.com" + ], + "parameters": [ + "query" + ] + }, + "Mail.ru": { + "domains": [ + "go.mail.ru" + ], + "parameters": [ + "q" + ] + }, + "Mamma": { + "domains": [ + "www.mamma.com", + "mamma75.mamma.com" + ], + "parameters": [ + "query" + ] + }, + "Marktplaats": { + "domains": [ + "www.marktplaats.nl" + ], + "parameters": [ + "query" + ] + }, + "Maxwebsearch": { + "domains": [ + "maxwebsearch.com" + ], + "parameters": [ + "query" + ] + }, + "Meinestadt": { + "domains": [ + "www.meinestadt.de" + ], + "parameters": [ + "words" + ] + }, + "Meta": { + "domains": [ + "meta.ua" + ], + "parameters": [ + "q" + ] + }, + "MetaCrawler.de": { + "domains": [ + "s1.metacrawler.de", + "s2.metacrawler.de", + "s3.metacrawler.de" + ], + "parameters": [ + "qry" + ] + }, + "Metager": { + "domains": [ + "meta.rrzn.uni-hannover.de", + "www.metager.de" + ], + "parameters": [ + "eingabe" + ] + }, + "Metager2": { + "domains": [ + "metager2.de" + ], + "parameters": [ + "q" + ] + }, + "Mister Wong": { + "domains": [ + "www.mister-wong.com", + "www.mister-wong.de" + ], + "parameters": [ + "Keywords" + ] + }, + "Monstercrawler": { + "domains": [ + "www.monstercrawler.com" + ], + "parameters": [ + "qry" + ] + }, + "Mozbot": { + "domains": [ + "www.mozbot.fr", + "www.mozbot.co.uk", + "www.mozbot.com" + ], + "parameters": [ + "q" + ] + }, + "MySearch": { + "domains": [ + "www.mysearch.com", + "ms114.mysearch.com", + "ms146.mysearch.com", + "kf.mysearch.myway.com", + "ki.mysearch.myway.com", + "search.myway.com", + "search.mywebsearch.com" + ], + "parameters": [ + "searchfor", + "searchFor" + ] + }, + "Najdi": { + "domains": [ + "www.najdi.si" + ], + "parameters": [ + "q" + ] + }, + "Nate": { + "domains": [ + "search.nate.com" + ], + "parameters": [ + "q" + ] + }, + "Naver": { + "domains": [ + "search.naver.com" + ], + "parameters": [ + "query" + ] + }, + "Naver Images": { + "domains": [ + "image.search.naver.com", + "imagesearch.naver.com" + ], + "parameters": [ + "query" + ] + }, + "Needtofind": { + "domains": [ + "ko.search.need2find.com" + ], + "parameters": [ + "searchfor" + ] + }, + "Neti": { + "domains": [ + "www.neti.ee" + ], + "parameters": [ + "query" + ] + }, + "Nifty": { + "domains": [ + "search.nifty.com" + ], + "parameters": [ + "q" + ] + }, + "Nigma": { + "domains": [ + "nigma.ru" + ], + "parameters": [ + "s" + ] + }, + "Onet": { + "domains": [ + "szukaj.onet.pl" + ], + "parameters": [ + "qt" + ] + }, + "Online.no": { + "domains": [ + "online.no" + ], + "parameters": [ + "q" + ] + }, + "Opplysningen 1881": { + "domains": [ + "www.1881.no" + ], + "parameters": [ + "Query" + ] + }, + "Orange": { + "domains": [ + "busca.orange.es", + "search.orange.co.uk" + ], + "parameters": [ + "q" + ] + }, + "Paperball": { + "domains": [ + "www.paperball.de" + ], + "parameters": [ + "q" + ] + }, + "PeoplePC": { + "domains": [ + "search.peoplepc.com" + ], + "parameters": [ + "q" + ] + }, + "Picsearch": { + "domains": [ + "www.picsearch.com" + ], + "parameters": [ + "q" + ] + }, + "Plazoo": { + "domains": [ + "www.plazoo.com" + ], + "parameters": [ + "q" + ] + }, + "Poisk.ru": { + "domains": [ + "www.plazoo.com" + ], + "parameters": [ + "q" + ] + }, + "PriceRunner": { + "domains": [ + "www.pricerunner.co.uk" + ], + "parameters": [ + "q" + ] + }, + "Qualigo": { + "domains": [ + "www.qualigo.at", + "www.qualigo.ch", + "www.qualigo.de", + "www.qualigo.nl" + ], + "parameters": [ + "q" + ] + }, + "RPMFind": { + "domains": [ + "rpmfind.net", + "fr2.rpmfind.net" + ], + "parameters": [ + "query" + ] + }, + "Rakuten": { + "domains": [ + "websearch.rakuten.co.jp" + ], + "parameters": [ + "qt" + ] + }, + "Rambler": { + "domains": [ + "nova.rambler.ru" + ], + "parameters": [ + "query", + "words" + ] + }, + "Road Runner Search": { + "domains": [ + "search.rr.com" + ], + "parameters": [ + "q" + ] + }, + "Sapo": { + "domains": [ + "pesquisa.sapo.pt" + ], + "parameters": [ + "q" + ] + }, + "Search This": { + "domains": [ + "www.searchthis.com" + ], + "parameters": [ + "q" + ] + }, + "Search.ch": { + "domains": [ + "www.search.ch" + ], + "parameters": [ + "q" + ] + }, + "Search.com": { + "domains": [ + "www.search.com" + ], + "parameters": [ + "q" + ] + }, + "SearchCanvas": { + "domains": [ + "www.searchcanvas.com" + ], + "parameters": [ + "q" + ] + }, + "Searchalot": { + "domains": [ + "searchalot.com" + ], + "parameters": [ + "q" + ] + }, + "SearchLock": { + "domains": [ + "searchlock.com" + ], + "parameters": [ + "q" + ] + }, + "Searchy": { + "domains": [ + "www.searchy.co.uk" + ], + "parameters": [ + "q" + ] + }, + "Seznam": { + "domains": [ + "search.seznam.cz" + ], + "parameters": [ + "q" + ] + }, + "Sharelook": { + "domains": [ + "www.sharelook.fr" + ], + "parameters": [ + "keyword" + ] + }, + "Skynet": { + "domains": [ + "www.skynet.be" + ], + "parameters": [ + "q" + ] + }, + "The Smart Search": { + "domains": [ + "thesmartsearch.net", + "www.thesmartsearch.net" + ], + "parameters": [ + "q" + ] + }, + "Softonic": { + "domains": [ + "search.softonic.com" + ], + "parameters": [ + "q" + ] + }, + "Sogou": { + "domains": [ + "www.sogou.com", + "www.soso.com" + ], + "parameters": [ + "query", + "w" + ] + }, + "Startpagina": { + "domains": [ + "startgoogle.startpagina.nl" + ], + "parameters": [ + "q" + ] + }, + "Startsiden": { + "domains": [ + "www.startsiden.no" + ], + "parameters": [ + "q" + ] + }, + "Suchmaschine.com": { + "domains": [ + "www.suchmaschine.com" + ], + "parameters": [ + "suchstr" + ] + }, + "Suchnase": { + "domains": [ + "www.suchnase.de" + ], + "parameters": [ + "q" + ] + }, + "Superpages": { + "domains": [ + "superpages.com" + ], + "parameters": [ + "C" + ] + }, + "T-Online": { + "domains": [ + "suche.t-online.de", + "brisbane.t-online.de", + "navigationshilfe.t-online.de" + ], + "parameters": [ + "q" + ] + }, + "TalkTalk": { + "domains": [ + "www.talktalk.co.uk" + ], + "parameters": [ + "query" + ] + }, + "Technorati": { + "domains": [ + "technorati.com" + ], + "parameters": [ + "q" + ] + }, + "Telstra": { + "domains": [ + "search.media.telstra.com.au" + ], + "parameters": [ + "find" + ] + }, + "Teoma": { + "domains": [ + "www.teoma.com" + ], + "parameters": [ + "q" + ] + }, + "Terra": { + "domains": [ + "buscador.terra.es", + "buscador.terra.cl", + "buscador.terra.com.br" + ], + "parameters": [ + "query" + ] + }, + "Tiscali": { + "domains": [ + "search.tiscali.it", + "search-dyn.tiscali.it", + "hledani.tiscali.cz" + ], + "parameters": [ + "q", + "key" + ] + }, + "Tixuma": { + "domains": [ + "www.tixuma.de" + ], + "parameters": [ + "sc" + ] + }, + "Toolbarhome": { + "domains": [ + "www.toolbarhome.com", + "vshare.toolbarhome.com" + ], + "parameters": [ + "q" + ] + }, + "Trouvez.com": { + "domains": [ + "www.trouvez.com" + ], + "parameters": [ + "query" + ] + }, + "TrovaRapido": { + "domains": [ + "www.trovarapido.com" + ], + "parameters": [ + "q" + ] + }, + "Trusted-Search": { + "domains": [ + "www.trusted--search.com" + ], + "parameters": [ + "w" + ] + }, + "Tut.by": { + "domains": [ + "search.tut.by" + ], + "parameters": [ + "query" + ] + }, + "Twingly": { + "domains": [ + "www.twingly.com" + ], + "parameters": [ + "q" + ] + }, + "UKR.net": { + "domains": [ + "search.ukr.net" + ], + "parameters": [ + "q" + ] + }, + "URL.ORGanizier": { + "domains": [ + "www.url.org" + ], + "parameters": [ + "q" + ] + }, + "Vinden": { + "domains": [ + "www.vinden.nl" + ], + "parameters": [ + "q" + ] + }, + "Vindex": { + "domains": [ + "www.vindex.nl", + "search.vindex.nl" + ], + "parameters": [ + "search_for" + ] + }, + "Virgilio": { + "domains": [ + "ricerca.virgilio.it", + "ricercaimmagini.virgilio.it", + "ricercavideo.virgilio.it", + "ricercanews.virgilio.it", + "mobile.virgilio.it" + ], + "parameters": [ + "qs" + ] + }, + "Vi-view": { + "domains": [ + "viview.inspsearch.com" + ], + "parameters": [ + "q" + ] + }, + "Voila": { + "domains": [ + "search.ke.voila.fr", + "www.lemoteur.fr" + ], + "parameters": [ + "rdata", + "kw" + ] + }, + "Volny": { + "domains": [ + "web.volny.cz" + ], + "parameters": [ + "search" + ] + }, + "WWW": { + "domains": [ + "search.www.ee" + ], + "parameters": [ + "query" + ] + }, + "Walhello": { + "domains": [ + "www.walhello.info", + "www.walhello.com", + "www.walhello.de", + "www.walhello.nl" + ], + "parameters": [ + "key" + ] + }, + "Web.de": { + "domains": [ + "suche.web.de" + ], + "parameters": [ + "su" + ] + }, + "Web.nl": { + "domains": [ + "www.web.nl" + ], + "parameters": [ + "zoekwoord" + ] + }, + "WebSearch": { + "domains": [ + "www.websearch.com" + ], + "parameters": [ + "qkw", + "q" + ] + }, + "Weborama": { + "domains": [ + "www.weborama.com" + ], + "parameters": [ + "QUERY" + ] + }, + "Winamp": { + "domains": [ + "search.winamp.com" + ], + "parameters": [ + "q" + ] + }, + "Wirtualna Polska": { + "domains": [ + "szukaj.wp.pl" + ], + "parameters": [ + "szukaj" + ] + }, + "Witch": { + "domains": [ + "www.witch.de" + ], + "parameters": [ + "search" + ] + }, + "X-recherche": { + "domains": [ + "www.x-recherche.com" + ], + "parameters": [ + "MOTS" + ] + }, + "Yahoo!": { + "domains": [ + "search.yahoo.com", + "yahoo.com", + "ar.search.yahoo.com", + "ar.yahoo.com", + "au.search.yahoo.com", + "au.yahoo.com", + "br.search.yahoo.com", + "br.yahoo.com", + "cade.searchde.yahoo.com", + "cade.yahoo.com", + "chinese.searchinese.yahoo.com", + "chinese.yahoo.com", + "cn.search.yahoo.com", + "cn.yahoo.com", + "de.search.yahoo.com", + "de.yahoo.com", + "dk.search.yahoo.com", + "dk.yahoo.com", + "es.search.yahoo.com", + "es.yahoo.com", + "espanol.searchpanol.yahoo.com", + "espanol.searchpanol.yahoo.com", + "espanol.yahoo.com", + "espanol.yahoo.com", + "fr.search.yahoo.com", + "fr.yahoo.com", + "ie.search.yahoo.com", + "ie.yahoo.com", + "it.search.yahoo.com", + "it.yahoo.com", + "kr.search.yahoo.com", + "kr.yahoo.com", + "mx.search.yahoo.com", + "mx.yahoo.com", + "no.search.yahoo.com", + "no.yahoo.com", + "nz.search.yahoo.com", + "nz.yahoo.com", + "one.cn.yahoo.com", + "one.searchn.yahoo.com", + "qc.search.yahoo.com", + "qc.search.yahoo.com", + "qc.search.yahoo.com", + "qc.yahoo.com", + "qc.yahoo.com", + "se.search.yahoo.com", + "se.search.yahoo.com", + "se.yahoo.com", + "search.searcharch.yahoo.com", + "search.yahoo.com", + "uk.search.yahoo.com", + "uk.yahoo.com", + "www.yahoo.co.jp", + "search.yahoo.co.jp", + "m.chiebukuro.yahoo.co.jp", + "detail.chiebukuro.yahoo.co.jp", + "www.cercato.it", + "search.offerbox.com", + "ys.mirostart.com" + ], + "parameters": [ + "p", + "q" + ] + }, + "Yahoo! Images": { + "domains": [ + "image.yahoo.cn", + "images.search.yahoo.com", + "image.search.yahoo.co.jp" + ], + "parameters": [ + "p", + "q" + ] + }, + "Yam": { + "domains": [ + "search.yam.com" + ], + "parameters": [ + "k" + ] + }, + "Yandex": { + "domains": [ + "yandex.ru", + "yandex.ua", + "yandex.com", + "yandex.by", + "www.yandex.ru", + "www.yandex.ua", + "www.yandex.com", + "www.yandex.by" + ], + "parameters": [ + "text" + ] + }, + "Yandex Images": { + "domains": [ + "images.yandex.ru", + "images.yandex.ua", + "images.yandex.com" + ], + "parameters": [ + "text" + ] + }, + "Yasni": { + "domains": [ + "www.yasni.de", + "www.yasni.com", + "www.yasni.co.uk", + "www.yasni.ch", + "www.yasni.at" + ], + "parameters": [ + "query" + ] + }, + "Yatedo": { + "domains": [ + "www.yatedo.com", + "www.yatedo.fr" + ], + "parameters": [ + "q" + ] + }, + "Yellowpages": { + "domains": [ + "www.yellowpages.com", + "www.yellowpages.com.au", + "www.yellowpages.ca" + ], + "parameters": [ + "q", + "search_terms" + ] + }, + "Yippy": { + "domains": [ + "search.yippy.com" + ], + "parameters": [ + "q", + "query" + ] + }, + "YouGoo": { + "domains": [ + "www.yougoo.fr" + ], + "parameters": [ + "q" + ] + }, + "Zapmeta": { + "domains": [ + "www.zapmeta.com", + "www.zapmeta.nl", + "www.zapmeta.de", + "uk.zapmeta.com" + ], + "parameters": [ + "q", + "query" + ] + }, + "Zhongsou": { + "domains": [ + "p.zhongsou.com" + ], + "parameters": [ + "w" + ] + }, + "Zoek": { + "domains": [ + "www3.zoek.nl" + ], + "parameters": [ + "q" + ] + }, + "Zoeken": { + "domains": [ + "www.zoeken.nl" + ], + "parameters": [ + "q" + ] + }, + "Zoohoo": { + "domains": [ + "zoohoo.cz" + ], + "parameters": [ + "q" + ] + }, + "all.by": { + "domains": [ + "all.by" + ], + "parameters": [ + "query" + ] + }, + "arama": { + "domains": [ + "arama.com" + ], + "parameters": [ + "q" + ] + }, + "blekko": { + "domains": [ + "blekko.com" + ], + "parameters": [ + "q" + ] + }, + "canoe.ca": { + "domains": [ + "web.canoe.ca" + ], + "parameters": [ + "q" + ] + }, + "dmoz": { + "domains": [ + "dmoz.org", + "editors.dmoz.org" + ], + "parameters": [ + "q" + ] + }, + "earthlink": { + "domains": [ + "search.earthlink.net" + ], + "parameters": [ + "q" + ] + }, + "eo": { + "domains": [ + "eo.st" + ], + "parameters": [ + "x_query" + ] + }, + "goo": { + "domains": [ + "search.goo.ne.jp", + "ocnsearch.goo.ne.jp" + ], + "parameters": [ + "MT" + ] + }, + "maailm": { + "domains": [ + "www.maailm.com" + ], + "parameters": [ + "tekst" + ] + }, + "qip": { + "domains": [ + "search.qip.ru" + ], + "parameters": [ + "query" + ] + }, + "SoSoDesk": { + "domains": [ + "www.soso.com", + "sosodesktop.com", + "search.sosodesktop.com" + ], + "parameters": [ + "q" + ] + }, + "Snapdo": { + "domains": [ + "search.snapdo.com" + ], + "parameters": [ + "q" + ] + }, + "suche.info": { + "domains": [ + "suche.info" + ], + "parameters": [ + "q" + ] + }, + "uol.com.br": { + "domains": [ + "busca.uol.com.br" + ], + "parameters": [ + "q" + ] + } + }, + "social": { + "Badoo": { + "domains": [ + "badoo.com" + ] + }, + "Bebo": { + "domains": [ + "bebo.com" + ] + }, + "BlackPlanet": { + "domains": [ + "blackplanet.com" + ] + }, + "Bloglovin'": { + "domains": [ + "bloglovin.com" + ] + }, + "Buzznet": { + "domains": [ + "wayn.com", + "buzznet.com" + ] + }, + "Classmates": { + "domains": [ + "classmates.com" + ] + }, + "Cyworld": { + "domains": [ + "global.cyworld.com" + ] + }, + "DeviantArt":{ + "domains": [ + "deviantart.com" + ] + }, + "Delicious":{ + "domains": [ + "delicious.com" + ] + }, + "Discus": { + "domains": [ + "redirect.disqus.com", + "disq.us", + "disqus.com" + ] + }, + "Donanimhaber": { + "domains": [ + "donanimhaber.com" + ] + }, + "Douban": { + "domains": [ + "douban.com" + ] + }, + "Facebook": { + "domains": [ + "facebook.com", + "fb.me", + "m.facebook.com", + "l.facebook.com", + "lm.facebook.com" + ] + }, + "Flickr": { + "domains": [ + "flickr.com" + ] + }, + "Flixster": { + "domains": [ + "flixster.com" + ] + }, + "Flipboard": { + "domains": [ + "flipboard.com" + ] + }, + "Fotolog": { + "domains": [ + "fotolog.com" + ] + }, + "Foursquare": { + "domains": [ + "foursquare.com" + ] + }, + "Friends Reunited": { + "domains": [ + "friendsreunited.com" + ] + }, + "Friendster": { + "domains": [ + "friendster.com" + ] + }, + "Gaia Online": { + "domains": [ + "gaiaonline.com" + ] + }, + "Geni": { + "domains": [ + "geni.com" + ] + }, + "GitHub": { + "domains": [ + "github.com" + ] + }, + "Google+": { + "domains": [ + "url.google.com", + "plus.google.com", + "plus.url.google.com" + ] + }, + "Habbo": { + "domains": [ + "habbo.com" + ] + }, + "Hacker News": { + "domains": [ + "news.ycombinator.com" + ] + }, + "Hocam.com": { + "domains": [ + "hocam.com" + ] + }, + "Hyves": { + "domains": [ + "hyves.nl" + ] + }, + "Iconosquare": { + "domains": [ + "iconosquare.com" + ] + }, + "Identi.ca": { + "domains": [ + "identi.ca" + ] + }, + "Imgur": { + "domains": [ + "imgur.com" + ] + }, + "Inci Sozluk": { + "domains": [ + "inci.sozlukspot.com", + "incisozluk.com", + "incisozluk.cc" + ] + }, + "Instagram": { + "domains": [ + "instagram.com" + ] + }, + "Instela": { + "domains": [ + "instela.com" + ] + }, + "Last.fm": { + "domains": [ + "lastfm.ru" + ] + }, + "LinkedIn": { + "domains": [ + "linkedin.com", + "lnkd.in" + ] + }, + "LiveJournal": { + "domains": [ + "livejournal.ru" + ] + }, + "Mail.ru": { + "domains": [ + "my.mail.ru" + ] + }, + "Medium": { + "domains": [ + "medium.com" + ] + }, + "Meetup": { + "domains": [ + "meetup.com" + ] + }, + "Messenger": { + "domains": [ + "messenger.com" + ] + }, + "Mixi": { + "domains": [ + "mixi.jp" + ] + }, + "MoiKrug.ru": { + "domains": [ + "moikrug.ru" + ] + }, + "Multiply": { + "domains": [ + "multiply.com" + ] + }, + "MyHeritage": { + "domains": [ + "myheritage.com" + ] + }, + "MyLife": { + "domains": [ + "mylife.ru" + ] + }, + "Myspace": { + "domains": [ + "myspace.com" + ] + }, + "Nasza-klasa.pl": { + "domains": [ + "nk.pl" + ] + }, + "Netlog": { + "domains": [ + "netlog.com" + ] + }, + "Odnoklassniki": { + "domains": [ + "odnoklassniki.ru", + "ok.ru" + ] + }, + "Orkut": { + "domains": [ + "orkut.com" + ] + }, + "Paper.li": { + "domains": [ + "paper.li" + ] + }, + "Pinterest": { + "domains": [ + "pinterest.com" + ] + }, + "Plaxo": { + "domains": [ + "plaxo.com" + ] + }, + "Pocket": { + "domains": [ + "getpocket.com" + ] + }, + "Polyvore": { + "domains": [ + "polyvore.com" + ] + }, + "Quora": { + "domains": [ + "quora.com" + ] + }, + "Qzone": { + "domains": [ + "qzone.qq.com" + ] + }, + "Reddit": { + "domains": [ + "reddit.com" + ] + }, + "Renren": { + "domains": [ + "renren.com" + ] + }, + "Skyrock": { + "domains": [ + "skyrock.com" + ] + }, + "Snapchat": { + "domains": [ + "snapchat.com" + ] + }, + "Sonico.com": { + "domains": [ + "sonico.com" + ] + }, + "SourceForge": { + "domains": [ + "sourceforge.net" + ] + }, + "StackOverflow": { + "domains": [ + "stackoverflow.com" + ] + }, + "StudiVZ": { + "domains": [ + "studivz.net" + ] + }, + "StumbleUpon": { + "domains": [ + "stumbleupon.com" + ] + }, + "Tagged": { + "domains": [ + "login.tagged.com" + ] + }, + "Taringa!": { + "domains": [ + "taringa.net" + ] + }, + "TikTok": { + "domains": [ + "tiktok.com", + "tiktokcdn.com" + ] + }, + "Tuenti": { + "domains": [ + "tuenti.com" + ] + }, + "Tumblr": { + "domains": [ + "tumblr.com", + "umblr.com", + "t.umblr.com" + ] + }, + "Twitter": { + "domains": [ + "twitter.com", + "t.co" + ] + }, + "Twitch":{ + "domains": [ + "twitch.tv", + "twitch.com" + ] + }, + "Viadeo": { + "domains": [ + "viadeo.com" + ] + }, + "Vimeo": { + "domains": [ + "vimeo.com" + ] + }, + "Vkontakte": { + "domains": [ + "vk.com", + "vkontakte.ru" + ] + }, + "Wanelo": { + "domains": [ + "wanelo.com" + ] + }, + "WAYN": { + "domains": [ + "wayn.com" + ] + }, + "WeeWorld": { + "domains": [ + "weeworld.com" + ] + }, + "Weibo": { + "domains": [ + "weibo.com", + "t.cn" + ] + }, + "Windows Live Spaces": { + "domains": [ + "login.live.com" + ] + }, + "XING": { + "domains": [ + "xing.com" + ] + }, + "Xanga": { + "domains": [ + "xanga.com" + ] + }, + "hi5": { + "domains": [ + "hi5.com" + ] + }, + "myYearbook": { + "domains": [ + "myyearbook.com" + ] + }, + "ITU Sozluk": { + "domains": [ + "itusozluk.com" + ] + }, + "Eksi Sozluk": { + "domains": [ + "Sozluk.com", + "sourtimes.org" + ] + }, + "Uludag Sozluk": { + "domains": [ + "uludagsozluk.com", + "ulusozluk.com" + ] + }, + "vKruguDruzei.ru": { + "domains": [ + "vkrugudruzei.ru" + ] + }, + "Whirlpool": { + "domains": [ + "forums.whirlpool.net.au" + ] + }, + "Youtube": { + "domains": [ + "youtube.com", + "youtu.be" + ] + } + }, + "unknown": { + "Google": { + "domains": [ + "support.google.com", + "developers.google.com", + "maps.google.com", + "accounts.google.com", + "drive.google.com", + "sites.google.com", + "groups.google.com", + "groups.google.co.uk", + "news.google.co.uk" + ] + }, + "Yahoo!": { + "domains": [ + "finance.yahoo.com", + "news.yahoo.com", + "eurosport.yahoo.com", + "sports.yahoo.com", + "astrology.yahoo.com", + "travel.yahoo.com", + "answers.yahoo.com", + "screen.yahoo.com", + "weather.yahoo.com", + "messenger.yahoo.com", + "games.yahoo.com", + "shopping.yahoo.net", + "movies.yahoo.com", + "cars.yahoo.com", + "lifestyle.yahoo.com", + "omg.yahoo.com", + "match.yahoo.net" + ] + } + } +} +` diff --git a/vendor/github.com/Shopify/goreferrer/dev.yml b/vendor/github.com/Shopify/goreferrer/dev.yml new file mode 100644 index 0000000000..50417d8ab0 --- /dev/null +++ b/vendor/github.com/Shopify/goreferrer/dev.yml @@ -0,0 +1,15 @@ +name: goreferrer + +up: + - go: + version: "1.17" + modules: on + - custom: + name: Setup go dependencies + meet: echo 'go mod failed to install packages'; false + met?: go mod download + +commands: + test: + run: go get -race -t ./... && go test -race ./... + desc: 'run unit tests' diff --git a/vendor/github.com/Shopify/goreferrer/referrer.go b/vendor/github.com/Shopify/goreferrer/referrer.go new file mode 100644 index 0000000000..b9a2263e12 --- /dev/null +++ b/vendor/github.com/Shopify/goreferrer/referrer.go @@ -0,0 +1,76 @@ +package goreferrer + +type ReferrerType int + +const ( + Invalid ReferrerType = iota + Indirect + Direct + Email + Search + Social +) + +func (r ReferrerType) String() string { + switch r { + default: + return "invalid" + case Indirect: + return "indirect" + case Direct: + return "direct" + case Email: + return "email" + case Search: + return "search" + case Social: + return "social" + } +} + +type Referrer struct { + Type ReferrerType + Label string + URL string + Subdomain string + Domain string + Tld string + Path string + Query string + GoogleType GoogleSearchType +} + +func (r *Referrer) RegisteredDomain() string { + if r.Domain != "" && r.Tld != "" { + return r.Domain + "." + r.Tld + } + + return "" +} + +func (r *Referrer) Host() string { + if r.Subdomain != "" { + return r.Subdomain + "." + r.RegisteredDomain() + } + + return r.RegisteredDomain() +} + +type GoogleSearchType int + +const ( + NotGoogleSearch GoogleSearchType = iota + OrganicSearch + Adwords +) + +func (g GoogleSearchType) String() string { + switch g { + default: + return "not google search" + case OrganicSearch: + return "organic google search" + case Adwords: + return "google adwords referrer" + } +} diff --git a/vendor/github.com/Shopify/goreferrer/rich_url.go b/vendor/github.com/Shopify/goreferrer/rich_url.go new file mode 100644 index 0000000000..882d0767a5 --- /dev/null +++ b/vendor/github.com/Shopify/goreferrer/rich_url.go @@ -0,0 +1,53 @@ +package goreferrer + +import ( + "net/url" + "strings" + + "golang.org/x/net/publicsuffix" +) + +type richUrl struct { + *url.URL + Subdomain string + Domain string + Tld string +} + +func parseRichUrl(s string) (*richUrl, bool) { + u, err := url.Parse(s) + if err != nil { + return nil, false + } + + // assume a default scheme of http:// + if u.Scheme == "" { + s = "http://" + s + u, err = url.Parse(s) + if err != nil { + return nil, false + } + } + + tld, _ := publicsuffix.PublicSuffix(u.Host) + if tld == "" || len(u.Host)-len(tld) < 2 { + return nil, false + } + + hostWithoutTld := u.Host[:len(u.Host)-len(tld)-1] + lastDot := strings.LastIndex(hostWithoutTld, ".") + if lastDot == -1 { + return &richUrl{URL: u, Domain: hostWithoutTld, Tld: tld}, true + } + + return &richUrl{ + URL: u, + Subdomain: hostWithoutTld[:lastDot], + Domain: hostWithoutTld[lastDot+1:], + Tld: tld, + }, true +} + +func (u *richUrl) RegisteredDomain() string { + return u.Domain + "." + u.Tld +} diff --git a/vendor/github.com/Shopify/goreferrer/rules.go b/vendor/github.com/Shopify/goreferrer/rules.go new file mode 100644 index 0000000000..f6cc72d179 --- /dev/null +++ b/vendor/github.com/Shopify/goreferrer/rules.go @@ -0,0 +1,206 @@ +package goreferrer + +import ( + "encoding/json" + "io" + "net/url" + "path" + "strings" +) + +type DomainRule struct { + Type ReferrerType + Label string + Domain string + Parameters []string +} + +type UaRule struct { + Url string + Domain string + Tld string +} + +func (u UaRule) RegisteredDomain() string { + if u.Domain == "" || u.Tld == "" { + return "" + } + + return u.Domain + "." + u.Tld +} + +type RuleSet struct { + DomainRules map[string]DomainRule + UaRules map[string]UaRule +} + +func NewRuleSet() RuleSet { + return RuleSet{ + DomainRules: make(map[string]DomainRule), + UaRules: make(map[string]UaRule), + } +} + +func (r RuleSet) Merge(other RuleSet) { + for k, v := range other.DomainRules { + r.DomainRules[k] = v + } + for k, v := range other.UaRules { + r.UaRules[k] = v + } +} + +func (r RuleSet) Parse(URL string) Referrer { + return r.ParseWith(URL, nil, "") +} + +func (r RuleSet) ParseWith(URL string, domains []string, agent string) Referrer { + ref := Referrer{ + Type: Indirect, + URL: strings.Trim(URL, " \t\r\n"), + } + + uaRule := r.getUaRule(agent) + if ref.URL == "" { + ref.URL = uaRule.Url + } + if ref.URL == "" { + ref.Type = Direct + return ref + } + + u, ok := parseRichUrl(ref.URL) + if !ok { + ref.Type = Invalid + return ref + } + + ref.Subdomain = u.Subdomain + ref.Domain = u.Domain + ref.Tld = u.Tld + ref.Path = cleanPath(u.Path) + + if ref.Domain == "" { + ref.Domain = uaRule.Domain + } + if ref.Tld == "" { + ref.Tld = uaRule.Tld + } + + for _, domain := range domains { + if u.Host == domain { + ref.Type = Direct + return ref + } + } + + variations := []string{ + path.Join(u.Host, u.Path), + path.Join(u.RegisteredDomain(), u.Path), + u.Host, + u.RegisteredDomain(), + } + + for _, host := range variations { + domainRule, exists := r.DomainRules[host] + if !exists { + continue + } + + query := getQuery(u.Query(), domainRule.Parameters) + if query == "" { + values, err := url.ParseQuery(u.Fragment) + if err == nil { + query = getQuery(values, domainRule.Parameters) + } + } + + ref.Type = domainRule.Type + ref.Label = domainRule.Label + ref.Query = query + ref.GoogleType = googleSearchType(ref) + return ref + } + + ref.Label = strings.Title(u.Domain) + return ref +} + +func (r *RuleSet) getUaRule(agent string) UaRule { + for pattern, rule := range r.UaRules { + if strings.Contains(agent, pattern) { + return rule + } + } + + return UaRule{} +} + +func getQuery(values url.Values, params []string) string { + for _, param := range params { + query := values.Get(param) + if query != "" { + return query + } + } + + return "" +} + +func googleSearchType(ref Referrer) GoogleSearchType { + if ref.Type != Search || !strings.Contains(ref.Label, "Google") { + return NotGoogleSearch + } + + if strings.HasPrefix(ref.Path, "/aclk") || strings.HasPrefix(ref.Path, "/pagead/aclk") { + return Adwords + } + + return OrganicSearch +} + +func cleanPath(path string) string { + if i := strings.Index(path, ";"); i != -1 { + return path[:i] + } + return path +} + +type jsonRule struct { + Domains []string + Parameters []string +} + +type jsonRules struct { + Email map[string]jsonRule + Search map[string]jsonRule + Social map[string]jsonRule +} + +func LoadJsonDomainRules(reader io.Reader) (map[string]DomainRule, error) { + var decoded jsonRules + if err := json.NewDecoder(reader).Decode(&decoded); err != nil { + return nil, err + } + + rules := NewRuleSet() + rules.Merge(extractRules(decoded.Email, Email)) + rules.Merge(extractRules(decoded.Search, Search)) + rules.Merge(extractRules(decoded.Social, Social)) + return rules.DomainRules, nil +} + +func extractRules(ruleMap map[string]jsonRule, Type ReferrerType) RuleSet { + rules := NewRuleSet() + for label, jsonRule := range ruleMap { + for _, domain := range jsonRule.Domains { + rules.DomainRules[domain] = DomainRule{ + Type: Type, + Label: label, + Domain: domain, + Parameters: jsonRule.Parameters, + } + } + } + return rules +} diff --git a/vendor/github.com/andybalholm/brotli/LICENSE b/vendor/github.com/andybalholm/brotli/LICENSE new file mode 100644 index 0000000000..33b7cdd2db --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/andybalholm/brotli/README.md b/vendor/github.com/andybalholm/brotli/README.md new file mode 100644 index 0000000000..00625211d7 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/README.md @@ -0,0 +1,14 @@ +This package is a brotli compressor and decompressor implemented in Go. +It was translated from the reference implementation (https://github.com/google/brotli) +with the `c2go` tool at https://github.com/andybalholm/c2go. + +I have been working on new compression algorithms (not translated from C) +in the matchfinder package. +You can use them with the NewWriterV2 function. +Currently they give better results than the old implementation +(at least for compressing my test file, Newton’s *Opticks*) +on levels 2 to 6. + +I am using it in production with https://github.com/andybalholm/redwood. + +API documentation is found at https://pkg.go.dev/github.com/andybalholm/brotli?tab=doc. diff --git a/vendor/github.com/andybalholm/brotli/backward_references.go b/vendor/github.com/andybalholm/brotli/backward_references.go new file mode 100644 index 0000000000..008c054d1c --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/backward_references.go @@ -0,0 +1,185 @@ +package brotli + +import ( + "sync" +) + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Function to find backward reference copies. */ + +func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint { + if distance <= max_distance { + var distance_plus_3 uint = distance + 3 + var offset0 uint = distance_plus_3 - uint(dist_cache[0]) + var offset1 uint = distance_plus_3 - uint(dist_cache[1]) + if distance == uint(dist_cache[0]) { + return 0 + } else if distance == uint(dist_cache[1]) { + return 1 + } else if offset0 < 7 { + return (0x9750468 >> (4 * offset0)) & 0xF + } else if offset1 < 7 { + return (0xFDB1ACE >> (4 * offset1)) & 0xF + } else if distance == uint(dist_cache[2]) { + return 2 + } else if distance == uint(dist_cache[3]) { + return 3 + } + } + + return distance + numDistanceShortCodes - 1 +} + +var hasherSearchResultPool sync.Pool + +func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var insert_length uint = *last_insert_len + var pos_end uint = position + num_bytes + var store_end uint + if num_bytes >= hasher.StoreLookahead() { + store_end = position + num_bytes - hasher.StoreLookahead() + 1 + } else { + store_end = position + } + var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params) + var apply_random_heuristics uint = position + random_heuristics_window_size + var gap uint = 0 + /* Set maximum distance, see section 9.1. of the spec. */ + + const kMinScore uint = scoreBase + 100 + + /* For speed up heuristics for random data. */ + + /* Minimum score to accept a backward reference. */ + hasher.PrepareDistanceCache(dist_cache) + sr2, _ := hasherSearchResultPool.Get().(*hasherSearchResult) + if sr2 == nil { + sr2 = &hasherSearchResult{} + } + sr, _ := hasherSearchResultPool.Get().(*hasherSearchResult) + if sr == nil { + sr = &hasherSearchResult{} + } + + for position+hasher.HashTypeLength() < pos_end { + var max_length uint = pos_end - position + var max_distance uint = brotli_min_size_t(position, max_backward_limit) + sr.len = 0 + sr.len_code_delta = 0 + sr.distance = 0 + sr.score = kMinScore + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, sr) + if sr.score > kMinScore { + /* Found a match. Let's look for something even better ahead. */ + var delayed_backward_references_in_row int = 0 + max_length-- + for ; ; max_length-- { + var cost_diff_lazy uint = 175 + if params.quality < minQualityForExtensiveReferenceSearch { + sr2.len = brotli_min_size_t(sr.len-1, max_length) + } else { + sr2.len = 0 + } + sr2.len_code_delta = 0 + sr2.distance = 0 + sr2.score = kMinScore + max_distance = brotli_min_size_t(position+1, max_backward_limit) + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, sr2) + if sr2.score >= sr.score+cost_diff_lazy { + /* Ok, let's just write one byte for now and start a match from the + next byte. */ + position++ + + insert_length++ + *sr = *sr2 + delayed_backward_references_in_row++ + if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end { + continue + } + } + + break + } + + apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size + max_distance = brotli_min_size_t(position, max_backward_limit) + { + /* The first 16 codes are special short-codes, + and the minimum offset is 1. */ + var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache) + if (sr.distance <= (max_distance + gap)) && distance_code > 0 { + dist_cache[3] = dist_cache[2] + dist_cache[2] = dist_cache[1] + dist_cache[1] = dist_cache[0] + dist_cache[0] = int(sr.distance) + hasher.PrepareDistanceCache(dist_cache) + } + + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code)) + } + + *num_literals += insert_length + insert_length = 0 + /* Put the hash keys into the table, if there are enough bytes left. + Depending on the hasher implementation, it can push all positions + in the given range or only a subset of them. + Avoid hash poisoning with RLE data. */ + { + var range_start uint = position + 2 + var range_end uint = brotli_min_size_t(position+sr.len, store_end) + if sr.distance < sr.len>>2 { + range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2))) + } + + hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end) + } + + position += sr.len + } else { + insert_length++ + position++ + + /* If we have not seen matches for a long time, we can skip some + match lookups. Unsuccessful match lookups are very very expensive + and this kind of a heuristic speeds up compression quite + a lot. */ + if position > apply_random_heuristics { + /* Going through uncompressible data, jump. */ + if position > apply_random_heuristics+4*random_heuristics_window_size { + var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4) + /* It is quite a long time since we saw a copy, so we assume + that this data is not compressible, and store hashes less + often. Hashes of non compressible data are less likely to + turn out to be useful in the future, too, so we store less of + them to not to flood out the hash table of good compressible + data. */ + + var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin) + for ; position < pos_jump; position += 4 { + hasher.Store(ringbuffer, ringbuffer_mask, position) + insert_length += 4 + } + } else { + var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2) + var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin) + for ; position < pos_jump; position += 2 { + hasher.Store(ringbuffer, ringbuffer_mask, position) + insert_length += 2 + } + } + } + } + } + + insert_length += pos_end - position + *last_insert_len = insert_length + + hasherSearchResultPool.Put(sr) + hasherSearchResultPool.Put(sr2) +} diff --git a/vendor/github.com/andybalholm/brotli/backward_references_hq.go b/vendor/github.com/andybalholm/brotli/backward_references_hq.go new file mode 100644 index 0000000000..21629c1cdb --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/backward_references_hq.go @@ -0,0 +1,796 @@ +package brotli + +import "math" + +type zopfliNode struct { + length uint32 + distance uint32 + dcode_insert_length uint32 + u struct { + cost float32 + next uint32 + shortcut uint32 + } +} + +const maxEffectiveDistanceAlphabetSize = 544 + +const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */ + +var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1} + +var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3} + +func initZopfliNodes(array []zopfliNode, length uint) { + var stub zopfliNode + var i uint + stub.length = 1 + stub.distance = 0 + stub.dcode_insert_length = 0 + stub.u.cost = kInfinity + for i = 0; i < length; i++ { + array[i] = stub + } +} + +func zopfliNodeCopyLength(self *zopfliNode) uint32 { + return self.length & 0x1FFFFFF +} + +func zopfliNodeLengthCode(self *zopfliNode) uint32 { + var modifier uint32 = self.length >> 25 + return zopfliNodeCopyLength(self) + 9 - modifier +} + +func zopfliNodeCopyDistance(self *zopfliNode) uint32 { + return self.distance +} + +func zopfliNodeDistanceCode(self *zopfliNode) uint32 { + var short_code uint32 = self.dcode_insert_length >> 27 + if short_code == 0 { + return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1 + } else { + return short_code - 1 + } +} + +func zopfliNodeCommandLength(self *zopfliNode) uint32 { + return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF) +} + +/* Histogram based cost model for zopflification. */ +type zopfliCostModel struct { + cost_cmd_ [numCommandSymbols]float32 + cost_dist_ []float32 + distance_histogram_size uint32 + literal_costs_ []float32 + min_cost_cmd_ float32 + num_bytes_ uint +} + +func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) { + var distance_histogram_size uint32 = dist.alphabet_size + if distance_histogram_size > maxEffectiveDistanceAlphabetSize { + distance_histogram_size = maxEffectiveDistanceAlphabetSize + } + + self.num_bytes_ = num_bytes + self.literal_costs_ = make([]float32, (num_bytes + 2)) + self.cost_dist_ = make([]float32, (dist.alphabet_size)) + self.distance_histogram_size = distance_histogram_size +} + +func cleanupZopfliCostModel(self *zopfliCostModel) { + self.literal_costs_ = nil + self.cost_dist_ = nil +} + +func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) { + var sum uint = 0 + var missing_symbol_sum uint + var log2sum float32 + var missing_symbol_cost float32 + var i uint + for i = 0; i < histogram_size; i++ { + sum += uint(histogram[i]) + } + + log2sum = float32(fastLog2(sum)) + missing_symbol_sum = sum + if !literal_histogram { + for i = 0; i < histogram_size; i++ { + if histogram[i] == 0 { + missing_symbol_sum++ + } + } + } + + missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2 + for i = 0; i < histogram_size; i++ { + if histogram[i] == 0 { + cost[i] = missing_symbol_cost + continue + } + + /* Shannon bits for this symbol. */ + cost[i] = log2sum - float32(fastLog2(uint(histogram[i]))) + + /* Cannot be coded with less than 1 bit */ + if cost[i] < 1 { + cost[i] = 1 + } + } +} + +func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) { + var histogram_literal [numLiteralSymbols]uint32 + var histogram_cmd [numCommandSymbols]uint32 + var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32 + var cost_literal [numLiteralSymbols]float32 + var pos uint = position - last_insert_len + var min_cost_cmd float32 = kInfinity + var cost_cmd []float32 = self.cost_cmd_[:] + var literal_costs []float32 + + histogram_literal = [numLiteralSymbols]uint32{} + histogram_cmd = [numCommandSymbols]uint32{} + histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{} + + for i := range commands { + var inslength uint = uint(commands[i].insert_len_) + var copylength uint = uint(commandCopyLen(&commands[i])) + var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF + var cmdcode uint = uint(commands[i].cmd_prefix_) + var j uint + + histogram_cmd[cmdcode]++ + if cmdcode >= 128 { + histogram_dist[distcode]++ + } + + for j = 0; j < inslength; j++ { + histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++ + } + + pos += inslength + copylength + } + + setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:]) + setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd) + setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_) + + for i := 0; i < numCommandSymbols; i++ { + min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i]) + } + + self.min_cost_cmd_ = min_cost_cmd + { + literal_costs = self.literal_costs_ + var literal_carry float32 = 0.0 + num_bytes := int(self.num_bytes_) + literal_costs[0] = 0.0 + for i := 0; i < num_bytes; i++ { + literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]] + literal_costs[i+1] = literal_costs[i] + literal_carry + literal_carry -= literal_costs[i+1] - literal_costs[i] + } + } +} + +func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) { + var literal_costs []float32 = self.literal_costs_ + var literal_carry float32 = 0.0 + var cost_dist []float32 = self.cost_dist_ + var cost_cmd []float32 = self.cost_cmd_[:] + var num_bytes uint = self.num_bytes_ + var i uint + estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:]) + literal_costs[0] = 0.0 + for i = 0; i < num_bytes; i++ { + literal_carry += literal_costs[i+1] + literal_costs[i+1] = literal_costs[i] + literal_carry + literal_carry -= literal_costs[i+1] - literal_costs[i] + } + + for i = 0; i < numCommandSymbols; i++ { + cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i)))) + } + + for i = 0; uint32(i) < self.distance_histogram_size; i++ { + cost_dist[i] = float32(fastLog2(uint(20 + uint32(i)))) + } + + self.min_cost_cmd_ = float32(fastLog2(11)) +} + +func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 { + return self.cost_cmd_[cmdcode] +} + +func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 { + return self.cost_dist_[distcode] +} + +func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 { + return self.literal_costs_[to] - self.literal_costs_[from] +} + +func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 { + return self.min_cost_cmd_ +} + +/* REQUIRES: len >= 2, start_pos <= pos */ +/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */ +/* Maintains the "ZopfliNode array invariant". */ +func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) { + var next *zopfliNode = &nodes[pos+len] + next.length = uint32(len | (len+9-len_code)<<25) + next.distance = uint32(dist) + next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos)) + next.u.cost = cost +} + +type posData struct { + pos uint + distance_cache [4]int + costdiff float32 + cost float32 +} + +/* Maintains the smallest 8 cost difference together with their positions */ +type startPosQueue struct { + q_ [8]posData + idx_ uint +} + +func initStartPosQueue(self *startPosQueue) { + self.idx_ = 0 +} + +func startPosQueueSize(self *startPosQueue) uint { + return brotli_min_size_t(self.idx_, 8) +} + +func startPosQueuePush(self *startPosQueue, posdata *posData) { + var offset uint = ^(self.idx_) & 7 + self.idx_++ + var len uint = startPosQueueSize(self) + var i uint + var q []posData = self.q_[:] + q[offset] = *posdata + + /* Restore the sorted order. In the list of |len| items at most |len - 1| + adjacent element comparisons / swaps are required. */ + for i = 1; i < len; i++ { + if q[offset&7].costdiff > q[(offset+1)&7].costdiff { + var tmp posData = q[offset&7] + q[offset&7] = q[(offset+1)&7] + q[(offset+1)&7] = tmp + } + + offset++ + } +} + +func startPosQueueAt(self *startPosQueue, k uint) *posData { + return &self.q_[(k-self.idx_)&7] +} + +/* Returns the minimum possible copy length that can improve the cost of any */ +/* future position. */ +func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint { + var min_cost float32 = start_cost + var len uint = 2 + var next_len_bucket uint = 4 + /* Compute the minimum possible cost of reaching any future position. */ + + var next_len_offset uint = 10 + for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost { + /* We already reached (pos + len) with no more cost than the minimum + possible cost of reaching anything from this pos, so there is no point in + looking for lengths <= len. */ + len++ + + if len == next_len_offset { + /* We reached the next copy length code bucket, so we add one more + extra bit to the minimum cost. */ + min_cost += 1.0 + + next_len_offset += next_len_bucket + next_len_bucket *= 2 + } + } + + return uint(len) +} + +/* REQUIRES: nodes[pos].cost < kInfinity + REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ +func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 { + var clen uint = uint(zopfliNodeCopyLength(&nodes[pos])) + var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF) + var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos])) + + /* Since |block_start + pos| is the end position of the command, the copy part + starts from |block_start + pos - clen|. Distances that are greater than + this or greater than |max_backward_limit| + |gap| are static dictionary + references, and do not update the last distances. + Also distance code 0 (last distance) does not update the last distances. */ + if pos == 0 { + return 0 + } else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 { + return uint32(pos) + } else { + return nodes[pos-clen-ilen].u.shortcut + } +} + +/* Fills in dist_cache[0..3] with the last four distances (as defined by + Section 4. of the Spec) that would be used at (block_start + pos) if we + used the shortest path of commands from block_start, computed from + nodes[0..pos]. The last four distances at block_start are in + starting_dist_cache[0..3]. + REQUIRES: nodes[pos].cost < kInfinity + REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ +func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) { + var idx int = 0 + var p uint = uint(nodes[pos].u.shortcut) + for idx < 4 && p > 0 { + var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF) + var clen uint = uint(zopfliNodeCopyLength(&nodes[p])) + var dist uint = uint(zopfliNodeCopyDistance(&nodes[p])) + dist_cache[idx] = int(dist) + idx++ + + /* Because of prerequisite, p >= clen + ilen >= 2. */ + p = uint(nodes[p-clen-ilen].u.shortcut) + } + + for ; idx < 4; idx++ { + dist_cache[idx] = starting_dist_cache[0] + starting_dist_cache = starting_dist_cache[1:] + } +} + +/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it + is eligible. */ +func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) { + /* Save cost, because ComputeDistanceCache invalidates it. */ + var node_cost float32 = nodes[pos].u.cost + nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes) + if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) { + var posdata posData + posdata.pos = pos + posdata.cost = node_cost + posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos) + computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:]) + startPosQueuePush(queue, &posdata) + } +} + +/* Returns longest copy length. */ +func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint { + var cur_ix uint = block_start + pos + var cur_ix_masked uint = cur_ix & ringbuffer_mask + var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit) + var max_len uint = num_bytes - pos + var max_zopfli_len uint = maxZopfliLen(params) + var max_iters uint = maxZopfliCandidates(params) + var min_len uint + var result uint = 0 + var k uint + var gap uint = 0 + + evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes) + { + var posdata *posData = startPosQueueAt(queue, 0) + var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos)) + min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos) + } + + /* Go over the command starting positions in order of increasing cost + difference. */ + for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ { + var posdata *posData = startPosQueueAt(queue, k) + var start uint = posdata.pos + var inscode uint16 = getInsertLengthCode(pos - start) + var start_costdiff float32 = posdata.costdiff + var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos) + var best_len uint = min_len - 1 + var j uint = 0 + /* Look for last distance matches using the distance cache from this + starting position. */ + for ; j < numDistanceShortCodes && best_len < max_len; j++ { + var idx uint = uint(kDistanceCacheIndex[j]) + var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j]) + var prev_ix uint = cur_ix - backward + var len uint = 0 + var continuation byte = ringbuffer[cur_ix_masked+best_len] + if cur_ix_masked+best_len > ringbuffer_mask { + break + } + + if backward > max_distance+gap { + /* Word dictionary -> ignore. */ + continue + } + + if backward <= max_distance { + /* Regular backward reference. */ + if prev_ix >= cur_ix { + continue + } + + prev_ix &= ringbuffer_mask + if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] { + continue + } + + len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len) + } else { + continue + } + { + var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j) + var l uint + for l = best_len + 1; l <= len; l++ { + var copycode uint16 = getCopyLengthCode(l) + var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0) + var tmp float32 + if cmdcode < 128 { + tmp = base_cost + } else { + tmp = dist_cost + } + var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) + if cost < nodes[pos+l].u.cost { + updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost) + result = brotli_max_size_t(result, l) + } + + best_len = l + } + } + } + + /* At higher iterations look only for new last distance matches, since + looking only for new command start positions with the same distances + does not help much. */ + if k >= 2 { + continue + } + { + /* Loop through all possible copy lengths at this position. */ + var len uint = min_len + for j = 0; j < num_matches; j++ { + var match backwardMatch = matches[j] + var dist uint = uint(match.distance) + var is_dictionary_match bool = (dist > max_distance+gap) + var dist_code uint = dist + numDistanceShortCodes - 1 + var dist_symbol uint16 + var distextra uint32 + var distnumextra uint32 + var dist_cost float32 + var max_match_len uint + /* We already tried all possible last distance matches, so we can use + normal distance code here. */ + prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra) + + distnumextra = uint32(dist_symbol) >> 10 + dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF) + + /* Try all copy lengths up until the maximum copy length corresponding + to this distance. If the distance refers to the static dictionary, or + the maximum length is long enough, try only one maximum length. */ + max_match_len = backwardMatchLength(&match) + + if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) { + len = max_match_len + } + + for ; len <= max_match_len; len++ { + var len_code uint + if is_dictionary_match { + len_code = backwardMatchLengthCode(&match) + } else { + len_code = len + } + var copycode uint16 = getCopyLengthCode(len_code) + var cmdcode uint16 = combineLengthCodes(inscode, copycode, false) + var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) + if cost < nodes[pos+len].u.cost { + updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost) + if len > result { + result = len + } + } + } + } + } + } + + return result +} + +func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint { + var index uint = num_bytes + var num_commands uint = 0 + for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 { + index-- + } + nodes[index].u.next = math.MaxUint32 + for index != 0 { + var len uint = uint(zopfliNodeCommandLength(&nodes[index])) + index -= uint(len) + nodes[index].u.next = uint32(len) + num_commands++ + } + + return num_commands +} + +/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ +func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var pos uint = 0 + var offset uint32 = nodes[0].u.next + var i uint + var gap uint = 0 + for i = 0; offset != math.MaxUint32; i++ { + var next *zopfliNode = &nodes[uint32(pos)+offset] + var copy_length uint = uint(zopfliNodeCopyLength(next)) + var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF) + pos += insert_length + offset = next.u.next + if i == 0 { + insert_length += *last_insert_len + *last_insert_len = 0 + } + { + var distance uint = uint(zopfliNodeCopyDistance(next)) + var len_code uint = uint(zopfliNodeLengthCode(next)) + var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit) + var is_dictionary bool = (distance > max_distance+gap) + var dist_code uint = uint(zopfliNodeDistanceCode(next)) + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)) + + if !is_dictionary && dist_code > 0 { + dist_cache[3] = dist_cache[2] + dist_cache[2] = dist_cache[1] + dist_cache[1] = dist_cache[0] + dist_cache[0] = int(distance) + } + } + + *num_literals += insert_length + pos += copy_length + } + + *last_insert_len += num_bytes - pos +} + +func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var max_zopfli_len uint = maxZopfliLen(params) + var queue startPosQueue + var cur_match_pos uint = 0 + var i uint + nodes[0].length = 0 + nodes[0].u.cost = 0 + initStartPosQueue(&queue) + for i = 0; i+3 < num_bytes; i++ { + var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes) + if skip < longCopyQuickStep { + skip = 0 + } + cur_match_pos += uint(num_matches[i]) + if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len { + skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip) + } + + if skip > 1 { + skip-- + for skip != 0 { + i++ + if i+3 >= num_bytes { + break + } + evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes) + cur_match_pos += uint(num_matches[i]) + skip-- + } + } + } + + return computeShortestPathFromNodes(num_bytes, nodes) +} + +/* Computes the shortest path of commands from position to at most + position + num_bytes. + + On return, path->size() is the number of commands found and path[i] is the + length of the i-th command (copy length plus insert length). + Note that the sum of the lengths of all commands can be less than num_bytes. + + On return, the nodes[0..num_bytes] array will have the following + "ZopfliNode array invariant": + For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then + (1) nodes[i].copy_length() >= 2 + (2) nodes[i].command_length() <= i and + (3) nodes[i - nodes[i].command_length()].cost < kInfinity + + REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */ +func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var max_zopfli_len uint = maxZopfliLen(params) + var model zopfliCostModel + var queue startPosQueue + var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch + var store_end uint + if num_bytes >= hasher.StoreLookahead() { + store_end = position + num_bytes - hasher.StoreLookahead() + 1 + } else { + store_end = position + } + var i uint + var gap uint = 0 + var lz_matches_offset uint = 0 + nodes[0].length = 0 + nodes[0].u.cost = 0 + initZopfliCostModel(&model, ¶ms.dist, num_bytes) + zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) + initStartPosQueue(&queue) + for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ { + var pos uint = position + i + var max_distance uint = brotli_min_size_t(pos, max_backward_limit) + var skip uint + var num_matches uint + num_matches = findAllMatchesH10(hasher, ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:]) + if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len { + matches[0] = matches[num_matches-1] + num_matches = 1 + } + + skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes) + if skip < longCopyQuickStep { + skip = 0 + } + if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len { + skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip) + } + + if skip > 1 { + /* Add the tail of the copy to the hasher. */ + hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end)) + + skip-- + for skip != 0 { + i++ + if i+hasher.HashTypeLength()-1 >= num_bytes { + break + } + evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes) + skip-- + } + } + } + + cleanupZopfliCostModel(&model) + return computeShortestPathFromNodes(num_bytes, nodes) +} + +func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { + var nodes []zopfliNode + nodes = make([]zopfliNode, (num_bytes + 1)) + initZopfliNodes(nodes, num_bytes+1) + zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) + zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) + nodes = nil +} + +func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var num_matches []uint32 = make([]uint32, num_bytes) + var matches_size uint = 4 * num_bytes + var store_end uint + if num_bytes >= hasher.StoreLookahead() { + store_end = position + num_bytes - hasher.StoreLookahead() + 1 + } else { + store_end = position + } + var cur_match_pos uint = 0 + var i uint + var orig_num_literals uint + var orig_last_insert_len uint + var orig_dist_cache [4]int + var orig_num_commands int + var model zopfliCostModel + var nodes []zopfliNode + var matches []backwardMatch = make([]backwardMatch, matches_size) + var gap uint = 0 + var shadow_matches uint = 0 + var new_array []backwardMatch + for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ { + var pos uint = position + i + var max_distance uint = brotli_min_size_t(pos, max_backward_limit) + var max_length uint = num_bytes - i + var num_found_matches uint + var cur_match_end uint + var j uint + + /* Ensure that we have enough free slots. */ + if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches { + var new_size uint = matches_size + if new_size == 0 { + new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches + } + + for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches { + new_size *= 2 + } + + new_array = make([]backwardMatch, new_size) + if matches_size != 0 { + copy(new_array, matches[:matches_size]) + } + + matches = new_array + matches_size = new_size + } + + num_found_matches = findAllMatchesH10(hasher.(*h10), ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:]) + cur_match_end = cur_match_pos + num_found_matches + for j = cur_match_pos; j+1 < cur_match_end; j++ { + assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1])) + } + + num_matches[i] = uint32(num_found_matches) + if num_found_matches > 0 { + var match_len uint = backwardMatchLength(&matches[cur_match_end-1]) + if match_len > maxZopfliLenQuality11 { + var skip uint = match_len - 1 + matches[cur_match_pos] = matches[cur_match_end-1] + cur_match_pos++ + num_matches[i] = 1 + + /* Add the tail of the copy to the hasher. */ + hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end)) + var pos uint = i + for i := 0; i < int(skip); i++ { + num_matches[pos+1:][i] = 0 + } + i += skip + } else { + cur_match_pos = cur_match_end + } + } + } + + orig_num_literals = *num_literals + orig_last_insert_len = *last_insert_len + copy(orig_dist_cache[:], dist_cache[:4]) + orig_num_commands = len(*commands) + nodes = make([]zopfliNode, (num_bytes + 1)) + initZopfliCostModel(&model, ¶ms.dist, num_bytes) + for i = 0; i < 2; i++ { + initZopfliNodes(nodes, num_bytes+1) + if i == 0 { + zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) + } else { + zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len) + } + + *commands = (*commands)[:orig_num_commands] + *num_literals = orig_num_literals + *last_insert_len = orig_last_insert_len + copy(dist_cache, orig_dist_cache[:4]) + zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) + zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) + } + + cleanupZopfliCostModel(&model) + nodes = nil + matches = nil + num_matches = nil +} diff --git a/vendor/github.com/andybalholm/brotli/bit_cost.go b/vendor/github.com/andybalholm/brotli/bit_cost.go new file mode 100644 index 0000000000..0005fc15e6 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/bit_cost.go @@ -0,0 +1,436 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Functions to estimate the bit cost of Huffman trees. */ +func shannonEntropy(population []uint32, size uint, total *uint) float64 { + var sum uint = 0 + var retval float64 = 0 + var population_end []uint32 = population[size:] + var p uint + for -cap(population) < -cap(population_end) { + p = uint(population[0]) + population = population[1:] + sum += p + retval -= float64(p) * fastLog2(p) + } + + if sum != 0 { + retval += float64(sum) * fastLog2(sum) + } + *total = sum + return retval +} + +func bitsEntropy(population []uint32, size uint) float64 { + var sum uint + var retval float64 = shannonEntropy(population, size, &sum) + if retval < float64(sum) { + /* At least one bit per literal is needed. */ + retval = float64(sum) + } + + return retval +} + +const kOneSymbolHistogramCost float64 = 12 +const kTwoSymbolHistogramCost float64 = 20 +const kThreeSymbolHistogramCost float64 = 28 +const kFourSymbolHistogramCost float64 = 37 + +func populationCostLiteral(histogram *histogramLiteral) float64 { + var data_size uint = histogramDataSizeLiteral() + var count int = 0 + var s [5]uint + var bits float64 = 0.0 + var i uint + if histogram.total_count_ == 0 { + return kOneSymbolHistogramCost + } + + for i = 0; i < data_size; i++ { + if histogram.data_[i] > 0 { + s[count] = i + count++ + if count > 4 { + break + } + } + } + + if count == 1 { + return kOneSymbolHistogramCost + } + + if count == 2 { + return kTwoSymbolHistogramCost + float64(histogram.total_count_) + } + + if count == 3 { + var histo0 uint32 = histogram.data_[s[0]] + var histo1 uint32 = histogram.data_[s[1]] + var histo2 uint32 = histogram.data_[s[2]] + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) + } + + if count == 4 { + var histo [4]uint32 + var h23 uint32 + var histomax uint32 + for i = 0; i < 4; i++ { + histo[i] = histogram.data_[s[i]] + } + + /* Sort */ + for i = 0; i < 4; i++ { + var j uint + for j = i + 1; j < 4; j++ { + if histo[j] > histo[i] { + var tmp uint32 = histo[j] + histo[j] = histo[i] + histo[i] = tmp + } + } + } + + h23 = histo[2] + histo[3] + histomax = brotli_max_uint32_t(h23, histo[0]) + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) + } + { + var max_depth uint = 1 + var depth_histo = [codeLengthCodes]uint32{0} + /* In this loop we compute the entropy of the histogram and simultaneously + build a simplified histogram of the code length codes where we use the + zero repeat code 17, but we don't use the non-zero repeat code 16. */ + + var log2total float64 = fastLog2(histogram.total_count_) + for i = 0; i < data_size; { + if histogram.data_[i] > 0 { + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = + = log2(total_count) - log2(count(symbol)) */ + + var depth uint = uint(log2p + 0.5) + /* Approximate the bit depth by round(-log2(P(symbol))) */ + bits += float64(histogram.data_[i]) * log2p + + if depth > 15 { + depth = 15 + } + + if depth > max_depth { + max_depth = depth + } + + depth_histo[depth]++ + i++ + } else { + var reps uint32 = 1 + /* Compute the run length of zeros and add the appropriate number of 0 + and 17 code length codes to the code length code histogram. */ + + var k uint + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { + reps++ + } + + i += uint(reps) + if i == data_size { + /* Don't add any cost for the last zero run, since these are encoded + only implicitly. */ + break + } + + if reps < 3 { + depth_histo[0] += reps + } else { + reps -= 2 + for reps > 0 { + depth_histo[repeatZeroCodeLength]++ + + /* Add the 3 extra bits for the 17 code length code. */ + bits += 3 + + reps >>= 3 + } + } + } + } + + /* Add the estimated encoding cost of the code length code histogram. */ + bits += float64(18 + 2*max_depth) + + /* Add the entropy of the code length code histogram. */ + bits += bitsEntropy(depth_histo[:], codeLengthCodes) + } + + return bits +} + +func populationCostCommand(histogram *histogramCommand) float64 { + var data_size uint = histogramDataSizeCommand() + var count int = 0 + var s [5]uint + var bits float64 = 0.0 + var i uint + if histogram.total_count_ == 0 { + return kOneSymbolHistogramCost + } + + for i = 0; i < data_size; i++ { + if histogram.data_[i] > 0 { + s[count] = i + count++ + if count > 4 { + break + } + } + } + + if count == 1 { + return kOneSymbolHistogramCost + } + + if count == 2 { + return kTwoSymbolHistogramCost + float64(histogram.total_count_) + } + + if count == 3 { + var histo0 uint32 = histogram.data_[s[0]] + var histo1 uint32 = histogram.data_[s[1]] + var histo2 uint32 = histogram.data_[s[2]] + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) + } + + if count == 4 { + var histo [4]uint32 + var h23 uint32 + var histomax uint32 + for i = 0; i < 4; i++ { + histo[i] = histogram.data_[s[i]] + } + + /* Sort */ + for i = 0; i < 4; i++ { + var j uint + for j = i + 1; j < 4; j++ { + if histo[j] > histo[i] { + var tmp uint32 = histo[j] + histo[j] = histo[i] + histo[i] = tmp + } + } + } + + h23 = histo[2] + histo[3] + histomax = brotli_max_uint32_t(h23, histo[0]) + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) + } + { + var max_depth uint = 1 + var depth_histo = [codeLengthCodes]uint32{0} + /* In this loop we compute the entropy of the histogram and simultaneously + build a simplified histogram of the code length codes where we use the + zero repeat code 17, but we don't use the non-zero repeat code 16. */ + + var log2total float64 = fastLog2(histogram.total_count_) + for i = 0; i < data_size; { + if histogram.data_[i] > 0 { + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = + = log2(total_count) - log2(count(symbol)) */ + + var depth uint = uint(log2p + 0.5) + /* Approximate the bit depth by round(-log2(P(symbol))) */ + bits += float64(histogram.data_[i]) * log2p + + if depth > 15 { + depth = 15 + } + + if depth > max_depth { + max_depth = depth + } + + depth_histo[depth]++ + i++ + } else { + var reps uint32 = 1 + /* Compute the run length of zeros and add the appropriate number of 0 + and 17 code length codes to the code length code histogram. */ + + var k uint + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { + reps++ + } + + i += uint(reps) + if i == data_size { + /* Don't add any cost for the last zero run, since these are encoded + only implicitly. */ + break + } + + if reps < 3 { + depth_histo[0] += reps + } else { + reps -= 2 + for reps > 0 { + depth_histo[repeatZeroCodeLength]++ + + /* Add the 3 extra bits for the 17 code length code. */ + bits += 3 + + reps >>= 3 + } + } + } + } + + /* Add the estimated encoding cost of the code length code histogram. */ + bits += float64(18 + 2*max_depth) + + /* Add the entropy of the code length code histogram. */ + bits += bitsEntropy(depth_histo[:], codeLengthCodes) + } + + return bits +} + +func populationCostDistance(histogram *histogramDistance) float64 { + var data_size uint = histogramDataSizeDistance() + var count int = 0 + var s [5]uint + var bits float64 = 0.0 + var i uint + if histogram.total_count_ == 0 { + return kOneSymbolHistogramCost + } + + for i = 0; i < data_size; i++ { + if histogram.data_[i] > 0 { + s[count] = i + count++ + if count > 4 { + break + } + } + } + + if count == 1 { + return kOneSymbolHistogramCost + } + + if count == 2 { + return kTwoSymbolHistogramCost + float64(histogram.total_count_) + } + + if count == 3 { + var histo0 uint32 = histogram.data_[s[0]] + var histo1 uint32 = histogram.data_[s[1]] + var histo2 uint32 = histogram.data_[s[2]] + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) + } + + if count == 4 { + var histo [4]uint32 + var h23 uint32 + var histomax uint32 + for i = 0; i < 4; i++ { + histo[i] = histogram.data_[s[i]] + } + + /* Sort */ + for i = 0; i < 4; i++ { + var j uint + for j = i + 1; j < 4; j++ { + if histo[j] > histo[i] { + var tmp uint32 = histo[j] + histo[j] = histo[i] + histo[i] = tmp + } + } + } + + h23 = histo[2] + histo[3] + histomax = brotli_max_uint32_t(h23, histo[0]) + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) + } + { + var max_depth uint = 1 + var depth_histo = [codeLengthCodes]uint32{0} + /* In this loop we compute the entropy of the histogram and simultaneously + build a simplified histogram of the code length codes where we use the + zero repeat code 17, but we don't use the non-zero repeat code 16. */ + + var log2total float64 = fastLog2(histogram.total_count_) + for i = 0; i < data_size; { + if histogram.data_[i] > 0 { + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = + = log2(total_count) - log2(count(symbol)) */ + + var depth uint = uint(log2p + 0.5) + /* Approximate the bit depth by round(-log2(P(symbol))) */ + bits += float64(histogram.data_[i]) * log2p + + if depth > 15 { + depth = 15 + } + + if depth > max_depth { + max_depth = depth + } + + depth_histo[depth]++ + i++ + } else { + var reps uint32 = 1 + /* Compute the run length of zeros and add the appropriate number of 0 + and 17 code length codes to the code length code histogram. */ + + var k uint + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { + reps++ + } + + i += uint(reps) + if i == data_size { + /* Don't add any cost for the last zero run, since these are encoded + only implicitly. */ + break + } + + if reps < 3 { + depth_histo[0] += reps + } else { + reps -= 2 + for reps > 0 { + depth_histo[repeatZeroCodeLength]++ + + /* Add the 3 extra bits for the 17 code length code. */ + bits += 3 + + reps >>= 3 + } + } + } + } + + /* Add the estimated encoding cost of the code length code histogram. */ + bits += float64(18 + 2*max_depth) + + /* Add the entropy of the code length code histogram. */ + bits += bitsEntropy(depth_histo[:], codeLengthCodes) + } + + return bits +} diff --git a/vendor/github.com/andybalholm/brotli/bit_reader.go b/vendor/github.com/andybalholm/brotli/bit_reader.go new file mode 100644 index 0000000000..fba8687c69 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/bit_reader.go @@ -0,0 +1,266 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Bit reading helpers */ + +const shortFillBitWindowRead = (8 >> 1) + +var kBitMask = [33]uint32{ + 0x00000000, + 0x00000001, + 0x00000003, + 0x00000007, + 0x0000000F, + 0x0000001F, + 0x0000003F, + 0x0000007F, + 0x000000FF, + 0x000001FF, + 0x000003FF, + 0x000007FF, + 0x00000FFF, + 0x00001FFF, + 0x00003FFF, + 0x00007FFF, + 0x0000FFFF, + 0x0001FFFF, + 0x0003FFFF, + 0x0007FFFF, + 0x000FFFFF, + 0x001FFFFF, + 0x003FFFFF, + 0x007FFFFF, + 0x00FFFFFF, + 0x01FFFFFF, + 0x03FFFFFF, + 0x07FFFFFF, + 0x0FFFFFFF, + 0x1FFFFFFF, + 0x3FFFFFFF, + 0x7FFFFFFF, + 0xFFFFFFFF, +} + +func bitMask(n uint32) uint32 { + return kBitMask[n] +} + +type bitReader struct { + val_ uint64 + bit_pos_ uint32 + input []byte + input_len uint + byte_pos uint +} + +type bitReaderState struct { + val_ uint64 + bit_pos_ uint32 + input []byte + input_len uint + byte_pos uint +} + +/* Initializes the BrotliBitReader fields. */ + +/* Ensures that accumulator is not empty. + May consume up to sizeof(brotli_reg_t) - 1 bytes of input. + Returns false if data is required but there is no input available. + For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned + reading. */ +func bitReaderSaveState(from *bitReader, to *bitReaderState) { + to.val_ = from.val_ + to.bit_pos_ = from.bit_pos_ + to.input = from.input + to.input_len = from.input_len + to.byte_pos = from.byte_pos +} + +func bitReaderRestoreState(to *bitReader, from *bitReaderState) { + to.val_ = from.val_ + to.bit_pos_ = from.bit_pos_ + to.input = from.input + to.input_len = from.input_len + to.byte_pos = from.byte_pos +} + +func getAvailableBits(br *bitReader) uint32 { + return 64 - br.bit_pos_ +} + +/* Returns amount of unread bytes the bit reader still has buffered from the + BrotliInput, including whole bytes in br->val_. */ +func getRemainingBytes(br *bitReader) uint { + return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3)) +} + +/* Checks if there is at least |num| bytes left in the input ring-buffer + (excluding the bits remaining in br->val_). */ +func checkInputAmount(br *bitReader, num uint) bool { + return br.input_len-br.byte_pos >= num +} + +/* Guarantees that there are at least |n_bits| + 1 bits in accumulator. + Precondition: accumulator contains at least 1 bit. + |n_bits| should be in the range [1..24] for regular build. For portable + non-64-bit little-endian build only 16 bits are safe to request. */ +func fillBitWindow(br *bitReader, n_bits uint32) { + if br.bit_pos_ >= 32 { + br.val_ >>= 32 + br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */ + br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32 + br.byte_pos += 4 + } +} + +/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no + more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */ +func fillBitWindow16(br *bitReader) { + fillBitWindow(br, 17) +} + +/* Tries to pull one byte of input to accumulator. + Returns false if there is no input available. */ +func pullByte(br *bitReader) bool { + if br.byte_pos == br.input_len { + return false + } + + br.val_ >>= 8 + br.val_ |= (uint64(br.input[br.byte_pos])) << 56 + br.bit_pos_ -= 8 + br.byte_pos++ + return true +} + +/* Returns currently available bits. + The number of valid bits could be calculated by BrotliGetAvailableBits. */ +func getBitsUnmasked(br *bitReader) uint64 { + return br.val_ >> br.bit_pos_ +} + +/* Like BrotliGetBits, but does not mask the result. + The result contains at least 16 valid bits. */ +func get16BitsUnmasked(br *bitReader) uint32 { + fillBitWindow(br, 16) + return uint32(getBitsUnmasked(br)) +} + +/* Returns the specified number of bits from |br| without advancing bit + position. */ +func getBits(br *bitReader, n_bits uint32) uint32 { + fillBitWindow(br, n_bits) + return uint32(getBitsUnmasked(br)) & bitMask(n_bits) +} + +/* Tries to peek the specified amount of bits. Returns false, if there + is not enough input. */ +func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool { + for getAvailableBits(br) < n_bits { + if !pullByte(br) { + return false + } + } + + *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) + return true +} + +/* Advances the bit pos by |n_bits|. */ +func dropBits(br *bitReader, n_bits uint32) { + br.bit_pos_ += n_bits +} + +func bitReaderUnload(br *bitReader) { + var unused_bytes uint32 = getAvailableBits(br) >> 3 + var unused_bits uint32 = unused_bytes << 3 + br.byte_pos -= uint(unused_bytes) + if unused_bits == 64 { + br.val_ = 0 + } else { + br.val_ <<= unused_bits + } + + br.bit_pos_ += unused_bits +} + +/* Reads the specified number of bits from |br| and advances the bit pos. + Precondition: accumulator MUST contain at least |n_bits|. */ +func takeBits(br *bitReader, n_bits uint32, val *uint32) { + *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) + dropBits(br, n_bits) +} + +/* Reads the specified number of bits from |br| and advances the bit pos. + Assumes that there is enough input to perform BrotliFillBitWindow. */ +func readBits(br *bitReader, n_bits uint32) uint32 { + var val uint32 + fillBitWindow(br, n_bits) + takeBits(br, n_bits, &val) + return val +} + +/* Tries to read the specified amount of bits. Returns false, if there + is not enough input. |n_bits| MUST be positive. */ +func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool { + for getAvailableBits(br) < n_bits { + if !pullByte(br) { + return false + } + } + + takeBits(br, n_bits, val) + return true +} + +/* Advances the bit reader position to the next byte boundary and verifies + that any skipped bits are set to zero. */ +func bitReaderJumpToByteBoundary(br *bitReader) bool { + var pad_bits_count uint32 = getAvailableBits(br) & 0x7 + var pad_bits uint32 = 0 + if pad_bits_count != 0 { + takeBits(br, pad_bits_count, &pad_bits) + } + + return pad_bits == 0 +} + +/* Copies remaining input bytes stored in the bit reader to the output. Value + |num| may not be larger than BrotliGetRemainingBytes. The bit reader must be + warmed up again after this. */ +func copyBytes(dest []byte, br *bitReader, num uint) { + for getAvailableBits(br) >= 8 && num > 0 { + dest[0] = byte(getBitsUnmasked(br)) + dropBits(br, 8) + dest = dest[1:] + num-- + } + + copy(dest, br.input[br.byte_pos:][:num]) + br.byte_pos += num +} + +func initBitReader(br *bitReader) { + br.val_ = 0 + br.bit_pos_ = 64 +} + +func warmupBitReader(br *bitReader) bool { + /* Fixing alignment after unaligned BrotliFillWindow would result accumulator + overflow. If unalignment is caused by BrotliSafeReadBits, then there is + enough space in accumulator to fix alignment. */ + if getAvailableBits(br) == 0 { + if !pullByte(br) { + return false + } + } + + return true +} diff --git a/vendor/github.com/andybalholm/brotli/bitwriter.go b/vendor/github.com/andybalholm/brotli/bitwriter.go new file mode 100644 index 0000000000..dfc60360f3 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/bitwriter.go @@ -0,0 +1,56 @@ +package brotli + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Write bits into a byte array. */ + +type bitWriter struct { + dst []byte + + // Data waiting to be written is the low nbits of bits. + bits uint64 + nbits uint +} + +func (w *bitWriter) writeBits(nb uint, b uint64) { + w.bits |= b << w.nbits + w.nbits += nb + if w.nbits >= 32 { + bits := w.bits + w.bits >>= 32 + w.nbits -= 32 + w.dst = append(w.dst, + byte(bits), + byte(bits>>8), + byte(bits>>16), + byte(bits>>24), + ) + } +} + +func (w *bitWriter) writeSingleBit(bit bool) { + if bit { + w.writeBits(1, 1) + } else { + w.writeBits(1, 0) + } +} + +func (w *bitWriter) jumpToByteBoundary() { + dst := w.dst + for w.nbits != 0 { + dst = append(dst, byte(w.bits)) + w.bits >>= 8 + if w.nbits > 8 { // Avoid underflow + w.nbits -= 8 + } else { + w.nbits = 0 + } + } + w.bits = 0 + w.dst = dst +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter.go b/vendor/github.com/andybalholm/brotli/block_splitter.go new file mode 100644 index 0000000000..978a131474 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/block_splitter.go @@ -0,0 +1,144 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Block split point selection utilities. */ + +type blockSplit struct { + num_types uint + num_blocks uint + types []byte + lengths []uint32 + types_alloc_size uint + lengths_alloc_size uint +} + +const ( + kMaxLiteralHistograms uint = 100 + kMaxCommandHistograms uint = 50 + kLiteralBlockSwitchCost float64 = 28.1 + kCommandBlockSwitchCost float64 = 13.5 + kDistanceBlockSwitchCost float64 = 14.6 + kLiteralStrideLength uint = 70 + kCommandStrideLength uint = 40 + kSymbolsPerLiteralHistogram uint = 544 + kSymbolsPerCommandHistogram uint = 530 + kSymbolsPerDistanceHistogram uint = 544 + kMinLengthForBlockSplitting uint = 128 + kIterMulForRefining uint = 2 + kMinItersForRefining uint = 100 +) + +func countLiterals(cmds []command) uint { + var total_length uint = 0 + /* Count how many we have. */ + + for i := range cmds { + total_length += uint(cmds[i].insert_len_) + } + + return total_length +} + +func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) { + var pos uint = 0 + var from_pos uint = offset & mask + for i := range cmds { + var insert_len uint = uint(cmds[i].insert_len_) + if from_pos+insert_len > mask { + var head_size uint = mask + 1 - from_pos + copy(literals[pos:], data[from_pos:][:head_size]) + from_pos = 0 + pos += head_size + insert_len -= head_size + } + + if insert_len > 0 { + copy(literals[pos:], data[from_pos:][:insert_len]) + pos += insert_len + } + + from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask)) + } +} + +func myRand(seed *uint32) uint32 { + /* Initial seed should be 7. In this case, loop length is (1 << 29). */ + *seed *= 16807 + + return *seed +} + +func bitCost(count uint) float64 { + if count == 0 { + return -2.0 + } else { + return fastLog2(count) + } +} + +const histogramsPerBatch = 64 + +const clustersPerBatch = 16 + +func initBlockSplit(self *blockSplit) { + self.num_types = 0 + self.num_blocks = 0 + self.types = self.types[:0] + self.lengths = self.lengths[:0] + self.types_alloc_size = 0 + self.lengths_alloc_size = 0 +} + +func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { + { + var literals_count uint = countLiterals(cmds) + var literals []byte = make([]byte, literals_count) + + /* Create a continuous array of literals. */ + copyLiteralsToByteArray(cmds, data, pos, mask, literals) + + /* Create the block split on the array of literals. + Literal histograms have alphabet size 256. */ + splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split) + + literals = nil + } + { + var insert_and_copy_codes []uint16 = make([]uint16, len(cmds)) + /* Compute prefix codes for commands. */ + + for i := range cmds { + insert_and_copy_codes[i] = cmds[i].cmd_prefix_ + } + + /* Create the block split on the array of command prefixes. */ + splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) + + /* TODO: reuse for distances? */ + + insert_and_copy_codes = nil + } + { + var distance_prefixes []uint16 = make([]uint16, len(cmds)) + var j uint = 0 + /* Create a continuous array of distance prefixes. */ + + for i := range cmds { + var cmd *command = &cmds[i] + if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { + distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF + j++ + } + } + + /* Create the block split on the array of distance prefixes. */ + splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split) + + distance_prefixes = nil + } +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter_command.go b/vendor/github.com/andybalholm/brotli/block_splitter_command.go new file mode 100644 index 0000000000..9dec13e4d9 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/block_splitter_command.go @@ -0,0 +1,434 @@ +package brotli + +import "math" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { + var seed uint32 = 7 + var block_length uint = length / num_histograms + var i uint + clearHistogramsCommand(histograms, num_histograms) + for i = 0; i < num_histograms; i++ { + var pos uint = length * i / num_histograms + if i != 0 { + pos += uint(myRand(&seed) % uint32(block_length)) + } + + if pos+stride >= length { + pos = length - stride - 1 + } + + histogramAddVectorCommand(&histograms[i], data[pos:], stride) + } +} + +func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) { + var pos uint = 0 + if stride >= length { + stride = length + } else { + pos = uint(myRand(seed) % uint32(length-stride+1)) + } + + histogramAddVectorCommand(sample, data[pos:], stride) +} + +func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining + var seed uint32 = 7 + var iter uint + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms + for iter = 0; iter < iters; iter++ { + var sample histogramCommand + histogramClearCommand(&sample) + randomSampleCommand(&seed, data, length, stride, &sample) + histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample) + } +} + +/* Assigns a block id from the range [0, num_histograms) to each data element + in data[0..length) and fills in block_id[0..length) with the assigned values. + Returns the number of blocks, i.e. one plus the number of block switches. */ +func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { + var data_size uint = histogramDataSizeCommand() + var bitmaplen uint = (num_histograms + 7) >> 3 + var num_blocks uint = 1 + var i uint + var j uint + assert(num_histograms <= 256) + if num_histograms <= 1 { + for i = 0; i < length; i++ { + block_id[i] = 0 + } + + return 1 + } + + for i := 0; i < int(data_size*num_histograms); i++ { + insert_cost[i] = 0 + } + for i = 0; i < num_histograms; i++ { + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) + } + + for i = data_size; i != 0; { + i-- + for j = 0; j < num_histograms; j++ { + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) + } + } + + for i := 0; i < int(num_histograms); i++ { + cost[i] = 0 + } + for i := 0; i < int(length*bitmaplen); i++ { + switch_signal[i] = 0 + } + + /* After each iteration of this loop, cost[k] will contain the difference + between the minimum cost of arriving at the current byte position using + entropy code k, and the minimum cost of arriving at the current byte + position. This difference is capped at the block switch cost, and if it + reaches block switch cost, it means that when we trace back from the last + position, we need to switch here. */ + for i = 0; i < length; i++ { + var byte_ix uint = i + var ix uint = byte_ix * bitmaplen + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms + var min_cost float64 = 1e99 + var block_switch_cost float64 = block_switch_bitcost + var k uint + for k = 0; k < num_histograms; k++ { + /* We are coding the symbol in data[byte_ix] with entropy code k. */ + cost[k] += insert_cost[insert_cost_ix+k] + + if cost[k] < min_cost { + min_cost = cost[k] + block_id[byte_ix] = byte(k) + } + } + + /* More blocks for the beginning. */ + if byte_ix < 2000 { + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 + } + + for k = 0; k < num_histograms; k++ { + cost[k] -= min_cost + if cost[k] >= block_switch_cost { + var mask byte = byte(1 << (k & 7)) + cost[k] = block_switch_cost + assert(k>>3 < bitmaplen) + switch_signal[ix+(k>>3)] |= mask + /* Trace back from the last position and switch at the marked places. */ + } + } + } + { + var byte_ix uint = length - 1 + var ix uint = byte_ix * bitmaplen + var cur_id byte = block_id[byte_ix] + for byte_ix > 0 { + var mask byte = byte(1 << (cur_id & 7)) + assert(uint(cur_id)>>3 < bitmaplen) + byte_ix-- + ix -= bitmaplen + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { + if cur_id != block_id[byte_ix] { + cur_id = block_id[byte_ix] + num_blocks++ + } + } + + block_id[byte_ix] = cur_id + } + } + + return num_blocks +} + +var remapBlockIdsCommand_kInvalidId uint16 = 256 + +func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { + var next_id uint16 = 0 + var i uint + for i = 0; i < num_histograms; i++ { + new_id[i] = remapBlockIdsCommand_kInvalidId + } + + for i = 0; i < length; i++ { + assert(uint(block_ids[i]) < num_histograms) + if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId { + new_id[block_ids[i]] = next_id + next_id++ + } + } + + for i = 0; i < length; i++ { + block_ids[i] = byte(new_id[block_ids[i]]) + assert(uint(block_ids[i]) < num_histograms) + } + + assert(uint(next_id) <= num_histograms) + return uint(next_id) +} + +func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) { + var i uint + clearHistogramsCommand(histograms, num_histograms) + for i = 0; i < length; i++ { + histogramAddCommand(&histograms[block_ids[i]], uint(data[i])) + } +} + +var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32 + +func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { + var histogram_symbols []uint32 = make([]uint32, num_blocks) + var block_lengths []uint32 = make([]uint32, num_blocks) + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch + var all_histograms_size uint = 0 + var all_histograms_capacity uint = expected_num_clusters + var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity) + var cluster_size_size uint = 0 + var cluster_size_capacity uint = expected_num_clusters + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) + var num_clusters uint = 0 + var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch)) + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 + var pairs_capacity uint = max_num_pairs + 1 + var pairs []histogramPair = make([]histogramPair, pairs_capacity) + var pos uint = 0 + var clusters []uint32 + var num_final_clusters uint + var new_index []uint32 + var i uint + var sizes = [histogramsPerBatch]uint32{0} + var new_clusters = [histogramsPerBatch]uint32{0} + var symbols = [histogramsPerBatch]uint32{0} + var remap = [histogramsPerBatch]uint32{0} + + for i := 0; i < int(num_blocks); i++ { + block_lengths[i] = 0 + } + { + var block_idx uint = 0 + for i = 0; i < length; i++ { + assert(block_idx < num_blocks) + block_lengths[block_idx]++ + if i+1 == length || block_ids[i] != block_ids[i+1] { + block_idx++ + } + } + + assert(block_idx == num_blocks) + } + + for i = 0; i < num_blocks; i += histogramsPerBatch { + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) + var num_new_clusters uint + var j uint + for j = 0; j < num_to_combine; j++ { + var k uint + histogramClearCommand(&histograms[j]) + for k = 0; uint32(k) < block_lengths[i+j]; k++ { + histogramAddCommand(&histograms[j], uint(data[pos])) + pos++ + } + + histograms[j].bit_cost_ = populationCostCommand(&histograms[j]) + new_clusters[j] = uint32(j) + symbols[j] = uint32(j) + sizes[j] = 1 + } + + num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { + var _new_size uint + if all_histograms_capacity == 0 { + _new_size = all_histograms_size + num_new_clusters + } else { + _new_size = all_histograms_capacity + } + var new_array []histogramCommand + for _new_size < (all_histograms_size + num_new_clusters) { + _new_size *= 2 + } + new_array = make([]histogramCommand, _new_size) + if all_histograms_capacity != 0 { + copy(new_array, all_histograms[:all_histograms_capacity]) + } + + all_histograms = new_array + all_histograms_capacity = _new_size + } + + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) + for j = 0; j < num_new_clusters; j++ { + all_histograms[all_histograms_size] = histograms[new_clusters[j]] + all_histograms_size++ + cluster_size[cluster_size_size] = sizes[new_clusters[j]] + cluster_size_size++ + remap[new_clusters[j]] = uint32(j) + } + + for j = 0; j < num_to_combine; j++ { + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] + } + + num_clusters += num_new_clusters + assert(num_clusters == cluster_size_size) + assert(num_clusters == all_histograms_size) + } + + histograms = nil + + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) + if pairs_capacity < max_num_pairs+1 { + pairs = nil + pairs = make([]histogramPair, (max_num_pairs + 1)) + } + + clusters = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + clusters[i] = uint32(i) + } + + num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) + pairs = nil + cluster_size = nil + + new_index = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + new_index[i] = clusterBlocksCommand_kInvalidIndex + } + pos = 0 + { + var next_index uint32 = 0 + for i = 0; i < num_blocks; i++ { + var histo histogramCommand + var j uint + var best_out uint32 + var best_bits float64 + histogramClearCommand(&histo) + for j = 0; uint32(j) < block_lengths[i]; j++ { + histogramAddCommand(&histo, uint(data[pos])) + pos++ + } + + if i == 0 { + best_out = histogram_symbols[0] + } else { + best_out = histogram_symbols[i-1] + } + best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out]) + for j = 0; j < num_final_clusters; j++ { + var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]]) + if cur_bits < best_bits { + best_bits = cur_bits + best_out = clusters[j] + } + } + + histogram_symbols[i] = best_out + if new_index[best_out] == clusterBlocksCommand_kInvalidIndex { + new_index[best_out] = next_index + next_index++ + } + } + } + + clusters = nil + all_histograms = nil + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) + { + var cur_length uint32 = 0 + var block_idx uint = 0 + var max_type byte = 0 + for i = 0; i < num_blocks; i++ { + cur_length += block_lengths[i] + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { + var id byte = byte(new_index[histogram_symbols[i]]) + split.types[block_idx] = id + split.lengths[block_idx] = cur_length + max_type = brotli_max_uint8_t(max_type, id) + cur_length = 0 + block_idx++ + } + } + + split.num_blocks = block_idx + split.num_types = uint(max_type) + 1 + } + + new_index = nil + block_lengths = nil + histogram_symbols = nil +} + +func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + length := uint(len(data)) + var data_size uint = histogramDataSizeCommand() + var num_histograms uint = length/literals_per_histogram + 1 + var histograms []histogramCommand + if num_histograms > max_histograms { + num_histograms = max_histograms + } + + if length == 0 { + split.num_types = 1 + return + } else if length < kMinLengthForBlockSplitting { + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) + split.num_types = 1 + split.types[split.num_blocks] = 0 + split.lengths[split.num_blocks] = uint32(length) + split.num_blocks++ + return + } + + histograms = make([]histogramCommand, num_histograms) + + /* Find good entropy codes. */ + initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms) + + refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms) + { + var block_ids []byte = make([]byte, length) + var num_blocks uint = 0 + var bitmaplen uint = (num_histograms + 7) >> 3 + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) + var cost []float64 = make([]float64, num_histograms) + var switch_signal []byte = make([]byte, (length * bitmaplen)) + var new_id []uint16 = make([]uint16, num_histograms) + var iters uint + if params.quality < hqZopflificationQuality { + iters = 3 + } else { + iters = 10 + } + /* Find a good path through literals with the good entropy codes. */ + + var i uint + for i = 0; i < iters; i++ { + num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) + num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms) + buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms) + } + + insert_cost = nil + cost = nil + switch_signal = nil + new_id = nil + histograms = nil + clusterBlocksCommand(data, length, num_blocks, block_ids, split) + block_ids = nil + } +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter_distance.go b/vendor/github.com/andybalholm/brotli/block_splitter_distance.go new file mode 100644 index 0000000000..953530d518 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/block_splitter_distance.go @@ -0,0 +1,433 @@ +package brotli + +import "math" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { + var seed uint32 = 7 + var block_length uint = length / num_histograms + var i uint + clearHistogramsDistance(histograms, num_histograms) + for i = 0; i < num_histograms; i++ { + var pos uint = length * i / num_histograms + if i != 0 { + pos += uint(myRand(&seed) % uint32(block_length)) + } + + if pos+stride >= length { + pos = length - stride - 1 + } + + histogramAddVectorDistance(&histograms[i], data[pos:], stride) + } +} + +func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) { + var pos uint = 0 + if stride >= length { + stride = length + } else { + pos = uint(myRand(seed) % uint32(length-stride+1)) + } + + histogramAddVectorDistance(sample, data[pos:], stride) +} + +func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining + var seed uint32 = 7 + var iter uint + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms + for iter = 0; iter < iters; iter++ { + var sample histogramDistance + histogramClearDistance(&sample) + randomSampleDistance(&seed, data, length, stride, &sample) + histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample) + } +} + +/* Assigns a block id from the range [0, num_histograms) to each data element + in data[0..length) and fills in block_id[0..length) with the assigned values. + Returns the number of blocks, i.e. one plus the number of block switches. */ +func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { + var data_size uint = histogramDataSizeDistance() + var bitmaplen uint = (num_histograms + 7) >> 3 + var num_blocks uint = 1 + var i uint + var j uint + assert(num_histograms <= 256) + if num_histograms <= 1 { + for i = 0; i < length; i++ { + block_id[i] = 0 + } + + return 1 + } + + for i := 0; i < int(data_size*num_histograms); i++ { + insert_cost[i] = 0 + } + for i = 0; i < num_histograms; i++ { + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) + } + + for i = data_size; i != 0; { + i-- + for j = 0; j < num_histograms; j++ { + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) + } + } + + for i := 0; i < int(num_histograms); i++ { + cost[i] = 0 + } + for i := 0; i < int(length*bitmaplen); i++ { + switch_signal[i] = 0 + } + + /* After each iteration of this loop, cost[k] will contain the difference + between the minimum cost of arriving at the current byte position using + entropy code k, and the minimum cost of arriving at the current byte + position. This difference is capped at the block switch cost, and if it + reaches block switch cost, it means that when we trace back from the last + position, we need to switch here. */ + for i = 0; i < length; i++ { + var byte_ix uint = i + var ix uint = byte_ix * bitmaplen + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms + var min_cost float64 = 1e99 + var block_switch_cost float64 = block_switch_bitcost + var k uint + for k = 0; k < num_histograms; k++ { + /* We are coding the symbol in data[byte_ix] with entropy code k. */ + cost[k] += insert_cost[insert_cost_ix+k] + + if cost[k] < min_cost { + min_cost = cost[k] + block_id[byte_ix] = byte(k) + } + } + + /* More blocks for the beginning. */ + if byte_ix < 2000 { + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 + } + + for k = 0; k < num_histograms; k++ { + cost[k] -= min_cost + if cost[k] >= block_switch_cost { + var mask byte = byte(1 << (k & 7)) + cost[k] = block_switch_cost + assert(k>>3 < bitmaplen) + switch_signal[ix+(k>>3)] |= mask + /* Trace back from the last position and switch at the marked places. */ + } + } + } + { + var byte_ix uint = length - 1 + var ix uint = byte_ix * bitmaplen + var cur_id byte = block_id[byte_ix] + for byte_ix > 0 { + var mask byte = byte(1 << (cur_id & 7)) + assert(uint(cur_id)>>3 < bitmaplen) + byte_ix-- + ix -= bitmaplen + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { + if cur_id != block_id[byte_ix] { + cur_id = block_id[byte_ix] + num_blocks++ + } + } + + block_id[byte_ix] = cur_id + } + } + + return num_blocks +} + +var remapBlockIdsDistance_kInvalidId uint16 = 256 + +func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { + var next_id uint16 = 0 + var i uint + for i = 0; i < num_histograms; i++ { + new_id[i] = remapBlockIdsDistance_kInvalidId + } + + for i = 0; i < length; i++ { + assert(uint(block_ids[i]) < num_histograms) + if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId { + new_id[block_ids[i]] = next_id + next_id++ + } + } + + for i = 0; i < length; i++ { + block_ids[i] = byte(new_id[block_ids[i]]) + assert(uint(block_ids[i]) < num_histograms) + } + + assert(uint(next_id) <= num_histograms) + return uint(next_id) +} + +func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) { + var i uint + clearHistogramsDistance(histograms, num_histograms) + for i = 0; i < length; i++ { + histogramAddDistance(&histograms[block_ids[i]], uint(data[i])) + } +} + +var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32 + +func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { + var histogram_symbols []uint32 = make([]uint32, num_blocks) + var block_lengths []uint32 = make([]uint32, num_blocks) + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch + var all_histograms_size uint = 0 + var all_histograms_capacity uint = expected_num_clusters + var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity) + var cluster_size_size uint = 0 + var cluster_size_capacity uint = expected_num_clusters + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) + var num_clusters uint = 0 + var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch)) + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 + var pairs_capacity uint = max_num_pairs + 1 + var pairs []histogramPair = make([]histogramPair, pairs_capacity) + var pos uint = 0 + var clusters []uint32 + var num_final_clusters uint + var new_index []uint32 + var i uint + var sizes = [histogramsPerBatch]uint32{0} + var new_clusters = [histogramsPerBatch]uint32{0} + var symbols = [histogramsPerBatch]uint32{0} + var remap = [histogramsPerBatch]uint32{0} + + for i := 0; i < int(num_blocks); i++ { + block_lengths[i] = 0 + } + { + var block_idx uint = 0 + for i = 0; i < length; i++ { + assert(block_idx < num_blocks) + block_lengths[block_idx]++ + if i+1 == length || block_ids[i] != block_ids[i+1] { + block_idx++ + } + } + + assert(block_idx == num_blocks) + } + + for i = 0; i < num_blocks; i += histogramsPerBatch { + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) + var num_new_clusters uint + var j uint + for j = 0; j < num_to_combine; j++ { + var k uint + histogramClearDistance(&histograms[j]) + for k = 0; uint32(k) < block_lengths[i+j]; k++ { + histogramAddDistance(&histograms[j], uint(data[pos])) + pos++ + } + + histograms[j].bit_cost_ = populationCostDistance(&histograms[j]) + new_clusters[j] = uint32(j) + symbols[j] = uint32(j) + sizes[j] = 1 + } + + num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { + var _new_size uint + if all_histograms_capacity == 0 { + _new_size = all_histograms_size + num_new_clusters + } else { + _new_size = all_histograms_capacity + } + var new_array []histogramDistance + for _new_size < (all_histograms_size + num_new_clusters) { + _new_size *= 2 + } + new_array = make([]histogramDistance, _new_size) + if all_histograms_capacity != 0 { + copy(new_array, all_histograms[:all_histograms_capacity]) + } + + all_histograms = new_array + all_histograms_capacity = _new_size + } + + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) + for j = 0; j < num_new_clusters; j++ { + all_histograms[all_histograms_size] = histograms[new_clusters[j]] + all_histograms_size++ + cluster_size[cluster_size_size] = sizes[new_clusters[j]] + cluster_size_size++ + remap[new_clusters[j]] = uint32(j) + } + + for j = 0; j < num_to_combine; j++ { + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] + } + + num_clusters += num_new_clusters + assert(num_clusters == cluster_size_size) + assert(num_clusters == all_histograms_size) + } + + histograms = nil + + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) + if pairs_capacity < max_num_pairs+1 { + pairs = nil + pairs = make([]histogramPair, (max_num_pairs + 1)) + } + + clusters = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + clusters[i] = uint32(i) + } + + num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) + pairs = nil + cluster_size = nil + + new_index = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + new_index[i] = clusterBlocksDistance_kInvalidIndex + } + pos = 0 + { + var next_index uint32 = 0 + for i = 0; i < num_blocks; i++ { + var histo histogramDistance + var j uint + var best_out uint32 + var best_bits float64 + histogramClearDistance(&histo) + for j = 0; uint32(j) < block_lengths[i]; j++ { + histogramAddDistance(&histo, uint(data[pos])) + pos++ + } + + if i == 0 { + best_out = histogram_symbols[0] + } else { + best_out = histogram_symbols[i-1] + } + best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out]) + for j = 0; j < num_final_clusters; j++ { + var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]]) + if cur_bits < best_bits { + best_bits = cur_bits + best_out = clusters[j] + } + } + + histogram_symbols[i] = best_out + if new_index[best_out] == clusterBlocksDistance_kInvalidIndex { + new_index[best_out] = next_index + next_index++ + } + } + } + + clusters = nil + all_histograms = nil + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) + { + var cur_length uint32 = 0 + var block_idx uint = 0 + var max_type byte = 0 + for i = 0; i < num_blocks; i++ { + cur_length += block_lengths[i] + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { + var id byte = byte(new_index[histogram_symbols[i]]) + split.types[block_idx] = id + split.lengths[block_idx] = cur_length + max_type = brotli_max_uint8_t(max_type, id) + cur_length = 0 + block_idx++ + } + } + + split.num_blocks = block_idx + split.num_types = uint(max_type) + 1 + } + + new_index = nil + block_lengths = nil + histogram_symbols = nil +} + +func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + var data_size uint = histogramDataSizeDistance() + var num_histograms uint = length/literals_per_histogram + 1 + var histograms []histogramDistance + if num_histograms > max_histograms { + num_histograms = max_histograms + } + + if length == 0 { + split.num_types = 1 + return + } else if length < kMinLengthForBlockSplitting { + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) + split.num_types = 1 + split.types[split.num_blocks] = 0 + split.lengths[split.num_blocks] = uint32(length) + split.num_blocks++ + return + } + + histograms = make([]histogramDistance, num_histograms) + + /* Find good entropy codes. */ + initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms) + + refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms) + { + var block_ids []byte = make([]byte, length) + var num_blocks uint = 0 + var bitmaplen uint = (num_histograms + 7) >> 3 + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) + var cost []float64 = make([]float64, num_histograms) + var switch_signal []byte = make([]byte, (length * bitmaplen)) + var new_id []uint16 = make([]uint16, num_histograms) + var iters uint + if params.quality < hqZopflificationQuality { + iters = 3 + } else { + iters = 10 + } + /* Find a good path through literals with the good entropy codes. */ + + var i uint + for i = 0; i < iters; i++ { + num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) + num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms) + buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms) + } + + insert_cost = nil + cost = nil + switch_signal = nil + new_id = nil + histograms = nil + clusterBlocksDistance(data, length, num_blocks, block_ids, split) + block_ids = nil + } +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter_literal.go b/vendor/github.com/andybalholm/brotli/block_splitter_literal.go new file mode 100644 index 0000000000..1c895cf388 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/block_splitter_literal.go @@ -0,0 +1,433 @@ +package brotli + +import "math" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { + var seed uint32 = 7 + var block_length uint = length / num_histograms + var i uint + clearHistogramsLiteral(histograms, num_histograms) + for i = 0; i < num_histograms; i++ { + var pos uint = length * i / num_histograms + if i != 0 { + pos += uint(myRand(&seed) % uint32(block_length)) + } + + if pos+stride >= length { + pos = length - stride - 1 + } + + histogramAddVectorLiteral(&histograms[i], data[pos:], stride) + } +} + +func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) { + var pos uint = 0 + if stride >= length { + stride = length + } else { + pos = uint(myRand(seed) % uint32(length-stride+1)) + } + + histogramAddVectorLiteral(sample, data[pos:], stride) +} + +func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining + var seed uint32 = 7 + var iter uint + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms + for iter = 0; iter < iters; iter++ { + var sample histogramLiteral + histogramClearLiteral(&sample) + randomSampleLiteral(&seed, data, length, stride, &sample) + histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample) + } +} + +/* Assigns a block id from the range [0, num_histograms) to each data element + in data[0..length) and fills in block_id[0..length) with the assigned values. + Returns the number of blocks, i.e. one plus the number of block switches. */ +func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { + var data_size uint = histogramDataSizeLiteral() + var bitmaplen uint = (num_histograms + 7) >> 3 + var num_blocks uint = 1 + var i uint + var j uint + assert(num_histograms <= 256) + if num_histograms <= 1 { + for i = 0; i < length; i++ { + block_id[i] = 0 + } + + return 1 + } + + for i := 0; i < int(data_size*num_histograms); i++ { + insert_cost[i] = 0 + } + for i = 0; i < num_histograms; i++ { + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) + } + + for i = data_size; i != 0; { + i-- + for j = 0; j < num_histograms; j++ { + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) + } + } + + for i := 0; i < int(num_histograms); i++ { + cost[i] = 0 + } + for i := 0; i < int(length*bitmaplen); i++ { + switch_signal[i] = 0 + } + + /* After each iteration of this loop, cost[k] will contain the difference + between the minimum cost of arriving at the current byte position using + entropy code k, and the minimum cost of arriving at the current byte + position. This difference is capped at the block switch cost, and if it + reaches block switch cost, it means that when we trace back from the last + position, we need to switch here. */ + for i = 0; i < length; i++ { + var byte_ix uint = i + var ix uint = byte_ix * bitmaplen + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms + var min_cost float64 = 1e99 + var block_switch_cost float64 = block_switch_bitcost + var k uint + for k = 0; k < num_histograms; k++ { + /* We are coding the symbol in data[byte_ix] with entropy code k. */ + cost[k] += insert_cost[insert_cost_ix+k] + + if cost[k] < min_cost { + min_cost = cost[k] + block_id[byte_ix] = byte(k) + } + } + + /* More blocks for the beginning. */ + if byte_ix < 2000 { + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 + } + + for k = 0; k < num_histograms; k++ { + cost[k] -= min_cost + if cost[k] >= block_switch_cost { + var mask byte = byte(1 << (k & 7)) + cost[k] = block_switch_cost + assert(k>>3 < bitmaplen) + switch_signal[ix+(k>>3)] |= mask + /* Trace back from the last position and switch at the marked places. */ + } + } + } + { + var byte_ix uint = length - 1 + var ix uint = byte_ix * bitmaplen + var cur_id byte = block_id[byte_ix] + for byte_ix > 0 { + var mask byte = byte(1 << (cur_id & 7)) + assert(uint(cur_id)>>3 < bitmaplen) + byte_ix-- + ix -= bitmaplen + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { + if cur_id != block_id[byte_ix] { + cur_id = block_id[byte_ix] + num_blocks++ + } + } + + block_id[byte_ix] = cur_id + } + } + + return num_blocks +} + +var remapBlockIdsLiteral_kInvalidId uint16 = 256 + +func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { + var next_id uint16 = 0 + var i uint + for i = 0; i < num_histograms; i++ { + new_id[i] = remapBlockIdsLiteral_kInvalidId + } + + for i = 0; i < length; i++ { + assert(uint(block_ids[i]) < num_histograms) + if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId { + new_id[block_ids[i]] = next_id + next_id++ + } + } + + for i = 0; i < length; i++ { + block_ids[i] = byte(new_id[block_ids[i]]) + assert(uint(block_ids[i]) < num_histograms) + } + + assert(uint(next_id) <= num_histograms) + return uint(next_id) +} + +func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) { + var i uint + clearHistogramsLiteral(histograms, num_histograms) + for i = 0; i < length; i++ { + histogramAddLiteral(&histograms[block_ids[i]], uint(data[i])) + } +} + +var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32 + +func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { + var histogram_symbols []uint32 = make([]uint32, num_blocks) + var block_lengths []uint32 = make([]uint32, num_blocks) + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch + var all_histograms_size uint = 0 + var all_histograms_capacity uint = expected_num_clusters + var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity) + var cluster_size_size uint = 0 + var cluster_size_capacity uint = expected_num_clusters + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) + var num_clusters uint = 0 + var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch)) + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 + var pairs_capacity uint = max_num_pairs + 1 + var pairs []histogramPair = make([]histogramPair, pairs_capacity) + var pos uint = 0 + var clusters []uint32 + var num_final_clusters uint + var new_index []uint32 + var i uint + var sizes = [histogramsPerBatch]uint32{0} + var new_clusters = [histogramsPerBatch]uint32{0} + var symbols = [histogramsPerBatch]uint32{0} + var remap = [histogramsPerBatch]uint32{0} + + for i := 0; i < int(num_blocks); i++ { + block_lengths[i] = 0 + } + { + var block_idx uint = 0 + for i = 0; i < length; i++ { + assert(block_idx < num_blocks) + block_lengths[block_idx]++ + if i+1 == length || block_ids[i] != block_ids[i+1] { + block_idx++ + } + } + + assert(block_idx == num_blocks) + } + + for i = 0; i < num_blocks; i += histogramsPerBatch { + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) + var num_new_clusters uint + var j uint + for j = 0; j < num_to_combine; j++ { + var k uint + histogramClearLiteral(&histograms[j]) + for k = 0; uint32(k) < block_lengths[i+j]; k++ { + histogramAddLiteral(&histograms[j], uint(data[pos])) + pos++ + } + + histograms[j].bit_cost_ = populationCostLiteral(&histograms[j]) + new_clusters[j] = uint32(j) + symbols[j] = uint32(j) + sizes[j] = 1 + } + + num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { + var _new_size uint + if all_histograms_capacity == 0 { + _new_size = all_histograms_size + num_new_clusters + } else { + _new_size = all_histograms_capacity + } + var new_array []histogramLiteral + for _new_size < (all_histograms_size + num_new_clusters) { + _new_size *= 2 + } + new_array = make([]histogramLiteral, _new_size) + if all_histograms_capacity != 0 { + copy(new_array, all_histograms[:all_histograms_capacity]) + } + + all_histograms = new_array + all_histograms_capacity = _new_size + } + + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) + for j = 0; j < num_new_clusters; j++ { + all_histograms[all_histograms_size] = histograms[new_clusters[j]] + all_histograms_size++ + cluster_size[cluster_size_size] = sizes[new_clusters[j]] + cluster_size_size++ + remap[new_clusters[j]] = uint32(j) + } + + for j = 0; j < num_to_combine; j++ { + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] + } + + num_clusters += num_new_clusters + assert(num_clusters == cluster_size_size) + assert(num_clusters == all_histograms_size) + } + + histograms = nil + + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) + if pairs_capacity < max_num_pairs+1 { + pairs = nil + pairs = make([]histogramPair, (max_num_pairs + 1)) + } + + clusters = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + clusters[i] = uint32(i) + } + + num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) + pairs = nil + cluster_size = nil + + new_index = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + new_index[i] = clusterBlocksLiteral_kInvalidIndex + } + pos = 0 + { + var next_index uint32 = 0 + for i = 0; i < num_blocks; i++ { + var histo histogramLiteral + var j uint + var best_out uint32 + var best_bits float64 + histogramClearLiteral(&histo) + for j = 0; uint32(j) < block_lengths[i]; j++ { + histogramAddLiteral(&histo, uint(data[pos])) + pos++ + } + + if i == 0 { + best_out = histogram_symbols[0] + } else { + best_out = histogram_symbols[i-1] + } + best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out]) + for j = 0; j < num_final_clusters; j++ { + var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]]) + if cur_bits < best_bits { + best_bits = cur_bits + best_out = clusters[j] + } + } + + histogram_symbols[i] = best_out + if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex { + new_index[best_out] = next_index + next_index++ + } + } + } + + clusters = nil + all_histograms = nil + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) + { + var cur_length uint32 = 0 + var block_idx uint = 0 + var max_type byte = 0 + for i = 0; i < num_blocks; i++ { + cur_length += block_lengths[i] + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { + var id byte = byte(new_index[histogram_symbols[i]]) + split.types[block_idx] = id + split.lengths[block_idx] = cur_length + max_type = brotli_max_uint8_t(max_type, id) + cur_length = 0 + block_idx++ + } + } + + split.num_blocks = block_idx + split.num_types = uint(max_type) + 1 + } + + new_index = nil + block_lengths = nil + histogram_symbols = nil +} + +func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + var data_size uint = histogramDataSizeLiteral() + var num_histograms uint = length/literals_per_histogram + 1 + var histograms []histogramLiteral + if num_histograms > max_histograms { + num_histograms = max_histograms + } + + if length == 0 { + split.num_types = 1 + return + } else if length < kMinLengthForBlockSplitting { + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) + split.num_types = 1 + split.types[split.num_blocks] = 0 + split.lengths[split.num_blocks] = uint32(length) + split.num_blocks++ + return + } + + histograms = make([]histogramLiteral, num_histograms) + + /* Find good entropy codes. */ + initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms) + + refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms) + { + var block_ids []byte = make([]byte, length) + var num_blocks uint = 0 + var bitmaplen uint = (num_histograms + 7) >> 3 + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) + var cost []float64 = make([]float64, num_histograms) + var switch_signal []byte = make([]byte, (length * bitmaplen)) + var new_id []uint16 = make([]uint16, num_histograms) + var iters uint + if params.quality < hqZopflificationQuality { + iters = 3 + } else { + iters = 10 + } + /* Find a good path through literals with the good entropy codes. */ + + var i uint + for i = 0; i < iters; i++ { + num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) + num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms) + buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms) + } + + insert_cost = nil + cost = nil + switch_signal = nil + new_id = nil + histograms = nil + clusterBlocksLiteral(data, length, num_blocks, block_ids, split) + block_ids = nil + } +} diff --git a/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go b/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go new file mode 100644 index 0000000000..ee6552982b --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go @@ -0,0 +1,1539 @@ +package brotli + +import ( + "math" + "sync" +) + +const maxHuffmanTreeSize = (2*numCommandSymbols + 1) + +/* +The maximum size of Huffman dictionary for distances assuming that + + NPOSTFIX = 0 and NDIRECT = 0. +*/ +const maxSimpleDistanceAlphabetSize = 140 + +/* +Represents the range of values belonging to a prefix code: + + [offset, offset + 2^nbits) +*/ +type prefixCodeRange struct { + offset uint32 + nbits uint32 +} + +var kBlockLengthPrefixCode = [numBlockLenSymbols]prefixCodeRange{ + prefixCodeRange{1, 2}, + prefixCodeRange{5, 2}, + prefixCodeRange{9, 2}, + prefixCodeRange{13, 2}, + prefixCodeRange{17, 3}, + prefixCodeRange{25, 3}, + prefixCodeRange{33, 3}, + prefixCodeRange{41, 3}, + prefixCodeRange{49, 4}, + prefixCodeRange{65, 4}, + prefixCodeRange{81, 4}, + prefixCodeRange{97, 4}, + prefixCodeRange{113, 5}, + prefixCodeRange{145, 5}, + prefixCodeRange{177, 5}, + prefixCodeRange{209, 5}, + prefixCodeRange{241, 6}, + prefixCodeRange{305, 6}, + prefixCodeRange{369, 7}, + prefixCodeRange{497, 8}, + prefixCodeRange{753, 9}, + prefixCodeRange{1265, 10}, + prefixCodeRange{2289, 11}, + prefixCodeRange{4337, 12}, + prefixCodeRange{8433, 13}, + prefixCodeRange{16625, 24}, +} + +func blockLengthPrefixCode(len uint32) uint32 { + var code uint32 + if len >= 177 { + if len >= 753 { + code = 20 + } else { + code = 14 + } + } else if len >= 41 { + code = 7 + } else { + code = 0 + } + for code < (numBlockLenSymbols-1) && len >= kBlockLengthPrefixCode[code+1].offset { + code++ + } + return code +} + +func getBlockLengthPrefixCode(len uint32, code *uint, n_extra *uint32, extra *uint32) { + *code = uint(blockLengthPrefixCode(uint32(len))) + *n_extra = kBlockLengthPrefixCode[*code].nbits + *extra = len - kBlockLengthPrefixCode[*code].offset +} + +type blockTypeCodeCalculator struct { + last_type uint + second_last_type uint +} + +func initBlockTypeCodeCalculator(self *blockTypeCodeCalculator) { + self.last_type = 1 + self.second_last_type = 0 +} + +func nextBlockTypeCode(calculator *blockTypeCodeCalculator, type_ byte) uint { + var type_code uint + if uint(type_) == calculator.last_type+1 { + type_code = 1 + } else if uint(type_) == calculator.second_last_type { + type_code = 0 + } else { + type_code = uint(type_) + 2 + } + calculator.second_last_type = calculator.last_type + calculator.last_type = uint(type_) + return type_code +} + +/* +|nibblesbits| represents the 2 bits to encode MNIBBLES (0-3) + + REQUIRES: length > 0 + REQUIRES: length <= (1 << 24) +*/ +func encodeMlen(length uint, bits *uint64, numbits *uint, nibblesbits *uint64) { + var lg uint + if length == 1 { + lg = 1 + } else { + lg = uint(log2FloorNonZero(uint(uint32(length-1)))) + 1 + } + var tmp uint + if lg < 16 { + tmp = 16 + } else { + tmp = (lg + 3) + } + var mnibbles uint = tmp / 4 + assert(length > 0) + assert(length <= 1<<24) + assert(lg <= 24) + *nibblesbits = uint64(mnibbles) - 4 + *numbits = mnibbles * 4 + *bits = uint64(length) - 1 +} + +func storeCommandExtra(cmd *command, storage_ix *uint, storage []byte) { + var copylen_code uint32 = commandCopyLenCode(cmd) + var inscode uint16 = getInsertLengthCode(uint(cmd.insert_len_)) + var copycode uint16 = getCopyLengthCode(uint(copylen_code)) + var insnumextra uint32 = getInsertExtra(inscode) + var insextraval uint64 = uint64(cmd.insert_len_) - uint64(getInsertBase(inscode)) + var copyextraval uint64 = uint64(copylen_code) - uint64(getCopyBase(copycode)) + var bits uint64 = copyextraval<