From 6674d34fbc86835edf7245c5a18e643516b48fd7 Mon Sep 17 00:00:00 2001 From: mejgun Date: Mon, 16 Sep 2024 13:38:04 +0300 Subject: [PATCH 01/29] example config move from readme --- README.md | 90 +++----------------------------------------- config.default.json | 29 -------------- config.example.jsonc | 76 +++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 113 deletions(-) delete mode 100644 config.default.json create mode 100644 config.example.jsonc diff --git a/README.md b/README.md index 9c36f91..bacc1a8 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ ### What is this repository for? ### -This is part of another project: https://github.com/mesb1/xupnpd_youtube +This is yt-dlp based video restreamer, part of another project: https://github.com/mesb1/xupnpd_youtube ### Build ### -`cd src && go build` +``` +cd src +go build +``` ### Options ### @@ -18,85 +21,4 @@ Run with `--help` - 3 - extractor create error - 4 - streamer create error - 5 - web server error - - 6 - links cache error - -### Config explained ### -do not copypaste, comments are not allowed in this json. -use config.default.json instead. - -```jsonc -{ - // web server listen port - "port": 8080, - // restreamer config - "streamer": { - // show errors in headers (insecure) - "error-headers": false, - // do not strictly check video headers - "ignore-missing-headers": false, - // do not check video server certificate (insecure) - "ignore-ssl-errors": false, - // video file that will be shown on errors - "error-video": "corrupted.mp4", - // audio file that will be played on errors - // dwnlded here youtu.be/_b8KPiT1PxI (suggest your options) - "error-audio": "failed.m4a", - // how to set streamer's user-agent - // request - set from user's request (old default) - // extractor - set from extractor on app start (default) - // config - set from config - "set-user-agent": "extractor", - // custom user agent used if "set-user-agent" set to "config" - "user-agent": "Mozilla", - // proxy for restreamer - // empty - no proxy - // "env" - read proxy from environment variables (e.g. HTTP_PROXY="http://127.0.0.1:3128") - // proxy url - e.g. "socks5://127.0.0.1:9999" - "proxy": "env", - // min TLS version: "TLS 1.3", "TLS 1.2", etc. - "min-tls-version": "TLS 1.2 " - }, - // media extractor config - "extractor": { - // file path - "path": "yt-dlp", - // arguments for extractor - // args separator is ",,", not space - // {{.HEIGHT}} will be replaced with requested height (360/480/720) - // {{.URL}} will be replace with requested url - // also you can use {{.FORMAT}} - requested format (now - only mp4 or m4a) - "mp4": "-f,,(mp4)[height<={{.HEIGHT}}],,-g,,{{.URL}}", - // same for m4a - "m4a": "-f,,(m4a),,-g,,{{.URL}}", - // args for getting user-agent - "get-user-agent": "--dump-user-agent", - // custom options list to extractor, like proxy, etc. - // same rules as mp4/m4a - // HEIGHT/URL/.. templates also can be used - "custom-options": [ - "--option1,,value1", - "--option2", - "value2", - "--option3", - "very long value 3" - ] - }, - // logger config - "log": { - // log level - // debug/info/warning/error/nothing - "level": "info", - // log destination - // stdout/file/both - "output": "stdout", - // filename if writing to file - "filename": "log.txt" - }, - // links cache config - "cache": { - // links expire time - // time units are "s", "m", "h", e.g. "1h10m10s", "10h", "1s" - // "0s" will disable cache - "expire-time": "3h" - } -} + - 6 - links cache error \ No newline at end of file diff --git a/config.default.json b/config.default.json deleted file mode 100644 index 22b7e7d..0000000 --- a/config.default.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "port": 8080, - "streamer": { - "error-headers": false, - "ignore-missing-headers": false, - "ignore-ssl-errors": false, - "error-video": "corrupted.mp4", - "error-audio": "failed.m4a", - "set-user-agent": "extractor", - "user-agent": "Mozilla", - "proxy": "env", - "min-tls-version": "TLS 1.2" - }, - "extractor": { - "path": "yt-dlp", - "mp4": "-f,,(mp4)[height<={{.HEIGHT}}],,-g,,{{.URL}}", - "m4a": "-f,,(m4a),,-g,,{{.URL}}", - "get-user-agent": "--dump-user-agent", - "custom-options": [] - }, - "log": { - "level": "info", - "output": "stdout", - "filename": "log.txt" - }, - "cache": { - "expire-time": "3h" - } -} \ No newline at end of file diff --git a/config.example.jsonc b/config.example.jsonc new file mode 100644 index 0000000..4fe0d28 --- /dev/null +++ b/config.example.jsonc @@ -0,0 +1,76 @@ +{ + // web server listen port + "port": 8080, + // restreamer config + "streamer": { + // show errors in headers (insecure) + "error-headers": false, + // do not strictly check video headers + "ignore-missing-headers": false, + // do not check video server certificate (insecure) + "ignore-ssl-errors": false, + // video file that will be shown on errors + "error-video": "corrupted.mp4", + // audio file that will be played on errors + // dwnlded here youtu.be/_b8KPiT1PxI (suggest your options) + "error-audio": "failed.m4a", + // how to set streamer's user-agent + // request - set from user's request (old default) + // extractor - set from extractor on app start (default) + // config - set from config + "set-user-agent": "extractor", + // custom user agent used if "set-user-agent" set to "config" + "user-agent": "Mozilla", + // proxy for restreamer + // empty - no proxy + // "env" - read proxy from environment variables (e.g. HTTP_PROXY="http://127.0.0.1:3128") + // proxy url - e.g. "socks5://127.0.0.1:9999" + "proxy": "env", + // min TLS version: "TLS 1.3", "TLS 1.2", etc. + "min-tls-version": "TLS 1.2" + }, + // media extractor config + "extractor": { + // file path + "path": "yt-dlp", + // arguments for extractor + // args separator is ",,", not space + // {{.HEIGHT}} will be replaced with requested height (360/480/720) + // {{.URL}} will be replace with requested url + // also you can use {{.FORMAT}} - requested format (now - only mp4 or m4a) + "mp4": "-f,,(mp4)[height<={{.HEIGHT}}],,-g,,{{.URL}}", + // same for m4a + "m4a": "-f,,(m4a),,-g,,{{.URL}}", + // args for getting user-agent + "get-user-agent": "--dump-user-agent", + // custom options list to extractor, like proxy, etc. + // same rules as mp4/m4a + // HEIGHT/URL/.. templates also can be used + // "custom-options": [] + "custom-options": [ + "--option1,,value1", + "--option2", + "value2", + "--option3", + "very long value 3" + ] + }, + // logger config + "log": { + // log level + // debug/info/warning/error/nothing + "level": "info", + // log destination + // stdout/file/both + "output": "stdout", + // filename if writing to file + "filename": "log.txt" + }, + // links cache config + "cache": { + // links expire time + // time units are "s", "m", "h", e.g. "1h10m10s", "10h", "1s" + // "0s" will disable cache + "expire-time": "3h" + } +} \ No newline at end of file From 087e062b6c31e12ff6f702c21f5dbd73358b255b Mon Sep 17 00:00:00 2001 From: mejgun Date: Mon, 16 Sep 2024 13:54:06 +0300 Subject: [PATCH 02/29] build script rework --- .gitignore | 1 - README.md | 5 +---- build.go | 10 +++++----- build.sh | 17 +++++++++-------- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 1f8b8aa..ce68f86 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ bin/ -all.md5 ytproxy log.txt config.json diff --git a/README.md b/README.md index bacc1a8..1f55e3e 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,7 @@ This is yt-dlp based video restreamer, part of another project: https://github.c ### Build ### -``` -cd src -go build -``` +`cd src && go build` ### Options ### diff --git a/build.go b/build.go index 9357999..a5582fa 100644 --- a/build.go +++ b/build.go @@ -9,15 +9,15 @@ import ( ) const ( - binDir = "bin" - filePrefix = "yt-proxy" - goBin = "go" - myos = "linux" - myarch = "amd64" + binDir = "bin" + goBin = "go" + myos = "linux" + myarch = "amd64" ) func main() { total := len(knownArch) * len(knownOS) + filePrefix := os.Args[1] for os_ := range knownOS { for arch := range knownArch { file := fmt.Sprintf("%s-%s-%s", filePrefix, os_, arch) diff --git a/build.sh b/build.sh index f087f98..49237e5 100755 --- a/build.sh +++ b/build.sh @@ -1,15 +1,16 @@ #!/bin/bash +set -e + BinDir="bin" Md5File="all.md5" +FilePrefix="yt-proxy" date >${Md5File} -mkdir -p ${BinDir} || exit -rm ${BinDir}/${FilePrefix}* -rf || exit -cd src || exit -go run ../build.go || exit -cd ../$BinDir || exit -for i in $(ls -1 *); do - md5sum $i >>../${Md5File} || exit -done +mkdir -p ${BinDir} +rm ${BinDir}/${FilePrefix}* -rf +cd src +go run ../build.go ${FilePrefix} +cd ../$BinDir +md5sum -b ${FilePrefix}* >${Md5File} cd .. From 417cdf28f16084b824c0160a5d6bcc61e2e574a9 Mon Sep 17 00:00:00 2001 From: mejgun Date: Mon, 16 Sep 2024 17:52:22 +0300 Subject: [PATCH 03/29] reworking config 1/4 --- src/config/config.go | 64 +++++++++++++++++++++++++++++++++++++++- src/streamer/streamer.go | 49 +++++++++++++++--------------- 2 files changed, 89 insertions(+), 24 deletions(-) diff --git a/src/config/config.go b/src/config/config.go index c612cc4..444cb81 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -19,6 +19,68 @@ type configT struct { Cache cache.ConfigT `json:"cache"` } +func defaultConfig() configT { + fls := false + ext := streamer.Extractor + tv := streamer.TlsVersion(0) + var s = [4]string{"corrupted.mp4", + "failed.m4a", + "Mozilla", + "env", + } + return configT{ + PortInt: 8080, + Streamer: streamer.ConfigT{ + EnableErrorHeaders: &fls, + IgnoreMissingHeaders: &fls, + IgnoreSSLErrors: &fls, + ErrorVideoPath: &s[0], + ErrorAudioPath: &s[1], + SetUserAgent: &ext, + UserAgent: &s[2], + Proxy: &s[3], + MinTlsVersion: &tv, + }, + Extractor: extractor.ConfigT{}, + Log: logger.ConfigT{}, + Cache: cache.ConfigT{}, + } +} + +func appendConfig(src configT, dst configT) configT { + if dst.PortInt == 0 { + dst.PortInt = src.PortInt + } + if dst.Streamer.EnableErrorHeaders == nil { + dst.Streamer.EnableErrorHeaders = src.Streamer.EnableErrorHeaders + } + if dst.Streamer.IgnoreMissingHeaders == nil { + dst.Streamer.IgnoreMissingHeaders = src.Streamer.IgnoreMissingHeaders + } + if dst.Streamer.IgnoreSSLErrors == nil { + dst.Streamer.IgnoreSSLErrors = src.Streamer.IgnoreSSLErrors + } + if dst.Streamer.ErrorVideoPath == nil { + dst.Streamer.ErrorVideoPath = src.Streamer.ErrorVideoPath + } + if dst.Streamer.ErrorAudioPath == nil { + dst.Streamer.ErrorAudioPath = src.Streamer.ErrorAudioPath + } + if dst.Streamer.SetUserAgent == nil { + dst.Streamer.SetUserAgent = src.Streamer.SetUserAgent + } + if dst.Streamer.UserAgent == nil { + dst.Streamer.UserAgent = src.Streamer.UserAgent + } + if dst.Streamer.Proxy == nil { + dst.Streamer.Proxy = src.Streamer.Proxy + } + if dst.Streamer.MinTlsVersion == nil { + dst.Streamer.MinTlsVersion = src.Streamer.MinTlsVersion + } + return dst +} + func Read(path string) (configT, error) { var c configT b, err := os.ReadFile(path) @@ -39,5 +101,5 @@ func Read(path string) (configT, error) { }() err = json.Unmarshal(b, &c) - return c, err + return appendConfig(defaultConfig(), c), err } diff --git a/src/streamer/streamer.go b/src/streamer/streamer.go index eae51e5..3df020a 100644 --- a/src/streamer/streamer.go +++ b/src/streamer/streamer.go @@ -18,24 +18,27 @@ import ( const defaultErrorHeader = "Error-Header-" type ConfigT struct { - EnableErrorHeaders bool `json:"error-headers"` - IgnoreMissingHeaders bool `json:"ignore-missing-headers"` - IgnoreSSLErrors bool `json:"ignore-ssl-errors"` - ErrorVideoPath string `json:"error-video"` - ErrorAudioPath string `json:"error-audio"` - SetUserAgent SetUserAgentT `json:"set-user-agent"` - UserAgent string `json:"user-agent"` - Proxy string `json:"proxy"` - MinTlsVersion tlsVersion `json:"min-tls-version"` + EnableErrorHeaders *bool `json:"error-headers"` + IgnoreMissingHeaders *bool `json:"ignore-missing-headers"` + IgnoreSSLErrors *bool `json:"ignore-ssl-errors"` + ErrorVideoPath *string `json:"error-video"` + ErrorAudioPath *string `json:"error-audio"` + SetUserAgent *SetUserAgentT `json:"set-user-agent"` + UserAgent *string `json:"user-agent"` + Proxy *string `json:"proxy"` + MinTlsVersion *TlsVersion `json:"min-tls-version"` } -type tlsVersion uint16 +type TlsVersion uint16 -func (u *tlsVersion) UnmarshalJSON(b []byte) error { +func (u *TlsVersion) UnmarshalJSON(b []byte) error { var ( s string i uint16 ) + if u == nil { + return nil + } if err := json.Unmarshal(b, &s); err != nil { return err } @@ -47,7 +50,7 @@ func (u *tlsVersion) UnmarshalJSON(b []byte) error { } for i = 0; i < math.MaxUint16; i++ { if eq(tls.VersionName(i)) { - *u = tlsVersion(i) + *u = TlsVersion(i) return nil } } @@ -113,12 +116,12 @@ func New(conf ConfigT, log *logger.T, xt extractor.T) (T, error) { err error logs []string ) - s.errorVideoFile, err = readFile(conf.ErrorVideoPath) + s.errorVideoFile, err = readFile(*conf.ErrorVideoPath) if err != nil { return &s, err } s.errorVideoFile.contentType = "video/mp4" - s.errorAudioFile, err = readFile(conf.ErrorAudioPath) + s.errorAudioFile, err = readFile(*conf.ErrorAudioPath) if err != nil { return &s, err } @@ -229,7 +232,7 @@ func makeDoRequestFunc(conf ConfigT) (doRequestF, []string, error) { tr := &http.Transport{} logs := make([]string, 0) func() { - mintls := uint16(conf.MinTlsVersion) + mintls := uint16(*conf.MinTlsVersion) tr.TLSClientConfig = &tls.Config{MinVersion: mintls} if mintls > 0 { logs = append(logs, @@ -237,19 +240,19 @@ func makeDoRequestFunc(conf ConfigT) (doRequestF, []string, error) { tls.VersionName(mintls))) } }() - if conf.IgnoreSSLErrors { + if *conf.IgnoreSSLErrors { tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} logs = append(logs, "ignoring SSL errors") } - switch conf.Proxy { + switch *conf.Proxy { case "": logs = append(logs, "no proxy set") case "env": tr.Proxy = http.ProxyFromEnvironment logs = append(logs, "proxy set to environment") default: - logs = append(logs, fmt.Sprintf("proxy set to '%s'", conf.Proxy)) - u, err := url.Parse(conf.Proxy) + logs = append(logs, fmt.Sprintf("proxy set to '%s'", *conf.Proxy)) + u, err := url.Parse(*conf.Proxy) if err != nil { return func(r *http.Request) (*http.Response, error) { return &http.Response{}, nil @@ -267,7 +270,7 @@ func makeSendErrorVideoFunc(conf ConfigT) sendErrorFileF { return func(w http.ResponseWriter, err error, file fileT) error { w.Header().Set("Content-Length", fmt.Sprintf("%d", file.contentLength)) w.Header().Set("Content-Type", file.contentType) - if conf.EnableErrorHeaders { + if *conf.EnableErrorHeaders { hdrs, errs := errorToHeaders(err) for i := range hdrs { w.Header().Set(hdrs[i], errs[i]) @@ -279,7 +282,7 @@ func makeSendErrorVideoFunc(conf ConfigT) sendErrorFileF { } func makeSetHeaders(conf ConfigT) func(http.ResponseWriter, *http.Response) error { - headersStrictCheck := !conf.IgnoreMissingHeaders + headersStrictCheck := !*conf.IgnoreMissingHeaders return func(w http.ResponseWriter, res *http.Response) error { h1, ok := res.Header["Content-Length"] if !ok && headersStrictCheck { @@ -312,7 +315,7 @@ func makeSetHeaders(conf ConfigT) func(http.ResponseWriter, *http.Response) erro } func makeSetStreamerUserAgent(conf ConfigT, xt extractor.T, log *logger.T) (func(*http.Request) string, error) { - switch conf.SetUserAgent { + switch *conf.SetUserAgent { case Request: log.LogDebug("Streamer User-Agent set to request-set") return func(r *http.Request) string { @@ -328,7 +331,7 @@ func makeSetStreamerUserAgent(conf ConfigT, xt extractor.T, log *logger.T) (func ua := conf.UserAgent log.LogDebug("Streamer User-Agent set to", ua) return func(r *http.Request) string { - return ua + return *ua }, nil default: return func(r *http.Request) string { return "" }, From 2420b49c389b35d3d294d6f7e6e464e9532f39b1 Mon Sep 17 00:00:00 2001 From: mejgun Date: Mon, 16 Sep 2024 19:37:34 +0300 Subject: [PATCH 04/29] reworking config 2/4 --- src/config/config.go | 37 ++++++++++++++++++++++++++++++++++--- src/extractor/extractor.go | 20 ++++++++++---------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/config/config.go b/src/config/config.go index 444cb81..d3005e9 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -28,6 +28,12 @@ func defaultConfig() configT { "Mozilla", "env", } + var e = [4]string{"yt-dlp", + "-f,,(mp4)[height<={{.HEIGHT}}],,-g,,{{.URL}}", + "-f,,(m4a),,-g,,{{.URL}}", + "--dump-user-agent", + } + co := make([]string, 0) return configT{ PortInt: 8080, Streamer: streamer.ConfigT{ @@ -41,16 +47,25 @@ func defaultConfig() configT { Proxy: &s[3], MinTlsVersion: &tv, }, - Extractor: extractor.ConfigT{}, - Log: logger.ConfigT{}, - Cache: cache.ConfigT{}, + Extractor: extractor.ConfigT{ + Path: &e[0], + MP4: &e[1], + M4A: &e[2], + GetUserAgent: &e[2], + CustomOptions: &co, + }, + Log: logger.ConfigT{}, + Cache: cache.ConfigT{}, } } +// add second config options to first func appendConfig(src configT, dst configT) configT { + // general options if dst.PortInt == 0 { dst.PortInt = src.PortInt } + // streamer if dst.Streamer.EnableErrorHeaders == nil { dst.Streamer.EnableErrorHeaders = src.Streamer.EnableErrorHeaders } @@ -78,6 +93,22 @@ func appendConfig(src configT, dst configT) configT { if dst.Streamer.MinTlsVersion == nil { dst.Streamer.MinTlsVersion = src.Streamer.MinTlsVersion } + // extractor + if dst.Extractor.Path == nil { + dst.Extractor.Path = src.Extractor.Path + } + if dst.Extractor.MP4 == nil { + dst.Extractor.MP4 = src.Extractor.MP4 + } + if dst.Extractor.M4A == nil { + dst.Extractor.M4A = src.Extractor.M4A + } + if dst.Extractor.GetUserAgent == nil { + dst.Extractor.GetUserAgent = src.Extractor.GetUserAgent + } + if dst.Extractor.CustomOptions == nil { + dst.Extractor.CustomOptions = src.Extractor.CustomOptions + } return dst } diff --git a/src/extractor/extractor.go b/src/extractor/extractor.go index 97193df..3c62de7 100644 --- a/src/extractor/extractor.go +++ b/src/extractor/extractor.go @@ -16,11 +16,11 @@ import ( const separator = ",," type ConfigT struct { - Path string `json:"path"` - MP4 string `json:"mp4"` - M4A string `json:"m4a"` - GetUserAgent string `json:"get-user-agent"` - CustomOptions []string `json:"custom-options"` + Path *string `json:"path"` + MP4 *string `json:"mp4"` + M4A *string `json:"m4a"` + GetUserAgent *string `json:"get-user-agent"` + CustomOptions *[]string `json:"custom-options"` } type ResultT struct { @@ -54,24 +54,24 @@ func New(c ConfigT, log *logger.T) (T, error) { e defaultExtractor err error ) - e.m4a, err = template.New("").Parse(c.M4A) + e.m4a, err = template.New("").Parse(*c.M4A) if err != nil { return &e, err } - e.mp4, err = template.New("").Parse(c.MP4) + e.mp4, err = template.New("").Parse(*c.MP4) if err != nil { return &e, err } e.customOptions = make([]*template.Template, 0) - for _, v := range c.CustomOptions { + for _, v := range *c.CustomOptions { b, err := template.New("").Parse(v) if err != nil { return &e, err } e.customOptions = append(e.customOptions, b) } - e.getUserAgent = c.GetUserAgent - e.path = c.Path + e.getUserAgent = *c.GetUserAgent + e.path = *c.Path e.logger = log return &e, nil } From 201bf922c813e1c4c374f5160c26979c859ad421 Mon Sep 17 00:00:00 2001 From: mejgun Date: Mon, 16 Sep 2024 19:49:19 +0300 Subject: [PATCH 05/29] reworking config 3/4 --- src/config/config.go | 21 +++++++++++++++++++-- src/logger/logger.go | 14 +++++++------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/config/config.go b/src/config/config.go index d3005e9..0ba90ec 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -34,6 +34,9 @@ func defaultConfig() configT { "--dump-user-agent", } co := make([]string, 0) + ll := logger.Info + lo := logger.Stdout + lf := "log.txt" return configT{ PortInt: 8080, Streamer: streamer.ConfigT{ @@ -51,10 +54,14 @@ func defaultConfig() configT { Path: &e[0], MP4: &e[1], M4A: &e[2], - GetUserAgent: &e[2], + GetUserAgent: &e[3], CustomOptions: &co, }, - Log: logger.ConfigT{}, + Log: logger.ConfigT{ + Level: &ll, + Output: &lo, + FileName: &lf, + }, Cache: cache.ConfigT{}, } } @@ -109,6 +116,16 @@ func appendConfig(src configT, dst configT) configT { if dst.Extractor.CustomOptions == nil { dst.Extractor.CustomOptions = src.Extractor.CustomOptions } + // logger + if dst.Log.Level == nil { + dst.Log.Level = src.Log.Level + } + if dst.Log.Output == nil { + dst.Log.Output = src.Log.Output + } + if dst.Log.FileName == nil { + dst.Log.FileName = src.Log.FileName + } return dst } diff --git a/src/logger/logger.go b/src/logger/logger.go index 4747ddb..35993ea 100644 --- a/src/logger/logger.go +++ b/src/logger/logger.go @@ -19,9 +19,9 @@ type T struct { } type ConfigT struct { - Level LevelT `json:"level"` - Output OutputT `json:"output"` - FileName string `json:"filename"` + Level *LevelT `json:"level"` + Output *OutputT `json:"output"` + FileName *string `json:"filename"` } type LevelT uint8 @@ -91,7 +91,7 @@ func New(conf ConfigT) (*T, error) { LogDebug: func(s string, i ...interface{}) {}, LogInfo: func(s string, i ...interface{}) {}, } - if conf.Level == Nothing { + if *conf.Level == Nothing { return &logger, nil } var ( @@ -101,9 +101,9 @@ func New(conf ConfigT) (*T, error) { return os.OpenFile( // will never close this file :| // should trap exit - conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) + *conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) } - switch conf.Output { + switch *conf.Output { case Stdout: l.SetOutput(os.Stdout) case File: @@ -124,7 +124,7 @@ func New(conf ConfigT) (*T, error) { fmt.Sprintf("[ %s ] %s:", str, s) + fmt.Sprintf(strings.Repeat(" %+v", len(i)), i...)) } - switch conf.Level { + switch *conf.Level { case Debug: logger.LogDebug = func(s string, i ...interface{}) { print("DEBUG", s, i) } fallthrough From e19c522700351afe5c1ec652c9c6ab3b7743f012 Mon Sep 17 00:00:00 2001 From: mejgun Date: Mon, 16 Sep 2024 19:56:13 +0300 Subject: [PATCH 06/29] reworking config 4/4 --- src/cache/cache.go | 26 ++++++++++---------------- src/config/config.go | 9 ++++++++- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/cache/cache.go b/src/cache/cache.go index 27f1bcd..dd6bdb0 100644 --- a/src/cache/cache.go +++ b/src/cache/cache.go @@ -1,6 +1,7 @@ package cache import ( + "fmt" "sync" "time" @@ -18,8 +19,6 @@ type ConfigT struct { ExpireTime *string `json:"expire-time"` } -const defaultExpireTime = 3 * time.Hour - func New(conf ConfigT, log *logger.T) (T, error) { defCache := func(t time.Duration) *defaultCache { return &defaultCache{ @@ -27,21 +26,16 @@ func New(conf ConfigT, log *logger.T) (T, error) { expireTime: t, } } - switch { - case conf.ExpireTime == nil: - log.LogDebug("cache", "no expire time set in config, using default 3h") - return defCache(defaultExpireTime), nil - default: - t, err := time.ParseDuration(*conf.ExpireTime) - if err != nil { - return &defaultCache{}, err - } - if t.Seconds() < 1 { - log.LogDebug("cache", "disabled by config") - return &emptyCache{}, nil - } - return defCache(t), nil + t, err := time.ParseDuration(*conf.ExpireTime) + if err != nil { + return &defaultCache{}, err + } + if t.Seconds() < 1 { + log.LogDebug("cache", "disabled by config") + return &emptyCache{}, nil } + log.LogDebug("cache", fmt.Sprintf("expire time set to %s", t)) + return defCache(t), nil } type defaultCache struct { diff --git a/src/config/config.go b/src/config/config.go index 0ba90ec..15965ad 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -37,6 +37,7 @@ func defaultConfig() configT { ll := logger.Info lo := logger.Stdout lf := "log.txt" + exp := "3h" return configT{ PortInt: 8080, Streamer: streamer.ConfigT{ @@ -62,7 +63,9 @@ func defaultConfig() configT { Output: &lo, FileName: &lf, }, - Cache: cache.ConfigT{}, + Cache: cache.ConfigT{ + ExpireTime: &exp, + }, } } @@ -126,6 +129,10 @@ func appendConfig(src configT, dst configT) configT { if dst.Log.FileName == nil { dst.Log.FileName = src.Log.FileName } + // cache + if dst.Cache.ExpireTime == nil { + dst.Cache.ExpireTime = src.Cache.ExpireTime + } return dst } From 7e9c8ff80904465a39362b8925266e0e27fde7e0 Mon Sep 17 00:00:00 2001 From: mejgun Date: Mon, 16 Sep 2024 23:04:35 +0300 Subject: [PATCH 07/29] logger rework --- src/logger/log.go | 65 +++++++++++++++++++++++++++++++++++++++ src/logger/logger.go | 60 ------------------------------------ src/logger/slog.go | 73 ++++++++++++++++++++++++++++++++++++++++++++ src/main.go | 2 +- 4 files changed, 139 insertions(+), 61 deletions(-) create mode 100644 src/logger/log.go create mode 100644 src/logger/slog.go diff --git a/src/logger/log.go b/src/logger/log.go new file mode 100644 index 0000000..62383cb --- /dev/null +++ b/src/logger/log.go @@ -0,0 +1,65 @@ +package logger + +import ( + "fmt" + "io" + "log" + "os" + "strings" +) + +func NewDefault(conf ConfigT) (*T, error) { + var logger = T{ + LogError: func(s string, i ...interface{}) {}, + LogWarning: func(s string, i ...interface{}) {}, + LogDebug: func(s string, i ...interface{}) {}, + LogInfo: func(s string, i ...interface{}) {}, + } + if *conf.Level == Nothing { + return &logger, nil + } + var ( + l *log.Logger = log.Default() + ) + open := func() (*os.File, error) { + return os.OpenFile( + // will never close this file :| + // should trap exit + *conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) + } + switch *conf.Output { + case Stdout: + l.SetOutput(os.Stdout) + case File: + f, err := open() + if err != nil { + return &logger, err + } + l.SetOutput(f) + case Both: + f, err := open() + if err != nil { + return &logger, err + } + l.SetOutput(io.MultiWriter(os.Stdout, f)) + } + print := func(str string, s string, i []interface{}) { + l.Printf( + fmt.Sprintf("[ %s ] %s:", str, s) + + fmt.Sprintf(strings.Repeat(" %+v", len(i)), i...)) + } + switch *conf.Level { + case Debug: + logger.LogDebug = func(s string, i ...interface{}) { print("DEBUG", s, i) } + fallthrough + case Info: + logger.LogInfo = func(s string, i ...interface{}) { print("INFO", s, i) } + fallthrough + case Warning: + logger.LogWarning = func(s string, i ...interface{}) { print("WARNING", s, i) } + fallthrough + case Error: + logger.LogError = func(s string, i ...interface{}) { print("ERROR", s, i) } + } + return &logger, nil +} diff --git a/src/logger/logger.go b/src/logger/logger.go index 35993ea..22465c6 100644 --- a/src/logger/logger.go +++ b/src/logger/logger.go @@ -3,10 +3,6 @@ package logger import ( "encoding/json" "fmt" - "io" - "log" - "os" - "strings" ) type logFuncT func(string, ...interface{}) @@ -83,59 +79,3 @@ func (o *OutputT) UnmarshalJSON(b []byte) error { } return nil } - -func New(conf ConfigT) (*T, error) { - var logger = T{ - LogError: func(s string, i ...interface{}) {}, - LogWarning: func(s string, i ...interface{}) {}, - LogDebug: func(s string, i ...interface{}) {}, - LogInfo: func(s string, i ...interface{}) {}, - } - if *conf.Level == Nothing { - return &logger, nil - } - var ( - l *log.Logger = log.Default() - ) - open := func() (*os.File, error) { - return os.OpenFile( - // will never close this file :| - // should trap exit - *conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) - } - switch *conf.Output { - case Stdout: - l.SetOutput(os.Stdout) - case File: - f, err := open() - if err != nil { - return &logger, err - } - l.SetOutput(f) - case Both: - f, err := open() - if err != nil { - return &logger, err - } - l.SetOutput(io.MultiWriter(os.Stdout, f)) - } - print := func(str string, s string, i []interface{}) { - l.Printf( - fmt.Sprintf("[ %s ] %s:", str, s) + - fmt.Sprintf(strings.Repeat(" %+v", len(i)), i...)) - } - switch *conf.Level { - case Debug: - logger.LogDebug = func(s string, i ...interface{}) { print("DEBUG", s, i) } - fallthrough - case Info: - logger.LogInfo = func(s string, i ...interface{}) { print("INFO", s, i) } - fallthrough - case Warning: - logger.LogWarning = func(s string, i ...interface{}) { print("WARNING", s, i) } - fallthrough - case Error: - logger.LogError = func(s string, i ...interface{}) { print("ERROR", s, i) } - } - return &logger, nil -} diff --git a/src/logger/slog.go b/src/logger/slog.go new file mode 100644 index 0000000..aa32ead --- /dev/null +++ b/src/logger/slog.go @@ -0,0 +1,73 @@ +package logger + +import ( + "io" + "log/slog" + "os" +) + +func NewSlog(conf ConfigT) (*T, error) { + var logger = T{ + LogError: func(s string, i ...interface{}) {}, + LogWarning: func(s string, i ...interface{}) {}, + LogDebug: func(s string, i ...interface{}) {}, + LogInfo: func(s string, i ...interface{}) {}, + } + if *conf.Level == Nothing { + return &logger, nil + } + open := func() (*os.File, error) { + return os.OpenFile( + // will never close this file :| + // should trap exit + *conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) + } + var ( + lvl slog.Level + l *slog.Logger + ) + switch *conf.Level { + case Debug: + lvl = slog.LevelDebug + case Info: + lvl = slog.LevelInfo + case Warning: + lvl = slog.LevelWarn + case Error: + lvl = slog.LevelError + } + mkLogger := func(dst io.Writer) { + l = slog.New( + slog.NewTextHandler(dst, + &slog.HandlerOptions{Level: lvl})) + } + switch *conf.Output { + case Stdout: + mkLogger(os.Stdout) + case File: + f, err := open() + if err != nil { + return &logger, err + } + mkLogger(f) + case Both: + f, err := open() + if err != nil { + return &logger, err + } + mkLogger(io.MultiWriter(os.Stdout, f)) + } + logger.LogDebug = func(s string, i ...interface{}) { + l.Debug(s, i...) + } + logger.LogInfo = func(s string, i ...interface{}) { + l.Info(s, i...) + } + logger.LogWarning = func(s string, i ...interface{}) { + l.Warn(s, i...) + } + logger.LogError = func(s string, i ...interface{}) { + l.Error(s, i...) + } + return &logger, nil +} diff --git a/src/main.go b/src/main.go index abe7f43..5b2b274 100644 --- a/src/main.go +++ b/src/main.go @@ -65,7 +65,7 @@ func main() { conf, err := config.Read(flags.config) checkOrExit(err, "Config", ConfigError) - log, err := logger.New(conf.Log) + log, err := logger.NewDefault(conf.Log) checkOrExit(err, "Logger", LoggerError) log.LogDebug("logger created") From 0c101d37cc8b43fc55d673287711f6c2cfaf137e Mon Sep 17 00:00:00 2001 From: mejgun Date: Tue, 17 Sep 2024 13:13:47 +0300 Subject: [PATCH 08/29] reworking modules struct --- src/cache/go.mod | 12 ------------ src/cmd/go.mod | 7 +++++++ src/{ => cmd}/main.go | 10 +++++----- src/{ => cmd}/main_test.go | 2 +- src/config/go.mod | 17 ----------------- src/extractor/go.mod | 7 ------- src/go.mod | 22 ---------------------- src/{ => lib}/cache/cache.go | 4 ++-- src/{ => lib}/config/config.go | 8 ++++---- src/{ => lib}/extractor/extractor.go | 2 +- src/lib/go.mod | 3 +++ src/{ => lib}/logger/log.go | 0 src/{ => lib}/logger/logger.go | 0 src/{ => lib}/logger/slog.go | 0 src/{ => lib}/streamer/streamer.go | 4 ++-- src/{ => lib}/streamer/streamer_test.go | 0 src/logger/go.mod | 3 --- src/streamer/go.mod | 13 ------------- 18 files changed, 25 insertions(+), 89 deletions(-) delete mode 100644 src/cache/go.mod create mode 100644 src/cmd/go.mod rename src/{ => cmd}/main.go (96%) rename src/{ => cmd}/main_test.go (97%) delete mode 100644 src/config/go.mod delete mode 100644 src/extractor/go.mod delete mode 100644 src/go.mod rename src/{ => lib}/cache/cache.go (97%) rename src/{ => lib}/config/config.go (97%) rename src/{ => lib}/extractor/extractor.go (99%) create mode 100644 src/lib/go.mod rename src/{ => lib}/logger/log.go (100%) rename src/{ => lib}/logger/logger.go (100%) rename src/{ => lib}/logger/slog.go (100%) rename src/{ => lib}/streamer/streamer.go (99%) rename src/{ => lib}/streamer/streamer_test.go (100%) delete mode 100644 src/logger/go.mod delete mode 100644 src/streamer/go.mod diff --git a/src/cache/go.mod b/src/cache/go.mod deleted file mode 100644 index dcd2b6c..0000000 --- a/src/cache/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -module ytproxy-linkscache - -go 1.22 - -replace ytproxy-extractor => ../extractor - -replace ytproxy-logger => ../logger - -require ( - ytproxy-extractor v0.0.0-00010101000000-000000000000 - ytproxy-logger v0.0.0-00010101000000-000000000000 -) diff --git a/src/cmd/go.mod b/src/cmd/go.mod new file mode 100644 index 0000000..13fdaa5 --- /dev/null +++ b/src/cmd/go.mod @@ -0,0 +1,7 @@ +module ytproxy + +go 1.22.6 + +replace lib => ../lib + +require lib v0.0.0-00010101000000-000000000000 diff --git a/src/main.go b/src/cmd/main.go similarity index 96% rename from src/main.go rename to src/cmd/main.go index 5b2b274..2208533 100644 --- a/src/main.go +++ b/src/cmd/main.go @@ -9,11 +9,11 @@ import ( "strings" "time" - config "ytproxy-config" - extractor "ytproxy-extractor" - cache "ytproxy-linkscache" - logger "ytproxy-logger" - streamer "ytproxy-streamer" + cache "lib/cache" + config "lib/config" + extractor "lib/extractor" + logger "lib/logger" + streamer "lib/streamer" ) const appVersion = "1.6.0" diff --git a/src/main_test.go b/src/cmd/main_test.go similarity index 97% rename from src/main_test.go rename to src/cmd/main_test.go index de9bd53..9bf874d 100644 --- a/src/main_test.go +++ b/src/cmd/main_test.go @@ -3,7 +3,7 @@ package main import ( "testing" - extractor "ytproxy-extractor" + extractor "lib/extractor" ) var testPairs = map[string]extractor.RequestT{ diff --git a/src/config/go.mod b/src/config/go.mod deleted file mode 100644 index 1dd0b67..0000000 --- a/src/config/go.mod +++ /dev/null @@ -1,17 +0,0 @@ -module ytproxy-config - -go 1.22 - -replace ( - ytproxy-extractor => ../extractor - ytproxy-linkscache => ../cache - ytproxy-logger => ../logger - ytproxy-streamer => ../streamer -) - -require ( - ytproxy-extractor v0.0.0-00010101000000-000000000000 - ytproxy-linkscache v0.0.0-00010101000000-000000000000 - ytproxy-logger v0.0.0-00010101000000-000000000000 - ytproxy-streamer v0.0.0-00010101000000-000000000000 -) diff --git a/src/extractor/go.mod b/src/extractor/go.mod deleted file mode 100644 index 1896994..0000000 --- a/src/extractor/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module ytproxy-extractor - -go 1.22 - -replace ytproxy-logger => ../logger - -require ytproxy-logger v0.0.0-00010101000000-000000000000 diff --git a/src/go.mod b/src/go.mod deleted file mode 100644 index 0f08f86..0000000 --- a/src/go.mod +++ /dev/null @@ -1,22 +0,0 @@ -module ytproxy - -go 1.22 - -replace ( - ytproxy-config => ./config - ytproxy-extractor => ./extractor - ytproxy-linkscache => ./cache - ytproxy-logger => ./logger - ytproxy-streamer => ./streamer -) - -require ( - ytproxy-config v0.0.0-00010101000000-000000000000 - ytproxy-extractor v0.0.0-00010101000000-000000000000 - ytproxy-linkscache v0.0.0-00010101000000-000000000000 -) - -require ( - ytproxy-logger v0.0.0-00010101000000-000000000000 - ytproxy-streamer v0.0.0-00010101000000-000000000000 -) diff --git a/src/cache/cache.go b/src/lib/cache/cache.go similarity index 97% rename from src/cache/cache.go rename to src/lib/cache/cache.go index dd6bdb0..015965f 100644 --- a/src/cache/cache.go +++ b/src/lib/cache/cache.go @@ -5,8 +5,8 @@ import ( "sync" "time" - extractor "ytproxy-extractor" - logger "ytproxy-logger" + extractor "lib/extractor" + logger "lib/logger" ) type T interface { diff --git a/src/config/config.go b/src/lib/config/config.go similarity index 97% rename from src/config/config.go rename to src/lib/config/config.go index 15965ad..2a527d2 100644 --- a/src/config/config.go +++ b/src/lib/config/config.go @@ -5,10 +5,10 @@ import ( "os" "strings" - extractor "ytproxy-extractor" - cache "ytproxy-linkscache" - logger "ytproxy-logger" - streamer "ytproxy-streamer" + cache "lib/cache" + extractor "lib/extractor" + logger "lib/logger" + streamer "lib/streamer" ) type configT struct { diff --git a/src/extractor/extractor.go b/src/lib/extractor/extractor.go similarity index 99% rename from src/extractor/extractor.go rename to src/lib/extractor/extractor.go index 3c62de7..d64f694 100644 --- a/src/extractor/extractor.go +++ b/src/lib/extractor/extractor.go @@ -10,7 +10,7 @@ import ( "text/template" "time" - logger "ytproxy-logger" + logger "lib/logger" ) const separator = ",," diff --git a/src/lib/go.mod b/src/lib/go.mod new file mode 100644 index 0000000..43732d1 --- /dev/null +++ b/src/lib/go.mod @@ -0,0 +1,3 @@ +module lib + +go 1.22.6 diff --git a/src/logger/log.go b/src/lib/logger/log.go similarity index 100% rename from src/logger/log.go rename to src/lib/logger/log.go diff --git a/src/logger/logger.go b/src/lib/logger/logger.go similarity index 100% rename from src/logger/logger.go rename to src/lib/logger/logger.go diff --git a/src/logger/slog.go b/src/lib/logger/slog.go similarity index 100% rename from src/logger/slog.go rename to src/lib/logger/slog.go diff --git a/src/streamer/streamer.go b/src/lib/streamer/streamer.go similarity index 99% rename from src/streamer/streamer.go rename to src/lib/streamer/streamer.go index 3df020a..b9d93dd 100644 --- a/src/streamer/streamer.go +++ b/src/lib/streamer/streamer.go @@ -11,8 +11,8 @@ import ( "os" "strings" - extractor "ytproxy-extractor" - logger "ytproxy-logger" + extractor "lib/extractor" + logger "lib/logger" ) const defaultErrorHeader = "Error-Header-" diff --git a/src/streamer/streamer_test.go b/src/lib/streamer/streamer_test.go similarity index 100% rename from src/streamer/streamer_test.go rename to src/lib/streamer/streamer_test.go diff --git a/src/logger/go.mod b/src/logger/go.mod deleted file mode 100644 index b6c2b67..0000000 --- a/src/logger/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module ytproxy-logger - -go 1.22 diff --git a/src/streamer/go.mod b/src/streamer/go.mod deleted file mode 100644 index 07fa94b..0000000 --- a/src/streamer/go.mod +++ /dev/null @@ -1,13 +0,0 @@ -module ytproxy-streamer - -go 1.22 - -replace ( - ytproxy-extractor => ../extractor - ytproxy-logger => ../logger -) - -require ( - ytproxy-extractor v0.0.0-00010101000000-000000000000 - ytproxy-logger v0.0.0-00010101000000-000000000000 -) From c8c9d1230ed50037b770f5f67dad405e7360256a Mon Sep 17 00:00:00 2001 From: mejgun Date: Tue, 17 Sep 2024 14:06:04 +0300 Subject: [PATCH 09/29] reworking logger abstraction --- src/cmd/main.go | 4 +- src/lib/cache/cache.go | 2 +- src/lib/config/config.go | 18 +++--- src/lib/extractor/extractor.go | 4 +- src/lib/logger/config/config.go | 72 ++++++++++++++++++++++++ src/lib/logger/impl/default/log.go | 78 ++++++++++++++++++++++++++ src/lib/logger/{ => impl/slog}/slog.go | 0 src/lib/logger/log.go | 65 --------------------- src/lib/logger/logger.go | 12 ++-- src/lib/logger/new.go | 10 ++++ src/lib/streamer/streamer.go | 6 +- 11 files changed, 182 insertions(+), 89 deletions(-) create mode 100644 src/lib/logger/config/config.go create mode 100644 src/lib/logger/impl/default/log.go rename src/lib/logger/{ => impl/slog}/slog.go (100%) delete mode 100644 src/lib/logger/log.go create mode 100644 src/lib/logger/new.go diff --git a/src/cmd/main.go b/src/cmd/main.go index 2208533..cc8ff68 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -65,7 +65,7 @@ func main() { conf, err := config.Read(flags.config) checkOrExit(err, "Config", ConfigError) - log, err := logger.NewDefault(conf.Log) + log, err := logger.New(conf.Log) checkOrExit(err, "Logger", LoggerError) log.LogDebug("logger created") @@ -118,7 +118,7 @@ func main() { } } -func getLink(query string, log *logger.T, cache cache.T, +func getLink(query string, log logger.T, cache cache.T, extractor extractor.T) (extractor.RequestT, extractor.ResultT, error) { now := time.Now() req := parseQuery(query) diff --git a/src/lib/cache/cache.go b/src/lib/cache/cache.go index 015965f..cd21cba 100644 --- a/src/lib/cache/cache.go +++ b/src/lib/cache/cache.go @@ -19,7 +19,7 @@ type ConfigT struct { ExpireTime *string `json:"expire-time"` } -func New(conf ConfigT, log *logger.T) (T, error) { +func New(conf ConfigT, log logger.T) (T, error) { defCache := func(t time.Duration) *defaultCache { return &defaultCache{ cache: make(map[extractor.RequestT]extractor.ResultT), diff --git a/src/lib/config/config.go b/src/lib/config/config.go index 2a527d2..876dcfd 100644 --- a/src/lib/config/config.go +++ b/src/lib/config/config.go @@ -7,16 +7,16 @@ import ( cache "lib/cache" extractor "lib/extractor" - logger "lib/logger" + logger_config "lib/logger/config" streamer "lib/streamer" ) type configT struct { - PortInt uint16 `json:"port"` - Streamer streamer.ConfigT `json:"streamer"` - Extractor extractor.ConfigT `json:"extractor"` - Log logger.ConfigT `json:"log"` - Cache cache.ConfigT `json:"cache"` + PortInt uint16 `json:"port"` + Streamer streamer.ConfigT `json:"streamer"` + Extractor extractor.ConfigT `json:"extractor"` + Log logger_config.ConfigT `json:"log"` + Cache cache.ConfigT `json:"cache"` } func defaultConfig() configT { @@ -34,8 +34,8 @@ func defaultConfig() configT { "--dump-user-agent", } co := make([]string, 0) - ll := logger.Info - lo := logger.Stdout + ll := logger_config.Info + lo := logger_config.Stdout lf := "log.txt" exp := "3h" return configT{ @@ -58,7 +58,7 @@ func defaultConfig() configT { GetUserAgent: &e[3], CustomOptions: &co, }, - Log: logger.ConfigT{ + Log: logger_config.ConfigT{ Level: &ll, Output: &lo, FileName: &lf, diff --git a/src/lib/extractor/extractor.go b/src/lib/extractor/extractor.go index d64f694..dabbbd4 100644 --- a/src/lib/extractor/extractor.go +++ b/src/lib/extractor/extractor.go @@ -40,7 +40,7 @@ type defaultExtractor struct { m4a *template.Template customOptions []*template.Template getUserAgent string - logger *logger.T + logger logger.T } type RequestT struct { @@ -49,7 +49,7 @@ type RequestT struct { FORMAT string } -func New(c ConfigT, log *logger.T) (T, error) { +func New(c ConfigT, log logger.T) (T, error) { var ( e defaultExtractor err error diff --git a/src/lib/logger/config/config.go b/src/lib/logger/config/config.go new file mode 100644 index 0000000..5a5a58c --- /dev/null +++ b/src/lib/logger/config/config.go @@ -0,0 +1,72 @@ +package logger + +import ( + "encoding/json" + "fmt" +) + +type ConfigT struct { + Level *LevelT `json:"level"` + Output *OutputT `json:"output"` + FileName *string `json:"filename"` +} + +type LevelT uint8 + +const ( + Debug LevelT = iota + Info + Warning + Error + Nothing +) + +func (l *LevelT) UnmarshalJSON(b []byte) error { + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + switch s { + case "debug": + *l = Debug + case "info": + *l = Info + case "warning": + *l = Warning + case "error": + *l = Error + case "nothing": + *l = Nothing + default: + return fmt.Errorf("cannot unmarshal %s as log level", b) + } + return nil +} + +type OutputT uint8 + +const ( + Stdout OutputT = iota + File + Both +) + +func (o *OutputT) UnmarshalJSON(b []byte) error { + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + switch s { + case "stdout": + *o = Stdout + case "file": + *o = File + case "both": + *o = Both + default: + return fmt.Errorf("cannot unmarshal %s as log output", b) + } + return nil +} diff --git a/src/lib/logger/impl/default/log.go b/src/lib/logger/impl/default/log.go new file mode 100644 index 0000000..d6d5a6c --- /dev/null +++ b/src/lib/logger/impl/default/log.go @@ -0,0 +1,78 @@ +package logger + +import ( + "fmt" + "io" + "log" + "os" + "strings" + + l "lib/logger/config" +) + +type loggerT struct { + lvl *l.LevelT + lgr *log.Logger +} + +func (t *loggerT) print(str string, s string, i []interface{}) { + t.lgr.Printf( + fmt.Sprintf("%-7s %s:", str, s) + + fmt.Sprintf(strings.Repeat(" %+v", len(i)), i...)) +} + +func (t *loggerT) checkAndPrint(lvl l.LevelT, str string, s string, i ...any) { + if *t.lvl <= lvl { + t.print(str, s, i) + } +} + +func (t *loggerT) LogError(s string, i ...any) { + t.checkAndPrint(l.Error, "ERROR", s, i) +} +func (t *loggerT) LogWarning(s string, i ...any) { + t.checkAndPrint(l.Warning, "WARNING", s, i) +} +func (t *loggerT) LogDebug(s string, i ...any) { + t.checkAndPrint(l.Debug, "DEBUG", s, i) + +} +func (t *loggerT) LogInfo(s string, i ...any) { + t.checkAndPrint(l.Info, "INFO", s, i) +} + +func New(conf l.ConfigT) (*loggerT, error) { + var ( + logger loggerT + lgr *log.Logger = log.Default() + ) + if *conf.Level == l.Nothing { + return &logger, nil + } + open := func() (*os.File, error) { + return os.OpenFile( + // will never close this file :| + // should trap exit + *conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) + } + switch *conf.Output { + case l.Stdout: + lgr.SetOutput(os.Stdout) + case l.File: + f, err := open() + if err != nil { + return &logger, err + } + lgr.SetOutput(f) + case l.Both: + f, err := open() + if err != nil { + return &logger, err + } + out := io.MultiWriter(os.Stdout, f) + lgr.SetOutput(out) + } + logger.lgr = lgr + logger.lvl = conf.Level + return &logger, nil +} diff --git a/src/lib/logger/slog.go b/src/lib/logger/impl/slog/slog.go similarity index 100% rename from src/lib/logger/slog.go rename to src/lib/logger/impl/slog/slog.go diff --git a/src/lib/logger/log.go b/src/lib/logger/log.go deleted file mode 100644 index 62383cb..0000000 --- a/src/lib/logger/log.go +++ /dev/null @@ -1,65 +0,0 @@ -package logger - -import ( - "fmt" - "io" - "log" - "os" - "strings" -) - -func NewDefault(conf ConfigT) (*T, error) { - var logger = T{ - LogError: func(s string, i ...interface{}) {}, - LogWarning: func(s string, i ...interface{}) {}, - LogDebug: func(s string, i ...interface{}) {}, - LogInfo: func(s string, i ...interface{}) {}, - } - if *conf.Level == Nothing { - return &logger, nil - } - var ( - l *log.Logger = log.Default() - ) - open := func() (*os.File, error) { - return os.OpenFile( - // will never close this file :| - // should trap exit - *conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) - } - switch *conf.Output { - case Stdout: - l.SetOutput(os.Stdout) - case File: - f, err := open() - if err != nil { - return &logger, err - } - l.SetOutput(f) - case Both: - f, err := open() - if err != nil { - return &logger, err - } - l.SetOutput(io.MultiWriter(os.Stdout, f)) - } - print := func(str string, s string, i []interface{}) { - l.Printf( - fmt.Sprintf("[ %s ] %s:", str, s) + - fmt.Sprintf(strings.Repeat(" %+v", len(i)), i...)) - } - switch *conf.Level { - case Debug: - logger.LogDebug = func(s string, i ...interface{}) { print("DEBUG", s, i) } - fallthrough - case Info: - logger.LogInfo = func(s string, i ...interface{}) { print("INFO", s, i) } - fallthrough - case Warning: - logger.LogWarning = func(s string, i ...interface{}) { print("WARNING", s, i) } - fallthrough - case Error: - logger.LogError = func(s string, i ...interface{}) { print("ERROR", s, i) } - } - return &logger, nil -} diff --git a/src/lib/logger/logger.go b/src/lib/logger/logger.go index 22465c6..9e01ccd 100644 --- a/src/lib/logger/logger.go +++ b/src/lib/logger/logger.go @@ -5,13 +5,11 @@ import ( "fmt" ) -type logFuncT func(string, ...interface{}) - -type T struct { - LogError logFuncT - LogWarning logFuncT - LogDebug logFuncT - LogInfo logFuncT +type T interface { + LogError(string, ...any) + LogWarning(string, ...any) + LogDebug(string, ...any) + LogInfo(string, ...any) } type ConfigT struct { diff --git a/src/lib/logger/new.go b/src/lib/logger/new.go new file mode 100644 index 0000000..b20725d --- /dev/null +++ b/src/lib/logger/new.go @@ -0,0 +1,10 @@ +package logger + +import ( + config "lib/logger/config" + def "lib/logger/impl/default" +) + +func New(conf config.ConfigT) (T, error) { + return def.New(conf) +} diff --git a/src/lib/streamer/streamer.go b/src/lib/streamer/streamer.go index b9d93dd..395333d 100644 --- a/src/lib/streamer/streamer.go +++ b/src/lib/streamer/streamer.go @@ -96,7 +96,7 @@ type streamer struct { sendErrorFile sendErrorFileF setHeaders func(http.ResponseWriter, *http.Response) error setStreamerUserAgent func(*http.Request) string - log *logger.T + log logger.T } type ( @@ -110,7 +110,7 @@ type fileT struct { contentLength int64 } -func New(conf ConfigT, log *logger.T, xt extractor.T) (T, error) { +func New(conf ConfigT, log logger.T, xt extractor.T) (T, error) { var ( s streamer err error @@ -314,7 +314,7 @@ func makeSetHeaders(conf ConfigT) func(http.ResponseWriter, *http.Response) erro } } -func makeSetStreamerUserAgent(conf ConfigT, xt extractor.T, log *logger.T) (func(*http.Request) string, error) { +func makeSetStreamerUserAgent(conf ConfigT, xt extractor.T, log logger.T) (func(*http.Request) string, error) { switch *conf.SetUserAgent { case Request: log.LogDebug("Streamer User-Agent set to request-set") From d2bbb38f33050e16cf5c167bbbb84d78c1daa1c6 Mon Sep 17 00:00:00 2001 From: mejgun Date: Tue, 17 Sep 2024 15:53:01 +0300 Subject: [PATCH 10/29] add json log & null log --- src/lib/config/config.go | 4 ++ src/lib/logger/config/config.go | 1 + src/lib/logger/impl/default/log.go | 3 -- src/lib/logger/impl/empty/empty.go | 13 +++++ src/lib/logger/impl/slog/slog.go | 71 ++++++++++++------------- src/lib/logger/logger.go | 83 ++++++------------------------ src/lib/logger/new.go | 10 ---- 7 files changed, 69 insertions(+), 116 deletions(-) create mode 100644 src/lib/logger/impl/empty/empty.go delete mode 100644 src/lib/logger/new.go diff --git a/src/lib/config/config.go b/src/lib/config/config.go index 876dcfd..9276dce 100644 --- a/src/lib/config/config.go +++ b/src/lib/config/config.go @@ -60,6 +60,7 @@ func defaultConfig() configT { }, Log: logger_config.ConfigT{ Level: &ll, + Json: &fls, Output: &lo, FileName: &lf, }, @@ -123,6 +124,9 @@ func appendConfig(src configT, dst configT) configT { if dst.Log.Level == nil { dst.Log.Level = src.Log.Level } + if dst.Log.Json == nil { + dst.Log.Json = src.Log.Json + } if dst.Log.Output == nil { dst.Log.Output = src.Log.Output } diff --git a/src/lib/logger/config/config.go b/src/lib/logger/config/config.go index 5a5a58c..6a5fff0 100644 --- a/src/lib/logger/config/config.go +++ b/src/lib/logger/config/config.go @@ -7,6 +7,7 @@ import ( type ConfigT struct { Level *LevelT `json:"level"` + Json *bool `json:"json"` Output *OutputT `json:"output"` FileName *string `json:"filename"` } diff --git a/src/lib/logger/impl/default/log.go b/src/lib/logger/impl/default/log.go index d6d5a6c..71fd295 100644 --- a/src/lib/logger/impl/default/log.go +++ b/src/lib/logger/impl/default/log.go @@ -46,9 +46,6 @@ func New(conf l.ConfigT) (*loggerT, error) { logger loggerT lgr *log.Logger = log.Default() ) - if *conf.Level == l.Nothing { - return &logger, nil - } open := func() (*os.File, error) { return os.OpenFile( // will never close this file :| diff --git a/src/lib/logger/impl/empty/empty.go b/src/lib/logger/impl/empty/empty.go new file mode 100644 index 0000000..a9b3d32 --- /dev/null +++ b/src/lib/logger/impl/empty/empty.go @@ -0,0 +1,13 @@ +package logger + +type loggerT struct { +} + +func (t *loggerT) LogError(string, ...any) {} +func (t *loggerT) LogWarning(string, ...any) {} +func (t *loggerT) LogDebug(string, ...any) {} +func (t *loggerT) LogInfo(string, ...any) {} + +func New() (*loggerT, error) { + return &loggerT{}, nil +} diff --git a/src/lib/logger/impl/slog/slog.go b/src/lib/logger/impl/slog/slog.go index aa32ead..f356918 100644 --- a/src/lib/logger/impl/slog/slog.go +++ b/src/lib/logger/impl/slog/slog.go @@ -4,18 +4,29 @@ import ( "io" "log/slog" "os" + + l "lib/logger/config" ) -func NewSlog(conf ConfigT) (*T, error) { - var logger = T{ - LogError: func(s string, i ...interface{}) {}, - LogWarning: func(s string, i ...interface{}) {}, - LogDebug: func(s string, i ...interface{}) {}, - LogInfo: func(s string, i ...interface{}) {}, - } - if *conf.Level == Nothing { - return &logger, nil - } +type loggerT struct { + lgr *slog.Logger +} + +func (t *loggerT) LogError(s string, i ...any) { + t.lgr.Error(s, i...) +} +func (t *loggerT) LogWarning(s string, i ...any) { + t.lgr.Warn(s, i...) +} +func (t *loggerT) LogDebug(s string, i ...any) { + t.lgr.Debug(s, i...) + +} +func (t *loggerT) LogInfo(s string, i ...any) { + t.lgr.Info(s, i...) +} + +func New(conf l.ConfigT) (*loggerT, error) { open := func() (*os.File, error) { return os.OpenFile( // will never close this file :| @@ -24,50 +35,40 @@ func NewSlog(conf ConfigT) (*T, error) { } var ( lvl slog.Level - l *slog.Logger + lgr *slog.Logger ) switch *conf.Level { - case Debug: + case l.Debug: lvl = slog.LevelDebug - case Info: + case l.Info: lvl = slog.LevelInfo - case Warning: + case l.Warning: lvl = slog.LevelWarn - case Error: + case l.Error: lvl = slog.LevelError } mkLogger := func(dst io.Writer) { - l = slog.New( - slog.NewTextHandler(dst, + lgr = slog.New( + slog.NewJSONHandler(dst, &slog.HandlerOptions{Level: lvl})) } switch *conf.Output { - case Stdout: + case l.Stdout: mkLogger(os.Stdout) - case File: + case l.File: f, err := open() if err != nil { - return &logger, err + return &loggerT{}, err } mkLogger(f) - case Both: + case l.Both: f, err := open() if err != nil { - return &logger, err + return &loggerT{}, err } mkLogger(io.MultiWriter(os.Stdout, f)) } - logger.LogDebug = func(s string, i ...interface{}) { - l.Debug(s, i...) - } - logger.LogInfo = func(s string, i ...interface{}) { - l.Info(s, i...) - } - logger.LogWarning = func(s string, i ...interface{}) { - l.Warn(s, i...) - } - logger.LogError = func(s string, i ...interface{}) { - l.Error(s, i...) - } - return &logger, nil + return &loggerT{ + lgr: lgr, + }, nil } diff --git a/src/lib/logger/logger.go b/src/lib/logger/logger.go index 9e01ccd..0411b93 100644 --- a/src/lib/logger/logger.go +++ b/src/lib/logger/logger.go @@ -1,79 +1,26 @@ package logger import ( - "encoding/json" - "fmt" + config "lib/logger/config" + logger_default "lib/logger/impl/default" + logger_empty "lib/logger/impl/empty" + logger_slog "lib/logger/impl/slog" ) +func New(conf config.ConfigT) (T, error) { + if *conf.Level == config.Nothing { + return logger_empty.New() + } + if *conf.Json { + return logger_slog.New(conf) + } else { + return logger_default.New(conf) + } +} + type T interface { LogError(string, ...any) LogWarning(string, ...any) LogDebug(string, ...any) LogInfo(string, ...any) } - -type ConfigT struct { - Level *LevelT `json:"level"` - Output *OutputT `json:"output"` - FileName *string `json:"filename"` -} - -type LevelT uint8 - -const ( - Debug LevelT = iota - Info - Warning - Error - Nothing -) - -func (l *LevelT) UnmarshalJSON(b []byte) error { - var s string - err := json.Unmarshal(b, &s) - if err != nil { - return err - } - switch s { - case "debug": - *l = Debug - case "info": - *l = Info - case "warning": - *l = Warning - case "error": - *l = Error - case "nothing": - *l = Nothing - default: - return fmt.Errorf("cannot unmarshal %s as log level", b) - } - return nil -} - -type OutputT uint8 - -const ( - Stdout OutputT = iota - File - Both -) - -func (o *OutputT) UnmarshalJSON(b []byte) error { - var s string - err := json.Unmarshal(b, &s) - if err != nil { - return err - } - switch s { - case "stdout": - *o = Stdout - case "file": - *o = File - case "both": - *o = Both - default: - return fmt.Errorf("cannot unmarshal %s as log output", b) - } - return nil -} diff --git a/src/lib/logger/new.go b/src/lib/logger/new.go deleted file mode 100644 index b20725d..0000000 --- a/src/lib/logger/new.go +++ /dev/null @@ -1,10 +0,0 @@ -package logger - -import ( - config "lib/logger/config" - def "lib/logger/impl/default" -) - -func New(conf config.ConfigT) (T, error) { - return def.New(conf) -} From c97afa8cb7d118c899b868bc0d83b680e11ba22b Mon Sep 17 00:00:00 2001 From: mejgun Date: Tue, 17 Sep 2024 21:35:45 +0300 Subject: [PATCH 11/29] rework default logger --- src/cmd/main.go | 2 +- src/lib/cache/cache.go | 63 +++------------------------ src/lib/cache/impl/default/default.go | 48 ++++++++++++++++++++ src/lib/cache/impl/empty/empty.go | 24 ++++++++++ src/lib/logger/impl/default/log.go | 28 +++++++++--- 5 files changed, 100 insertions(+), 65 deletions(-) create mode 100644 src/lib/cache/impl/default/default.go create mode 100644 src/lib/cache/impl/empty/empty.go diff --git a/src/cmd/main.go b/src/cmd/main.go index cc8ff68..39fcf59 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -110,7 +110,7 @@ func main() { Addr: ":" + port, } s.SetKeepAlivesEnabled(true) - log.LogInfo("Starting web server", port) + log.LogInfo("Starting web server", "port", port, "test") err = s.ListenAndServe() if err != nil { log.LogError("HTTP server start failed: ", err) diff --git a/src/lib/cache/cache.go b/src/lib/cache/cache.go index cd21cba..57792c6 100644 --- a/src/lib/cache/cache.go +++ b/src/lib/cache/cache.go @@ -2,9 +2,10 @@ package cache import ( "fmt" - "sync" "time" + cache_default "lib/cache/impl/default" + cache_empty "lib/cache/impl/empty" extractor "lib/extractor" logger "lib/logger" ) @@ -20,68 +21,14 @@ type ConfigT struct { } func New(conf ConfigT, log logger.T) (T, error) { - defCache := func(t time.Duration) *defaultCache { - return &defaultCache{ - cache: make(map[extractor.RequestT]extractor.ResultT), - expireTime: t, - } - } t, err := time.ParseDuration(*conf.ExpireTime) if err != nil { - return &defaultCache{}, err + return cache_default.New(0), err } if t.Seconds() < 1 { log.LogDebug("cache", "disabled by config") - return &emptyCache{}, nil + return cache_empty.New(), nil } log.LogDebug("cache", fmt.Sprintf("expire time set to %s", t)) - return defCache(t), nil -} - -type defaultCache struct { - sync.Mutex - cache map[extractor.RequestT]extractor.ResultT - expireTime time.Duration -} - -func (t *defaultCache) Add(req extractor.RequestT, res extractor.ResultT, - now time.Time) { - res.Expire = now.Add(t.expireTime) - t.Lock() - t.cache[req] = res - t.Unlock() -} - -func (t *defaultCache) Get(req extractor.RequestT) (extractor.ResultT, bool) { - t.Lock() - defer t.Unlock() - v, ok := t.cache[req] - return v, ok -} - -func (t *defaultCache) CleanExpired(now time.Time) []extractor.ResultT { - deleted := make([]extractor.ResultT, 0) - t.Lock() - for k, v := range t.cache { - if v.Expire.Before(now) { - delete(t.cache, k) - deleted = append(deleted, v) - } - } - t.Unlock() - return deleted -} - -type emptyCache struct{} - -func (t *emptyCache) Add(req extractor.RequestT, res extractor.ResultT, - now time.Time) { -} - -func (t *emptyCache) Get(req extractor.RequestT) (extractor.ResultT, bool) { - return extractor.ResultT{}, false -} - -func (t *emptyCache) CleanExpired(now time.Time) []extractor.ResultT { - return []extractor.ResultT{} + return cache_default.New(0), nil } diff --git a/src/lib/cache/impl/default/default.go b/src/lib/cache/impl/default/default.go new file mode 100644 index 0000000..833cc1d --- /dev/null +++ b/src/lib/cache/impl/default/default.go @@ -0,0 +1,48 @@ +package cache + +import ( + "lib/extractor" + "sync" + "time" +) + +func New(t time.Duration) *defaultCache { + return &defaultCache{ + cache: make(map[extractor.RequestT]extractor.ResultT), + expireTime: t, + } +} + +type defaultCache struct { + sync.Mutex + cache map[extractor.RequestT]extractor.ResultT + expireTime time.Duration +} + +func (t *defaultCache) Add(req extractor.RequestT, res extractor.ResultT, + now time.Time) { + res.Expire = now.Add(t.expireTime) + t.Lock() + t.cache[req] = res + t.Unlock() +} + +func (t *defaultCache) Get(req extractor.RequestT) (extractor.ResultT, bool) { + t.Lock() + defer t.Unlock() + v, ok := t.cache[req] + return v, ok +} + +func (t *defaultCache) CleanExpired(now time.Time) []extractor.ResultT { + deleted := make([]extractor.ResultT, 0) + t.Lock() + for k, v := range t.cache { + if v.Expire.Before(now) { + delete(t.cache, k) + deleted = append(deleted, v) + } + } + t.Unlock() + return deleted +} diff --git a/src/lib/cache/impl/empty/empty.go b/src/lib/cache/impl/empty/empty.go new file mode 100644 index 0000000..52de905 --- /dev/null +++ b/src/lib/cache/impl/empty/empty.go @@ -0,0 +1,24 @@ +package cache + +import ( + "lib/extractor" + "time" +) + +func New() *emptyCache { + return &emptyCache{} +} + +type emptyCache struct{} + +func (t *emptyCache) Add(req extractor.RequestT, res extractor.ResultT, + now time.Time) { +} + +func (t *emptyCache) Get(req extractor.RequestT) (extractor.ResultT, bool) { + return extractor.ResultT{}, false +} + +func (t *emptyCache) CleanExpired(now time.Time) []extractor.ResultT { + return []extractor.ResultT{} +} diff --git a/src/lib/logger/impl/default/log.go b/src/lib/logger/impl/default/log.go index 71fd295..1ca2548 100644 --- a/src/lib/logger/impl/default/log.go +++ b/src/lib/logger/impl/default/log.go @@ -5,7 +5,6 @@ import ( "io" "log" "os" - "strings" l "lib/logger/config" ) @@ -15,13 +14,30 @@ type loggerT struct { lgr *log.Logger } -func (t *loggerT) print(str string, s string, i []interface{}) { - t.lgr.Printf( - fmt.Sprintf("%-7s %s:", str, s) + - fmt.Sprintf(strings.Repeat(" %+v", len(i)), i...)) +func (t *loggerT) print(str string, s string, args []any) { + var ( + fmtstr string + arglen = len(args) + k int + ) + add := func(a string) { + if len(a) > 0 { + fmtstr = fmt.Sprintf("%s %s", fmtstr, a) + } + } + for k < arglen { + if k+1 < arglen { + add(fmt.Sprintf("%s=%+v", args[k], args[k+1])) + k = k + 2 + } else { + add(fmt.Sprintf("%+v", args[k])) + k++ + } + } + t.lgr.Println(fmt.Sprintf("%s %s.", str, s) + fmtstr) } -func (t *loggerT) checkAndPrint(lvl l.LevelT, str string, s string, i ...any) { +func (t *loggerT) checkAndPrint(lvl l.LevelT, str string, s string, i []any) { if *t.lvl <= lvl { t.print(str, s, i) } From bdef1ddaa1e9ee730adea80a32f8ab778cc98cfc Mon Sep 17 00:00:00 2001 From: mejgun Date: Tue, 17 Sep 2024 22:37:27 +0300 Subject: [PATCH 12/29] rework extractor --- src/cmd/main.go | 19 ++-- src/cmd/main_test.go | 4 +- src/lib/cache/cache.go | 8 +- src/lib/cache/impl/default/default.go | 15 +-- src/lib/cache/impl/empty/empty.go | 13 ++- src/lib/config/config.go | 14 +-- src/lib/extractor/config/config.go | 21 ++++ src/lib/extractor/extractor.go | 133 +++------------------- src/lib/extractor/impl/default/default.go | 126 ++++++++++++++++++++ src/lib/logger/impl/default/log.go | 6 +- src/lib/streamer/streamer.go | 12 +- 11 files changed, 210 insertions(+), 161 deletions(-) create mode 100644 src/lib/extractor/config/config.go create mode 100644 src/lib/extractor/impl/default/default.go diff --git a/src/cmd/main.go b/src/cmd/main.go index 39fcf59..1762aca 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -12,6 +12,7 @@ import ( cache "lib/cache" config "lib/config" extractor "lib/extractor" + extractor_config "lib/extractor/config" logger "lib/logger" streamer "lib/streamer" ) @@ -67,19 +68,22 @@ func main() { log, err := logger.New(conf.Log) checkOrExit(err, "Logger", LoggerError) - log.LogDebug("logger created") + status := func(s string) { + log.LogDebug("App starting", "status", s) + } + status("logger created") extr, err := extractor.New(conf.Extractor, log) checkOrExit(err, "Extractor", ExtractorError) - log.LogDebug("extractor created") + status("extractor created") cache, err := cache.New(conf.Cache, log) checkOrExit(err, "Cache", CacheError) - log.LogDebug("cache created") + status("cache created") restreamer, err := streamer.New(conf.Streamer, log, extr) checkOrExit(err, "Streamer", StreamerError) - log.LogDebug("streamer created") + status("streamer created") http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { log.LogInfo("Bad request", r.RemoteAddr, r.RequestURI) @@ -109,7 +113,6 @@ func main() { s := &http.Server{ Addr: ":" + port, } - s.SetKeepAlivesEnabled(true) log.LogInfo("Starting web server", "port", port, "test") err = s.ListenAndServe() if err != nil { @@ -119,7 +122,7 @@ func main() { } func getLink(query string, log logger.T, cache cache.T, - extractor extractor.T) (extractor.RequestT, extractor.ResultT, error) { + extractor extractor.T) (extractor_config.RequestT, extractor_config.ResultT, error) { now := time.Now() req := parseQuery(query) for _, v := range cache.CleanExpired(now) { @@ -139,8 +142,8 @@ func getLink(query string, log logger.T, cache cache.T, return req, res, nil } -func parseQuery(query string) extractor.RequestT { - var req extractor.RequestT +func parseQuery(query string) extractor_config.RequestT { + var req extractor_config.RequestT query = strings.TrimSpace(strings.TrimPrefix(query, "/play/")) splitted := strings.Split(query, "?/?") req.URL = splitted[0] diff --git a/src/cmd/main_test.go b/src/cmd/main_test.go index 9bf874d..0897de4 100644 --- a/src/cmd/main_test.go +++ b/src/cmd/main_test.go @@ -3,10 +3,10 @@ package main import ( "testing" - extractor "lib/extractor" + extractor_config "lib/extractor/config" ) -var testPairs = map[string]extractor.RequestT{ +var testPairs = map[string]extractor_config.RequestT{ "/play/youtu.be/jNQXAC9IVRw?/?vh=360&vf=mp4": { URL: "youtu.be/jNQXAC9IVRw", HEIGHT: "360", diff --git a/src/lib/cache/cache.go b/src/lib/cache/cache.go index 57792c6..42a10ea 100644 --- a/src/lib/cache/cache.go +++ b/src/lib/cache/cache.go @@ -6,14 +6,14 @@ import ( cache_default "lib/cache/impl/default" cache_empty "lib/cache/impl/empty" - extractor "lib/extractor" + extractor_config "lib/extractor/config" logger "lib/logger" ) type T interface { - Add(extractor.RequestT, extractor.ResultT, time.Time) - Get(extractor.RequestT) (extractor.ResultT, bool) - CleanExpired(time.Time) []extractor.ResultT + Add(extractor_config.RequestT, extractor_config.ResultT, time.Time) + Get(extractor_config.RequestT) (extractor_config.ResultT, bool) + CleanExpired(time.Time) []extractor_config.ResultT } type ConfigT struct { diff --git a/src/lib/cache/impl/default/default.go b/src/lib/cache/impl/default/default.go index 833cc1d..3e5847e 100644 --- a/src/lib/cache/impl/default/default.go +++ b/src/lib/cache/impl/default/default.go @@ -1,25 +1,26 @@ package cache import ( - "lib/extractor" "sync" "time" + + extractor_config "lib/extractor/config" ) func New(t time.Duration) *defaultCache { return &defaultCache{ - cache: make(map[extractor.RequestT]extractor.ResultT), + cache: make(map[extractor_config.RequestT]extractor_config.ResultT), expireTime: t, } } type defaultCache struct { sync.Mutex - cache map[extractor.RequestT]extractor.ResultT + cache map[extractor_config.RequestT]extractor_config.ResultT expireTime time.Duration } -func (t *defaultCache) Add(req extractor.RequestT, res extractor.ResultT, +func (t *defaultCache) Add(req extractor_config.RequestT, res extractor_config.ResultT, now time.Time) { res.Expire = now.Add(t.expireTime) t.Lock() @@ -27,15 +28,15 @@ func (t *defaultCache) Add(req extractor.RequestT, res extractor.ResultT, t.Unlock() } -func (t *defaultCache) Get(req extractor.RequestT) (extractor.ResultT, bool) { +func (t *defaultCache) Get(req extractor_config.RequestT) (extractor_config.ResultT, bool) { t.Lock() defer t.Unlock() v, ok := t.cache[req] return v, ok } -func (t *defaultCache) CleanExpired(now time.Time) []extractor.ResultT { - deleted := make([]extractor.ResultT, 0) +func (t *defaultCache) CleanExpired(now time.Time) []extractor_config.ResultT { + deleted := make([]extractor_config.ResultT, 0) t.Lock() for k, v := range t.cache { if v.Expire.Before(now) { diff --git a/src/lib/cache/impl/empty/empty.go b/src/lib/cache/impl/empty/empty.go index 52de905..39de86d 100644 --- a/src/lib/cache/impl/empty/empty.go +++ b/src/lib/cache/impl/empty/empty.go @@ -1,8 +1,9 @@ package cache import ( - "lib/extractor" "time" + + extractor_config "lib/extractor/config" ) func New() *emptyCache { @@ -11,14 +12,14 @@ func New() *emptyCache { type emptyCache struct{} -func (t *emptyCache) Add(req extractor.RequestT, res extractor.ResultT, +func (t *emptyCache) Add(req extractor_config.RequestT, res extractor_config.ResultT, now time.Time) { } -func (t *emptyCache) Get(req extractor.RequestT) (extractor.ResultT, bool) { - return extractor.ResultT{}, false +func (t *emptyCache) Get(req extractor_config.RequestT) (extractor_config.ResultT, bool) { + return extractor_config.ResultT{}, false } -func (t *emptyCache) CleanExpired(now time.Time) []extractor.ResultT { - return []extractor.ResultT{} +func (t *emptyCache) CleanExpired(now time.Time) []extractor_config.ResultT { + return []extractor_config.ResultT{} } diff --git a/src/lib/config/config.go b/src/lib/config/config.go index 9276dce..f171d59 100644 --- a/src/lib/config/config.go +++ b/src/lib/config/config.go @@ -6,17 +6,17 @@ import ( "strings" cache "lib/cache" - extractor "lib/extractor" + extractor_config "lib/extractor/config" logger_config "lib/logger/config" streamer "lib/streamer" ) type configT struct { - PortInt uint16 `json:"port"` - Streamer streamer.ConfigT `json:"streamer"` - Extractor extractor.ConfigT `json:"extractor"` - Log logger_config.ConfigT `json:"log"` - Cache cache.ConfigT `json:"cache"` + PortInt uint16 `json:"port"` + Streamer streamer.ConfigT `json:"streamer"` + Extractor extractor_config.ConfigT `json:"extractor"` + Log logger_config.ConfigT `json:"log"` + Cache cache.ConfigT `json:"cache"` } func defaultConfig() configT { @@ -51,7 +51,7 @@ func defaultConfig() configT { Proxy: &s[3], MinTlsVersion: &tv, }, - Extractor: extractor.ConfigT{ + Extractor: extractor_config.ConfigT{ Path: &e[0], MP4: &e[1], M4A: &e[2], diff --git a/src/lib/extractor/config/config.go b/src/lib/extractor/config/config.go new file mode 100644 index 0000000..26fd293 --- /dev/null +++ b/src/lib/extractor/config/config.go @@ -0,0 +1,21 @@ +package extractor + +import "time" + +type ConfigT struct { + Path *string `json:"path"` + MP4 *string `json:"mp4"` + M4A *string `json:"m4a"` + GetUserAgent *string `json:"get-user-agent"` + CustomOptions *[]string `json:"custom-options"` +} + +type ResultT struct { + URL string + Expire time.Time +} +type RequestT struct { + URL string + HEIGHT string + FORMAT string +} diff --git a/src/lib/extractor/extractor.go b/src/lib/extractor/extractor.go index dabbbd4..5d3abd6 100644 --- a/src/lib/extractor/extractor.go +++ b/src/lib/extractor/extractor.go @@ -1,140 +1,35 @@ package extractor import ( - "bytes" - "errors" - "fmt" - "os/exec" "strings" - "sync" - "text/template" - "time" + extractor_config "lib/extractor/config" + extractor_default "lib/extractor/impl/default" logger "lib/logger" ) const separator = ",," -type ConfigT struct { - Path *string `json:"path"` - MP4 *string `json:"mp4"` - M4A *string `json:"m4a"` - GetUserAgent *string `json:"get-user-agent"` - CustomOptions *[]string `json:"custom-options"` -} - -type ResultT struct { - URL string - Expire time.Time -} - type T interface { - Extract(RequestT) (ResultT, error) + Extract(extractor_config.RequestT) (extractor_config.ResultT, error) GetUserAgent() (string, error) } -type defaultExtractor struct { - sync.Mutex - path string - mp4 *template.Template - m4a *template.Template - customOptions []*template.Template - getUserAgent string - logger logger.T -} - -type RequestT struct { - URL string - HEIGHT string - FORMAT string -} - -func New(c ConfigT, log logger.T) (T, error) { - var ( - e defaultExtractor - err error - ) - e.m4a, err = template.New("").Parse(*c.M4A) - if err != nil { - return &e, err - } - e.mp4, err = template.New("").Parse(*c.MP4) - if err != nil { - return &e, err - } - e.customOptions = make([]*template.Template, 0) +func New(c extractor_config.ConfigT, log logger.T) (T, error) { + co := make([]string, 0) for _, v := range *c.CustomOptions { - b, err := template.New("").Parse(v) - if err != nil { - return &e, err - } - e.customOptions = append(e.customOptions, b) - } - e.getUserAgent = *c.GetUserAgent - e.path = *c.Path - e.logger = log - return &e, nil -} - -func (t *defaultExtractor) GetUserAgent() (string, error) { - return t.runCmd(t.getUserAgent) -} - -func (t *defaultExtractor) Extract(req RequestT) (ResultT, error) { - var ( - buf bytes.Buffer - err error + co = append(co, split(v)...) + } + return extractor_default.New( + *c.Path, + split(*c.MP4), + split(*c.M4A), + *c.GetUserAgent, + co, + log, ) - switch req.FORMAT { - case "m4a": - err = t.m4a.Execute(&buf, req) - case "mp4": - fallthrough - default: - err = t.mp4.Execute(&buf, req) - } - if err != nil { - return ResultT{}, err - } - bufOptions := make([]string, 0) - for _, v := range t.customOptions { - var b bytes.Buffer - err = v.Execute(&b, req) - if err != nil { - return ResultT{}, err - } - bufOptions = append(bufOptions, bytesToString(b)) - } - bufOptions = append(bufOptions, bytesToString(buf)) - out, err := t.runCmd(strings.Join(bufOptions, separator)) - if err != nil { - return ResultT{}, err - } - return ResultT{URL: out}, err -} - -func (t *defaultExtractor) runCmd(args string) (string, error) { - realargs := split(args) - t.Lock() - defer t.Unlock() - cmd := exec.Command(t.path, realargs...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - t.logger.LogDebug("Running", t.path, strings.Join(realargs, " ")) - err := cmd.Run() - outStr, errStr := bytesToString(stdout), bytesToString(stderr) - if err != nil { - combinedErrStr := fmt.Sprintf("%s\n%s\n%s", err.Error(), outStr, errStr) - return "", errors.New(combinedErrStr) - } - return outStr, nil } func split(s string) []string { return strings.Split(s, separator) } - -func bytesToString(s bytes.Buffer) string { - return strings.TrimSpace(s.String()) -} diff --git a/src/lib/extractor/impl/default/default.go b/src/lib/extractor/impl/default/default.go new file mode 100644 index 0000000..1ac11d2 --- /dev/null +++ b/src/lib/extractor/impl/default/default.go @@ -0,0 +1,126 @@ +package extractor + +import ( + "bytes" + "errors" + "fmt" + "os/exec" + "strings" + "sync" + "text/template" + + extractor_config "lib/extractor/config" + logger "lib/logger" +) + +func New(path string, mp4, m4a []string, get_user_agent string, + custom_options []string, log logger.T) (*defaultExtractor, error) { + var ( + e defaultExtractor + err error + ) + read := func(list []string) ([]*template.Template, error) { + res := make([]*template.Template, 0) + for _, v := range list { + t, err := template.New("").Parse(v) + if err != nil { + return res, err + } + res = append(res, t) + } + return res, nil + } + e.m4a, err = read(m4a) + if err != nil { + return &e, err + } + e.mp4, err = read(mp4) + if err != nil { + return &e, err + } + e.customOptions, err = read(custom_options) + if err != nil { + return &e, err + } + e.getUserAgent = get_user_agent + e.path = path + e.logger = log + return &e, nil +} + +type defaultExtractor struct { + sync.Mutex + path string + mp4 []*template.Template + m4a []*template.Template + customOptions []*template.Template + getUserAgent string + logger logger.T +} + +func (t *defaultExtractor) GetUserAgent() (string, error) { + return t.runCmd([]string{t.getUserAgent}) +} + +func (t *defaultExtractor) Extract(req extractor_config.RequestT, +) (extractor_config.ResultT, error) { + var ( + buf []string + bufOptions []string + err error + ) + execute := func(list []*template.Template) ([]string, error) { + buf := make([]string, 0) + for _, v := range list { + var b bytes.Buffer + err = v.Execute(&b, req) + if err != nil { + return buf, err + } + buf = append(buf, bytesToString(b)) + } + return buf, nil + } + switch req.FORMAT { + case "m4a": + buf, err = execute(t.m4a) + case "mp4": + fallthrough + default: + buf, err = execute(t.mp4) + } + if err != nil { + return extractor_config.ResultT{}, err + } + bufOptions, err = execute(t.customOptions) + if err != nil { + return extractor_config.ResultT{}, err + } + bufOptions = append(bufOptions, buf...) + out, err := t.runCmd(bufOptions) + if err != nil { + return extractor_config.ResultT{}, err + } + return extractor_config.ResultT{URL: out}, err +} + +func (t *defaultExtractor) runCmd(args []string) (string, error) { + t.Lock() + defer t.Unlock() + cmd := exec.Command(t.path, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + t.logger.LogDebug("Running", "path", t.path, "args", strings.Join(args, " ")) + err := cmd.Run() + outStr, errStr := bytesToString(stdout), bytesToString(stderr) + if err != nil { + combinedErrStr := fmt.Sprintf("%s\n%s\n%s", err.Error(), outStr, errStr) + return "", errors.New(combinedErrStr) + } + return outStr, nil +} + +func bytesToString(s bytes.Buffer) string { + return strings.TrimSpace(s.String()) +} diff --git a/src/lib/logger/impl/default/log.go b/src/lib/logger/impl/default/log.go index 1ca2548..0c81b42 100644 --- a/src/lib/logger/impl/default/log.go +++ b/src/lib/logger/impl/default/log.go @@ -27,14 +27,14 @@ func (t *loggerT) print(str string, s string, args []any) { } for k < arglen { if k+1 < arglen { - add(fmt.Sprintf("%s=%+v", args[k], args[k+1])) + add(fmt.Sprintf("%s=%s", args[k], args[k+1])) k = k + 2 } else { - add(fmt.Sprintf("%+v", args[k])) + add(fmt.Sprintf("%s", args[k])) k++ } } - t.lgr.Println(fmt.Sprintf("%s %s.", str, s) + fmtstr) + t.lgr.Println(fmt.Sprintf("%-7s %s.", str, s) + fmtstr) } func (t *loggerT) checkAndPrint(lvl l.LevelT, str string, s string, i []any) { diff --git a/src/lib/streamer/streamer.go b/src/lib/streamer/streamer.go index 395333d..2fbe532 100644 --- a/src/lib/streamer/streamer.go +++ b/src/lib/streamer/streamer.go @@ -12,6 +12,7 @@ import ( "strings" extractor "lib/extractor" + extractor_config "lib/extractor/config" logger "lib/logger" ) @@ -85,8 +86,8 @@ func (u *SetUserAgentT) UnmarshalJSON(b []byte) error { } type T interface { - Play(http.ResponseWriter, *http.Request, extractor.RequestT, extractor.ResultT) error - PlayError(http.ResponseWriter, extractor.RequestT, error) error + Play(http.ResponseWriter, *http.Request, extractor_config.RequestT, extractor_config.ResultT) error + PlayError(http.ResponseWriter, extractor_config.RequestT, error) error } type streamer struct { @@ -146,8 +147,8 @@ func New(conf ConfigT, log logger.T, xt extractor.T) (T, error) { func (t *streamer) Play( w http.ResponseWriter, req *http.Request, - reqst extractor.RequestT, - rest extractor.ResultT, + reqst extractor_config.RequestT, + rest extractor_config.ResultT, ) error { // t.log.LogDebug("Streamer request", rest) // fail := func(str string, err error) { @@ -181,7 +182,8 @@ func (t *streamer) Play( return nil } -func (t *streamer) PlayError(w http.ResponseWriter, req extractor.RequestT, err error) error { +func (t *streamer) PlayError(w http.ResponseWriter, req extractor_config.RequestT, + err error) error { var file *fileT if req.FORMAT == "mp4" { file = &t.errorVideoFile From d1f8c8fd54f04d874d2cdb010174ad52d98ed411 Mon Sep 17 00:00:00 2001 From: mejgun Date: Tue, 17 Sep 2024 22:45:49 +0300 Subject: [PATCH 13/29] add direct extractor --- src/lib/extractor/extractor.go | 28 +++++++++++++++---------- src/lib/extractor/impl/direct/direct.go | 21 +++++++++++++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 src/lib/extractor/impl/direct/direct.go diff --git a/src/lib/extractor/extractor.go b/src/lib/extractor/extractor.go index 5d3abd6..61864d6 100644 --- a/src/lib/extractor/extractor.go +++ b/src/lib/extractor/extractor.go @@ -5,6 +5,7 @@ import ( extractor_config "lib/extractor/config" extractor_default "lib/extractor/impl/default" + extractor_direct "lib/extractor/impl/direct" logger "lib/logger" ) @@ -16,18 +17,23 @@ type T interface { } func New(c extractor_config.ConfigT, log logger.T) (T, error) { - co := make([]string, 0) - for _, v := range *c.CustomOptions { - co = append(co, split(v)...) + switch *c.Path { + case "direct": + return extractor_direct.New() + default: + co := make([]string, 0) + for _, v := range *c.CustomOptions { + co = append(co, split(v)...) + } + return extractor_default.New( + *c.Path, + split(*c.MP4), + split(*c.M4A), + *c.GetUserAgent, + co, + log, + ) } - return extractor_default.New( - *c.Path, - split(*c.MP4), - split(*c.M4A), - *c.GetUserAgent, - co, - log, - ) } func split(s string) []string { diff --git a/src/lib/extractor/impl/direct/direct.go b/src/lib/extractor/impl/direct/direct.go new file mode 100644 index 0000000..c25ecd9 --- /dev/null +++ b/src/lib/extractor/impl/direct/direct.go @@ -0,0 +1,21 @@ +package extractor + +import ( + extractor_config "lib/extractor/config" +) + +func New() (*directExtractor, error) { + return &directExtractor{}, nil +} + +type directExtractor struct { +} + +func (t *directExtractor) GetUserAgent() (string, error) { + return "Mozilla", nil +} + +func (t *directExtractor) Extract(req extractor_config.RequestT, +) (extractor_config.ResultT, error) { + return extractor_config.ResultT{URL: req.URL}, nil +} From c92a872f7850d88fd127570dd3d0f4c3bf4fd37a Mon Sep 17 00:00:00 2001 From: mejgun Date: Tue, 17 Sep 2024 23:02:31 +0300 Subject: [PATCH 14/29] fix http prefix & force https --- src/cmd/main.go | 11 +++++++++-- src/cmd/main_test.go | 21 +++++++++++++++++++++ src/lib/logger/impl/default/log.go | 4 ++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/cmd/main.go b/src/cmd/main.go index 1762aca..1bc1254 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -50,7 +50,7 @@ func parseCLIFlags() flagsT { func main() { stdout := func(s string) { os.Stdout.WriteString(fmt.Sprintf("%s\n", s)) } - stderr := func(s string) { os.Stderr.WriteString(fmt.Sprintf("[ ERROR ] %s\n", s)) } + stderr := func(s string) { os.Stderr.WriteString(fmt.Sprintf("ERROR %s\n", s)) } flags := parseCLIFlags() if flags.version { stdout(appVersion) @@ -142,11 +142,18 @@ func getLink(query string, log logger.T, cache cache.T, return req, res, nil } +func remove_http(url string) string { + url = strings.TrimPrefix(url, "http:/") + url = strings.TrimPrefix(url, "https:/") + url = strings.TrimLeft(url, "/") + return url +} + func parseQuery(query string) extractor_config.RequestT { var req extractor_config.RequestT query = strings.TrimSpace(strings.TrimPrefix(query, "/play/")) splitted := strings.Split(query, "?/?") - req.URL = splitted[0] + req.URL = "https://" + splitted[0] req.HEIGHT = defaultVideoHeight req.FORMAT = defaultVideoFormat if len(splitted) != 2 { diff --git a/src/cmd/main_test.go b/src/cmd/main_test.go index 0897de4..c9b32e7 100644 --- a/src/cmd/main_test.go +++ b/src/cmd/main_test.go @@ -46,3 +46,24 @@ func TestParseQuery(t *testing.T) { } } } + +func TestRemoveHttp(t *testing.T) { + for _, v := range []struct { + link string + want string + }{ + {link: "youtu.be/", want: "youtu.be/"}, + {link: "https:////youtu.be/", want: "youtu.be/"}, + {link: "https:///youtu.be/", want: "youtu.be/"}, + {link: "https://youtu.be/", want: "youtu.be/"}, + {link: "https:/youtu.be/", want: "youtu.be/"}, + {link: "http:////www.youtu.be/", want: "www.youtu.be/"}, + {link: "http:///www.youtu.be/", want: "www.youtu.be/"}, + {link: "http://www.youtu.be/", want: "www.youtu.be/"}, + {link: "http:/www.youtu.be/", want: "www.youtu.be/"}, + } { + if r := remove_http(v.link); r != v.want { + t.Error("For", v.link, "expected", v.want, "got", r) + } + } +} diff --git a/src/lib/logger/impl/default/log.go b/src/lib/logger/impl/default/log.go index 0c81b42..e504ba4 100644 --- a/src/lib/logger/impl/default/log.go +++ b/src/lib/logger/impl/default/log.go @@ -27,10 +27,10 @@ func (t *loggerT) print(str string, s string, args []any) { } for k < arglen { if k+1 < arglen { - add(fmt.Sprintf("%s=%s", args[k], args[k+1])) + add(fmt.Sprintf("%s=%+v", args[k], args[k+1])) k = k + 2 } else { - add(fmt.Sprintf("%s", args[k])) + add(fmt.Sprintf("%+v", args[k])) k++ } } From 2715e5165702489d5586f957d035278fc5e23417 Mon Sep 17 00:00:00 2001 From: mejgun Date: Wed, 18 Sep 2024 12:55:04 +0300 Subject: [PATCH 15/29] add extractor force http links option --- src/cmd/main.go | 2 +- src/lib/config/config.go | 5 +++++ src/lib/extractor/config/config.go | 1 + src/lib/extractor/extractor.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/cmd/main.go b/src/cmd/main.go index 1bc1254..2dfd64a 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -153,7 +153,7 @@ func parseQuery(query string) extractor_config.RequestT { var req extractor_config.RequestT query = strings.TrimSpace(strings.TrimPrefix(query, "/play/")) splitted := strings.Split(query, "?/?") - req.URL = "https://" + splitted[0] + req.URL = splitted[0] req.HEIGHT = defaultVideoHeight req.FORMAT = defaultVideoFormat if len(splitted) != 2 { diff --git a/src/lib/config/config.go b/src/lib/config/config.go index f171d59..f40c8a0 100644 --- a/src/lib/config/config.go +++ b/src/lib/config/config.go @@ -21,6 +21,7 @@ type configT struct { func defaultConfig() configT { fls := false + tru := true ext := streamer.Extractor tv := streamer.TlsVersion(0) var s = [4]string{"corrupted.mp4", @@ -57,6 +58,7 @@ func defaultConfig() configT { M4A: &e[2], GetUserAgent: &e[3], CustomOptions: &co, + ForceHttp: &tru, }, Log: logger_config.ConfigT{ Level: &ll, @@ -120,6 +122,9 @@ func appendConfig(src configT, dst configT) configT { if dst.Extractor.CustomOptions == nil { dst.Extractor.CustomOptions = src.Extractor.CustomOptions } + if dst.Extractor.ForceHttp == nil { + dst.Extractor.ForceHttp = src.Extractor.ForceHttp + } // logger if dst.Log.Level == nil { dst.Log.Level = src.Log.Level diff --git a/src/lib/extractor/config/config.go b/src/lib/extractor/config/config.go index 26fd293..7f384f6 100644 --- a/src/lib/extractor/config/config.go +++ b/src/lib/extractor/config/config.go @@ -8,6 +8,7 @@ type ConfigT struct { M4A *string `json:"m4a"` GetUserAgent *string `json:"get-user-agent"` CustomOptions *[]string `json:"custom-options"` + ForceHttp *bool `json:"force-http"` } type ResultT struct { diff --git a/src/lib/extractor/extractor.go b/src/lib/extractor/extractor.go index 61864d6..d0dfe7e 100644 --- a/src/lib/extractor/extractor.go +++ b/src/lib/extractor/extractor.go @@ -17,6 +17,35 @@ type T interface { } func New(c extractor_config.ConfigT, log logger.T) (T, error) { + var ( + ext layer + err error + ) + ext.force_http = *c.ForceHttp + if ext.force_http { + log.LogDebug("extractor", "force-http", true) + } + ext.impl, err = real_new(c, log) + return &ext, err +} + +type layer struct { + impl T + force_http bool +} + +func (t *layer) Extract(req extractor_config.RequestT) (extractor_config.ResultT, error) { + if t.force_http { + req.URL = "https://" + req.URL + } + return t.impl.Extract(req) +} + +func (t *layer) GetUserAgent() (string, error) { + return t.impl.GetUserAgent() +} + +func real_new(c extractor_config.ConfigT, log logger.T) (T, error) { switch *c.Path { case "direct": return extractor_direct.New() From 52ab946c1022a00ec7d4be59cd7d0f47f925d08e Mon Sep 17 00:00:00 2001 From: mejgun Date: Wed, 18 Sep 2024 13:13:57 +0300 Subject: [PATCH 16/29] changelog edit --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d04fde..f27d11f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +- app refactored & reworked +### Added +- direct extractor (returning same url) +- json format logs +- disabling logs +- force https links to extractor options +- stripping (bad) http(s) prefix in url ## 1.6.0 - 2024-09-13 ### Added From b507e1cfbd841ae3825f1544cdc404838e1149ee Mon Sep 17 00:00:00 2001 From: mejgun Date: Wed, 18 Sep 2024 15:50:47 +0300 Subject: [PATCH 17/29] move logic to own module --- src/cmd/main.go | 84 +----------- src/lib/app/app.go | 129 ++++++++++++++++++ src/{cmd/main_test.go => lib/app/app_test.go} | 2 +- 3 files changed, 135 insertions(+), 80 deletions(-) create mode 100644 src/lib/app/app.go rename src/{cmd/main_test.go => lib/app/app_test.go} (99%) diff --git a/src/cmd/main.go b/src/cmd/main.go index 2dfd64a..4fbef29 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -4,26 +4,17 @@ import ( "flag" "fmt" "net/http" - "net/url" "os" - "strings" - "time" + app "lib/app" cache "lib/cache" config "lib/config" extractor "lib/extractor" - extractor_config "lib/extractor/config" logger "lib/logger" streamer "lib/streamer" ) -const appVersion = "1.6.0" - -const ( - defaultVideoHeight = "720" - defaultVideoFormat = "mp4" - defaultExpireTime = 3 * 60 * 60 -) +const appVersion = "2.0.0" type flagsT struct { version bool @@ -85,6 +76,8 @@ func main() { checkOrExit(err, "Streamer", StreamerError) status("streamer created") + app := app.New(log, cache, extr, restreamer) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { log.LogInfo("Bad request", r.RemoteAddr, r.RequestURI) log.LogDebug("Bad request", r) @@ -93,20 +86,7 @@ func main() { http.HandleFunc("/play/", func(w http.ResponseWriter, r *http.Request) { log.LogInfo("Play request", r.RemoteAddr, r.RequestURI) log.LogDebug("User request", r) - req, res, err := getLink(r.RequestURI, log, cache, extr) - if err != nil { - log.LogError("URL extract error", err) - restreamer.PlayError(w, req, err) - log.LogInfo("URL extract failed. Disconnecting", r.RemoteAddr) - return - } - err = restreamer.Play(w, r, req, res) - if err != nil { - log.LogError("Restream error", err) - restreamer.PlayError(w, req, err) - log.LogInfo("URL Restream failed. Disconnecting", r.RemoteAddr) - return - } + app.Run(w, r) log.LogInfo("Player disconnected", r.RemoteAddr) }) port := fmt.Sprintf("%d", conf.PortInt) @@ -120,57 +100,3 @@ func main() { os.Exit(WebServerError) } } - -func getLink(query string, log logger.T, cache cache.T, - extractor extractor.T) (extractor_config.RequestT, extractor_config.ResultT, error) { - now := time.Now() - req := parseQuery(query) - for _, v := range cache.CleanExpired(now) { - log.LogDebug("Clean expired cache", v) - } - log.LogInfo("Request", req) - if lnk, ok := cache.Get(req); ok { - return req, lnk, nil - } - res, err := extractor.Extract(req) - log.LogDebug("Not cached. Extractor returned", res) - if err != nil { - return req, res, err - } - cache.Add(req, res, now) - log.LogDebug("Cache add", res) - return req, res, nil -} - -func remove_http(url string) string { - url = strings.TrimPrefix(url, "http:/") - url = strings.TrimPrefix(url, "https:/") - url = strings.TrimLeft(url, "/") - return url -} - -func parseQuery(query string) extractor_config.RequestT { - var req extractor_config.RequestT - query = strings.TrimSpace(strings.TrimPrefix(query, "/play/")) - splitted := strings.Split(query, "?/?") - req.URL = splitted[0] - req.HEIGHT = defaultVideoHeight - req.FORMAT = defaultVideoFormat - if len(splitted) != 2 { - return req - } - tOpts, tErr := url.ParseQuery(splitted[1]) - if tErr == nil { - if tvh, ok := tOpts["vh"]; ok { - if tvh[0] == "360" || tvh[0] == "480" || tvh[0] == "720" { - req.HEIGHT = tvh[0] - } - } - if tvf, ok := tOpts["vf"]; ok { - if tvf[0] == "mp4" || tvf[0] == "m4a" { - req.FORMAT = tvf[0] - } - } - } - return req -} diff --git a/src/lib/app/app.go b/src/lib/app/app.go new file mode 100644 index 0000000..589cc92 --- /dev/null +++ b/src/lib/app/app.go @@ -0,0 +1,129 @@ +package app + +import ( + cache "lib/cache" + extractor "lib/extractor" + extractor_config "lib/extractor/config" + logger "lib/logger" + streamer "lib/streamer" + + "net/http" + "net/url" + "strings" + "time" +) + +const ( + defaultVideoHeight = "720" + defaultVideoFormat = "mp4" +) + +type T struct { + log logger.T + cache cache.T + extractor extractor.T + streamer streamer.T +} + +func New( + l logger.T, + c cache.T, + x extractor.T, + s streamer.T) *T { + return &T{ + log: l, + cache: c, + extractor: x, + streamer: s, + } +} + +func (t *T) Run(w http.ResponseWriter, r *http.Request) { + now := time.Now() + req := parseQuery(r.RequestURI) + t.log.LogInfo("Request", req) + if res, ok := t.cacheCheck(req, now); ok { + t.log.LogDebug("Link already cached", res) + t.play(w, r, req, res) + } else { + res, err := t.extractor.Extract(req) + if err != nil { + t.log.LogError("URL extract error", err) + t.playError(w, req, err) + } + t.log.LogDebug("Extractor returned", res) + t.cacheAdd(req, res, now) + t.play(w, r, req, res) + } +} + +func (t *T) play( + w http.ResponseWriter, + r *http.Request, + req extractor_config.RequestT, + res extractor_config.ResultT, +) { + if err := t.streamer.Play(w, r, req, res); err != nil { + t.log.LogError("Restream error", err) + t.playError(w, req, err) + } +} + +func (t *T) playError( + w http.ResponseWriter, + req extractor_config.RequestT, + err error, +) { + if err := t.streamer.PlayError(w, req, err); err != nil { + t.log.LogError("Error occured while playing error video", err) + } +} + +func (t *T) cacheCheck(req extractor_config.RequestT, now time.Time) (extractor_config.ResultT, bool) { + for _, v := range t.cache.CleanExpired(now) { + t.log.LogDebug("Clean expired cache", v) + } + return t.cache.Get(req) +} + +func (t *T) cacheAdd( + req extractor_config.RequestT, + res extractor_config.ResultT, + now time.Time, +) { + t.log.LogDebug("Cache add", res) + t.cache.Add(req, res, now) +} + +func remove_http(url string) string { + url = strings.TrimPrefix(url, "http:/") + url = strings.TrimPrefix(url, "https:/") + url = strings.TrimLeft(url, "/") + return url +} + +func parseQuery(query string) extractor_config.RequestT { + var req extractor_config.RequestT + query = strings.TrimSpace(strings.TrimPrefix(query, "/play/")) + splitted := strings.Split(query, "?/?") + req.URL = remove_http(splitted[0]) + req.HEIGHT = defaultVideoHeight + req.FORMAT = defaultVideoFormat + if len(splitted) != 2 { + return req + } + tOpts, tErr := url.ParseQuery(splitted[1]) + if tErr == nil { + if tvh, ok := tOpts["vh"]; ok { + if tvh[0] == "360" || tvh[0] == "480" || tvh[0] == "720" { + req.HEIGHT = tvh[0] + } + } + if tvf, ok := tOpts["vf"]; ok { + if tvf[0] == "mp4" || tvf[0] == "m4a" { + req.FORMAT = tvf[0] + } + } + } + return req +} diff --git a/src/cmd/main_test.go b/src/lib/app/app_test.go similarity index 99% rename from src/cmd/main_test.go rename to src/lib/app/app_test.go index c9b32e7..bd300c4 100644 --- a/src/cmd/main_test.go +++ b/src/lib/app/app_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "testing" From 3325154aa71d2d40925b81c30d0a9b09423b27b7 Mon Sep 17 00:00:00 2001 From: mejgun Date: Wed, 18 Sep 2024 16:57:00 +0300 Subject: [PATCH 18/29] add config host option --- src/cmd/main.go | 5 ++--- src/lib/config/config.go | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/cmd/main.go b/src/cmd/main.go index 4fbef29..951f1e8 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -89,11 +89,10 @@ func main() { app.Run(w, r) log.LogInfo("Player disconnected", r.RemoteAddr) }) - port := fmt.Sprintf("%d", conf.PortInt) s := &http.Server{ - Addr: ":" + port, + Addr: fmt.Sprintf("%s:%d", conf.Host, conf.PortInt), } - log.LogInfo("Starting web server", "port", port, "test") + log.LogInfo("Starting web server", "host", conf.Host, "port", conf.PortInt) err = s.ListenAndServe() if err != nil { log.LogError("HTTP server start failed: ", err) diff --git a/src/lib/config/config.go b/src/lib/config/config.go index f40c8a0..01ba31a 100644 --- a/src/lib/config/config.go +++ b/src/lib/config/config.go @@ -13,6 +13,7 @@ import ( type configT struct { PortInt uint16 `json:"port"` + Host string `json:"host"` Streamer streamer.ConfigT `json:"streamer"` Extractor extractor_config.ConfigT `json:"extractor"` Log logger_config.ConfigT `json:"log"` @@ -41,6 +42,7 @@ func defaultConfig() configT { exp := "3h" return configT{ PortInt: 8080, + Host: "0.0.0.0", Streamer: streamer.ConfigT{ EnableErrorHeaders: &fls, IgnoreMissingHeaders: &fls, @@ -78,6 +80,9 @@ func appendConfig(src configT, dst configT) configT { if dst.PortInt == 0 { dst.PortInt = src.PortInt } + if dst.Host == "" { + dst.Host = src.Host + } // streamer if dst.Streamer.EnableErrorHeaders == nil { dst.Streamer.EnableErrorHeaders = src.Streamer.EnableErrorHeaders From cb88ac0ca98cba63f8b7670bbff616c3b1da5aa0 Mon Sep 17 00:00:00 2001 From: mejgun Date: Wed, 18 Sep 2024 17:43:14 +0300 Subject: [PATCH 19/29] start sub configs (per site settings) --- CHANGELOG.md | 1 + src/cmd/main.go | 26 +++++++++++++++++++++++--- src/lib/app/app.go | 34 ++++++++++++++++++++++++++++++++-- src/lib/config/config.go | 23 ++++++++++++++++++++++- 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f27d11f..dfeb984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - json format logs - disabling logs - force https links to extractor options +- host setting in config - stripping (bad) http(s) prefix in url ## 1.6.0 - 2024-09-13 diff --git a/src/cmd/main.go b/src/cmd/main.go index 951f1e8..eae5e47 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -49,7 +49,7 @@ func main() { } checkOrExit := func(err error, name string, errorcode int) { if err != nil { - stderr(fmt.Sprintf("%s create error.", name)) + stderr(fmt.Sprintf("%s error.", name)) stderr(err.Error()) os.Exit(errorcode) } @@ -68,7 +68,7 @@ func main() { checkOrExit(err, "Extractor", ExtractorError) status("extractor created") - cache, err := cache.New(conf.Cache, log) + cache_, err := cache.New(conf.Cache, log) checkOrExit(err, "Cache", CacheError) status("cache created") @@ -76,7 +76,27 @@ func main() { checkOrExit(err, "Streamer", StreamerError) status("streamer created") - app := app.New(log, cache, extr, restreamer) + opts := make([]app.Option, 0) + for _, v := range conf.SubConfig { + subcheck := func(err error, name string, errorcode int) { + checkOrExit(err, v.Name+" "+name, errorcode) + } + xtr, err := extractor.New(v.Extractor, log) + subcheck(err, "Extractor", ExtractorError) + cch, err := cache.New(v.Cache, log) + subcheck(err, "Cache", CacheError) + strm, err := streamer.New(v.Streamer, log, extr) + subcheck(err, "Streamer", StreamerError) + opts = append(opts, app.Option{ + Name: v.Name, + Sites: v.Sites, + X: xtr, + S: strm, + C: cch, + }) + + } + app := app.New(log, cache_, extr, restreamer, opts) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { log.LogInfo("Bad request", r.RemoteAddr, r.RequestURI) diff --git a/src/lib/app/app.go b/src/lib/app/app.go index 589cc92..20e0cdd 100644 --- a/src/lib/app/app.go +++ b/src/lib/app/app.go @@ -18,24 +18,54 @@ const ( defaultVideoFormat = "mp4" ) +type appT struct { + cache cache.T + extractor extractor.T + streamer streamer.T + name string + sites []string +} + type T struct { log logger.T cache cache.T extractor extractor.T streamer streamer.T + list []appT +} + +type Option struct { + Name string + Sites []string + X extractor.T + S streamer.T + C cache.T } func New( l logger.T, c cache.T, x extractor.T, - s streamer.T) *T { - return &T{ + s streamer.T, + opts []Option) *T { + t := T{ log: l, cache: c, extractor: x, streamer: s, } + t.list = make([]appT, 0) + for _, v := range opts { + t.list = append(t.list, appT{ + cache: v.C, + extractor: v.X, + streamer: v.S, + name: v.Name, + sites: v.Sites, + }) + } + t.log.LogDebug("t=", "y", t) + return &t } func (t *T) Run(w http.ResponseWriter, r *http.Request) { diff --git a/src/lib/config/config.go b/src/lib/config/config.go index 01ba31a..505a225 100644 --- a/src/lib/config/config.go +++ b/src/lib/config/config.go @@ -2,6 +2,7 @@ package config import ( "encoding/json" + "fmt" "os" "strings" @@ -18,6 +19,13 @@ type configT struct { Extractor extractor_config.ConfigT `json:"extractor"` Log logger_config.ConfigT `json:"log"` Cache cache.ConfigT `json:"cache"` + SubConfig []subConfigT `json:"sub-config"` +} + +type subConfigT struct { + Name string `json:"name"` + Sites []string `json:"sites"` + configT } func defaultConfig() configT { @@ -170,5 +178,18 @@ func Read(path string) (configT, error) { }() err = json.Unmarshal(b, &c) - return appendConfig(defaultConfig(), c), err + if err != nil { + return c, err + } + c = appendConfig(defaultConfig(), c) + for k, v := range c.SubConfig { + if v.Name == "" { + return c, fmt.Errorf("sub-config name empty") + } + if len(v.Sites) == 0 { + return c, fmt.Errorf("sub-config sites empty") + } + c.SubConfig[k].configT = appendConfig(c, v.configT) + } + return c, nil } From c46f07c4b8a20332d4adac69d94ddf883806ca89 Mon Sep 17 00:00:00 2001 From: mejgun Date: Wed, 18 Sep 2024 22:13:07 +0300 Subject: [PATCH 20/29] move sources --- {src/cmd => cmd}/go.mod | 0 {src/cmd => cmd}/main.go | 0 {src/lib => lib}/app/app.go | 0 {src/lib => lib}/app/app_test.go | 0 {src/lib => lib}/cache/cache.go | 0 {src/lib => lib}/cache/impl/default/default.go | 0 {src/lib => lib}/cache/impl/empty/empty.go | 0 {src/lib => lib}/config/config.go | 0 {src/lib => lib}/extractor/config/config.go | 0 {src/lib => lib}/extractor/extractor.go | 0 {src/lib => lib}/extractor/impl/default/default.go | 0 {src/lib => lib}/extractor/impl/direct/direct.go | 0 {src/lib => lib}/go.mod | 0 {src/lib => lib}/logger/config/config.go | 0 {src/lib => lib}/logger/impl/default/log.go | 0 {src/lib => lib}/logger/impl/empty/empty.go | 0 {src/lib => lib}/logger/impl/slog/slog.go | 0 {src/lib => lib}/logger/logger.go | 0 {src/lib => lib}/streamer/streamer.go | 0 {src/lib => lib}/streamer/streamer_test.go | 0 20 files changed, 0 insertions(+), 0 deletions(-) rename {src/cmd => cmd}/go.mod (100%) rename {src/cmd => cmd}/main.go (100%) rename {src/lib => lib}/app/app.go (100%) rename {src/lib => lib}/app/app_test.go (100%) rename {src/lib => lib}/cache/cache.go (100%) rename {src/lib => lib}/cache/impl/default/default.go (100%) rename {src/lib => lib}/cache/impl/empty/empty.go (100%) rename {src/lib => lib}/config/config.go (100%) rename {src/lib => lib}/extractor/config/config.go (100%) rename {src/lib => lib}/extractor/extractor.go (100%) rename {src/lib => lib}/extractor/impl/default/default.go (100%) rename {src/lib => lib}/extractor/impl/direct/direct.go (100%) rename {src/lib => lib}/go.mod (100%) rename {src/lib => lib}/logger/config/config.go (100%) rename {src/lib => lib}/logger/impl/default/log.go (100%) rename {src/lib => lib}/logger/impl/empty/empty.go (100%) rename {src/lib => lib}/logger/impl/slog/slog.go (100%) rename {src/lib => lib}/logger/logger.go (100%) rename {src/lib => lib}/streamer/streamer.go (100%) rename {src/lib => lib}/streamer/streamer_test.go (100%) diff --git a/src/cmd/go.mod b/cmd/go.mod similarity index 100% rename from src/cmd/go.mod rename to cmd/go.mod diff --git a/src/cmd/main.go b/cmd/main.go similarity index 100% rename from src/cmd/main.go rename to cmd/main.go diff --git a/src/lib/app/app.go b/lib/app/app.go similarity index 100% rename from src/lib/app/app.go rename to lib/app/app.go diff --git a/src/lib/app/app_test.go b/lib/app/app_test.go similarity index 100% rename from src/lib/app/app_test.go rename to lib/app/app_test.go diff --git a/src/lib/cache/cache.go b/lib/cache/cache.go similarity index 100% rename from src/lib/cache/cache.go rename to lib/cache/cache.go diff --git a/src/lib/cache/impl/default/default.go b/lib/cache/impl/default/default.go similarity index 100% rename from src/lib/cache/impl/default/default.go rename to lib/cache/impl/default/default.go diff --git a/src/lib/cache/impl/empty/empty.go b/lib/cache/impl/empty/empty.go similarity index 100% rename from src/lib/cache/impl/empty/empty.go rename to lib/cache/impl/empty/empty.go diff --git a/src/lib/config/config.go b/lib/config/config.go similarity index 100% rename from src/lib/config/config.go rename to lib/config/config.go diff --git a/src/lib/extractor/config/config.go b/lib/extractor/config/config.go similarity index 100% rename from src/lib/extractor/config/config.go rename to lib/extractor/config/config.go diff --git a/src/lib/extractor/extractor.go b/lib/extractor/extractor.go similarity index 100% rename from src/lib/extractor/extractor.go rename to lib/extractor/extractor.go diff --git a/src/lib/extractor/impl/default/default.go b/lib/extractor/impl/default/default.go similarity index 100% rename from src/lib/extractor/impl/default/default.go rename to lib/extractor/impl/default/default.go diff --git a/src/lib/extractor/impl/direct/direct.go b/lib/extractor/impl/direct/direct.go similarity index 100% rename from src/lib/extractor/impl/direct/direct.go rename to lib/extractor/impl/direct/direct.go diff --git a/src/lib/go.mod b/lib/go.mod similarity index 100% rename from src/lib/go.mod rename to lib/go.mod diff --git a/src/lib/logger/config/config.go b/lib/logger/config/config.go similarity index 100% rename from src/lib/logger/config/config.go rename to lib/logger/config/config.go diff --git a/src/lib/logger/impl/default/log.go b/lib/logger/impl/default/log.go similarity index 100% rename from src/lib/logger/impl/default/log.go rename to lib/logger/impl/default/log.go diff --git a/src/lib/logger/impl/empty/empty.go b/lib/logger/impl/empty/empty.go similarity index 100% rename from src/lib/logger/impl/empty/empty.go rename to lib/logger/impl/empty/empty.go diff --git a/src/lib/logger/impl/slog/slog.go b/lib/logger/impl/slog/slog.go similarity index 100% rename from src/lib/logger/impl/slog/slog.go rename to lib/logger/impl/slog/slog.go diff --git a/src/lib/logger/logger.go b/lib/logger/logger.go similarity index 100% rename from src/lib/logger/logger.go rename to lib/logger/logger.go diff --git a/src/lib/streamer/streamer.go b/lib/streamer/streamer.go similarity index 100% rename from src/lib/streamer/streamer.go rename to lib/streamer/streamer.go diff --git a/src/lib/streamer/streamer_test.go b/lib/streamer/streamer_test.go similarity index 100% rename from src/lib/streamer/streamer_test.go rename to lib/streamer/streamer_test.go From cebb3b9b861d80f60132ffa4eadd7e1a23162fcb Mon Sep 17 00:00:00 2001 From: mejgun Date: Wed, 18 Sep 2024 22:47:05 +0300 Subject: [PATCH 21/29] make per site settings work --- lib/app/app.go | 85 ++++++++++++++++++++----------- lib/app/app_test.go | 82 +++++++++++++++++------------ lib/cache/cache.go | 2 +- lib/cache/impl/default/default.go | 6 +-- lib/cache/impl/empty/empty.go | 4 +- lib/logger/impl/default/log.go | 2 +- 6 files changed, 111 insertions(+), 70 deletions(-) diff --git a/lib/app/app.go b/lib/app/app.go index 20e0cdd..8e3c707 100644 --- a/lib/app/app.go +++ b/lib/app/app.go @@ -6,6 +6,7 @@ import ( extractor_config "lib/extractor/config" logger "lib/logger" streamer "lib/streamer" + "slices" "net/http" "net/url" @@ -27,11 +28,9 @@ type appT struct { } type T struct { - log logger.T - cache cache.T - extractor extractor.T - streamer streamer.T - list []appT + log logger.T + defaultApp appT + appList []appT } type Option struct { @@ -49,14 +48,17 @@ func New( s streamer.T, opts []Option) *T { t := T{ - log: l, - cache: c, - extractor: x, - streamer: s, + log: l, + defaultApp: appT{ + name: "default", + cache: c, + extractor: x, + streamer: s, + }, } - t.list = make([]appT, 0) + t.appList = make([]appT, 0) for _, v := range opts { - t.list = append(t.list, appT{ + t.appList = append(t.appList, appT{ cache: v.C, extractor: v.X, streamer: v.S, @@ -68,60 +70,83 @@ func New( return &t } +func (t *T) selectApp(rawURL string) appT { + for _, v := range t.appList { + if slices.Contains(v.sites, rawURL) { + return v + } + } + return t.defaultApp +} + +func parseUrlHost(rawURL string) (string, error) { + u, err := url.Parse("https://" + rawURL) + return u.Host, err +} + func (t *T) Run(w http.ResponseWriter, r *http.Request) { now := time.Now() req := parseQuery(r.RequestURI) - t.log.LogInfo("Request", req) - if res, ok := t.cacheCheck(req, now); ok { + resapp := t.selectApp(req.URL) + t.log.LogInfo("Request", req, "app", resapp.name) + if res, ok, expired := resapp.cacheCheck(req, now); ok { + if len(expired) > 0 { + t.log.LogDebug("Expired links", expired) + } t.log.LogDebug("Link already cached", res) - t.play(w, r, req, res) + resapp.play(w, r, req, res, t.log) } else { - res, err := t.extractor.Extract(req) + if len(expired) > 0 { + t.log.LogDebug("Expired links", expired) + } + res, err := resapp.extractor.Extract(req) if err != nil { t.log.LogError("URL extract error", err) - t.playError(w, req, err) + resapp.playError(w, req, err, t.log) } t.log.LogDebug("Extractor returned", res) - t.cacheAdd(req, res, now) - t.play(w, r, req, res) + resapp.cacheAdd(req, res, now, t.log) + resapp.play(w, r, req, res, t.log) } } -func (t *T) play( +func (t *appT) play( w http.ResponseWriter, r *http.Request, req extractor_config.RequestT, res extractor_config.ResultT, + logger logger.T, ) { if err := t.streamer.Play(w, r, req, res); err != nil { - t.log.LogError("Restream error", err) - t.playError(w, req, err) + logger.LogError("Restream error", err) + t.playError(w, req, err, logger) } } -func (t *T) playError( +func (t *appT) playError( w http.ResponseWriter, req extractor_config.RequestT, err error, + logger logger.T, ) { if err := t.streamer.PlayError(w, req, err); err != nil { - t.log.LogError("Error occured while playing error video", err) + logger.LogError("Error occured while playing error video", err) } } -func (t *T) cacheCheck(req extractor_config.RequestT, now time.Time) (extractor_config.ResultT, bool) { - for _, v := range t.cache.CleanExpired(now) { - t.log.LogDebug("Clean expired cache", v) - } - return t.cache.Get(req) +func (t *appT) cacheCheck(req extractor_config.RequestT, now time.Time) (extractor_config.ResultT, bool, []extractor_config.RequestT) { + expired := t.cache.CleanExpired(now) + res, ok := t.cache.Get(req) + return res, ok, expired } -func (t *T) cacheAdd( +func (t *appT) cacheAdd( req extractor_config.RequestT, res extractor_config.ResultT, now time.Time, + logger logger.T, ) { - t.log.LogDebug("Cache add", res) + logger.LogDebug("Cache add", res) t.cache.Add(req, res, now) } diff --git a/lib/app/app_test.go b/lib/app/app_test.go index bd300c4..816245e 100644 --- a/lib/app/app_test.go +++ b/lib/app/app_test.go @@ -6,40 +6,39 @@ import ( extractor_config "lib/extractor/config" ) -var testPairs = map[string]extractor_config.RequestT{ - "/play/youtu.be/jNQXAC9IVRw?/?vh=360&vf=mp4": { - URL: "youtu.be/jNQXAC9IVRw", - HEIGHT: "360", - FORMAT: "mp4", - }, - "/play/youtu.be/jNQXAC9IVRw?/?vh=720?vf=avi": { - URL: "youtu.be/jNQXAC9IVRw", - HEIGHT: "720", - FORMAT: defaultVideoFormat, - }, - "/play/youtu.be/jNQXAC9IVRw": { - URL: "youtu.be/jNQXAC9IVRw", - HEIGHT: defaultVideoHeight, - FORMAT: defaultVideoFormat, - }, - "/play/youtu.be/jNQXAC9IVRw?/?": { - URL: "youtu.be/jNQXAC9IVRw", - HEIGHT: defaultVideoHeight, - FORMAT: defaultVideoFormat, - }, - "/play/youtu.be/jNQXAC9IVRw?/?vf=avi": { - URL: "youtu.be/jNQXAC9IVRw", - HEIGHT: defaultVideoHeight, - FORMAT: defaultVideoFormat, - }, - "/play/youtu.be/jNQXAC9IVRw?/?vf=mp4": { - URL: "youtu.be/jNQXAC9IVRw", - HEIGHT: defaultVideoHeight, - FORMAT: "mp4", - }, -} - func TestParseQuery(t *testing.T) { + var testPairs = map[string]extractor_config.RequestT{ + "/play/youtu.be/jNQXAC9IVRw?/?vh=360&vf=mp4": { + URL: "youtu.be/jNQXAC9IVRw", + HEIGHT: "360", + FORMAT: "mp4", + }, + "/play/youtu.be/jNQXAC9IVRw?/?vh=720?vf=avi": { + URL: "youtu.be/jNQXAC9IVRw", + HEIGHT: "720", + FORMAT: defaultVideoFormat, + }, + "/play/youtu.be/jNQXAC9IVRw": { + URL: "youtu.be/jNQXAC9IVRw", + HEIGHT: defaultVideoHeight, + FORMAT: defaultVideoFormat, + }, + "/play/youtu.be/jNQXAC9IVRw?/?": { + URL: "youtu.be/jNQXAC9IVRw", + HEIGHT: defaultVideoHeight, + FORMAT: defaultVideoFormat, + }, + "/play/youtu.be/jNQXAC9IVRw?/?vf=avi": { + URL: "youtu.be/jNQXAC9IVRw", + HEIGHT: defaultVideoHeight, + FORMAT: defaultVideoFormat, + }, + "/play/youtu.be/jNQXAC9IVRw?/?vf=mp4": { + URL: "youtu.be/jNQXAC9IVRw", + HEIGHT: defaultVideoHeight, + FORMAT: "mp4", + }, + } for k, v := range testPairs { if r := parseQuery(k); r != v { t.Error("For", k, "expected", v, "got", r) @@ -67,3 +66,20 @@ func TestRemoveHttp(t *testing.T) { } } } + +func TestParseHost(t *testing.T) { + for _, v := range []struct { + link string + want string + }{ + {link: "youtu.be/1", want: "youtu.be"}, + {link: "youtu.be:443/?param=1&b=c", want: "youtu.be:443"}, + {link: "www.yyy.youtu.be/", want: "www.yyy.youtu.be"}, + {link: "youtu.be", want: "youtu.be"}, + } { + if r, err := parseUrlHost(v.link); r != v.want || err != nil { + t.Error("For", v.link, "expected", v.want, "got", r, err) + } + } + +} diff --git a/lib/cache/cache.go b/lib/cache/cache.go index 42a10ea..9303538 100644 --- a/lib/cache/cache.go +++ b/lib/cache/cache.go @@ -13,7 +13,7 @@ import ( type T interface { Add(extractor_config.RequestT, extractor_config.ResultT, time.Time) Get(extractor_config.RequestT) (extractor_config.ResultT, bool) - CleanExpired(time.Time) []extractor_config.ResultT + CleanExpired(time.Time) []extractor_config.RequestT } type ConfigT struct { diff --git a/lib/cache/impl/default/default.go b/lib/cache/impl/default/default.go index 3e5847e..edfb18a 100644 --- a/lib/cache/impl/default/default.go +++ b/lib/cache/impl/default/default.go @@ -35,13 +35,13 @@ func (t *defaultCache) Get(req extractor_config.RequestT) (extractor_config.Resu return v, ok } -func (t *defaultCache) CleanExpired(now time.Time) []extractor_config.ResultT { - deleted := make([]extractor_config.ResultT, 0) +func (t *defaultCache) CleanExpired(now time.Time) []extractor_config.RequestT { + deleted := make([]extractor_config.RequestT, 0) t.Lock() for k, v := range t.cache { if v.Expire.Before(now) { delete(t.cache, k) - deleted = append(deleted, v) + deleted = append(deleted, k) } } t.Unlock() diff --git a/lib/cache/impl/empty/empty.go b/lib/cache/impl/empty/empty.go index 39de86d..21ba3a7 100644 --- a/lib/cache/impl/empty/empty.go +++ b/lib/cache/impl/empty/empty.go @@ -20,6 +20,6 @@ func (t *emptyCache) Get(req extractor_config.RequestT) (extractor_config.Result return extractor_config.ResultT{}, false } -func (t *emptyCache) CleanExpired(now time.Time) []extractor_config.ResultT { - return []extractor_config.ResultT{} +func (t *emptyCache) CleanExpired(now time.Time) []extractor_config.RequestT { + return []extractor_config.RequestT{} } diff --git a/lib/logger/impl/default/log.go b/lib/logger/impl/default/log.go index e504ba4..7e53529 100644 --- a/lib/logger/impl/default/log.go +++ b/lib/logger/impl/default/log.go @@ -27,7 +27,7 @@ func (t *loggerT) print(str string, s string, args []any) { } for k < arglen { if k+1 < arglen { - add(fmt.Sprintf("%s=%+v", args[k], args[k+1])) + add(fmt.Sprintf("%s:%+v", args[k], args[k+1])) k = k + 2 } else { add(fmt.Sprintf("%+v", args[k])) From 6df7e6498e5c3499a56cb228daf0dbebfaa0ef27 Mon Sep 17 00:00:00 2001 From: mejgun Date: Wed, 18 Sep 2024 22:59:33 +0300 Subject: [PATCH 22/29] add logger layer --- cmd/main.go | 6 +++--- lib/app/app.go | 14 ++++++++------ lib/logger/logger.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index eae5e47..17372e3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -64,15 +64,15 @@ func main() { } status("logger created") - extr, err := extractor.New(conf.Extractor, log) + extr, err := extractor.New(conf.Extractor, logger.NewLayer(log, "Extractor")) checkOrExit(err, "Extractor", ExtractorError) status("extractor created") - cache_, err := cache.New(conf.Cache, log) + cache_, err := cache.New(conf.Cache, logger.NewLayer(log, "Cache")) checkOrExit(err, "Cache", CacheError) status("cache created") - restreamer, err := streamer.New(conf.Streamer, log, extr) + restreamer, err := streamer.New(conf.Streamer, logger.NewLayer(log, "Streamer"), extr) checkOrExit(err, "Streamer", StreamerError) status("streamer created") diff --git a/lib/app/app.go b/lib/app/app.go index 8e3c707..2897a86 100644 --- a/lib/app/app.go +++ b/lib/app/app.go @@ -85,24 +85,26 @@ func parseUrlHost(rawURL string) (string, error) { } func (t *T) Run(w http.ResponseWriter, r *http.Request) { + printExpired := func(links []extractor_config.RequestT) { + if len(links) > 0 { + t.log.LogDebug("Expired links", links) + } + } now := time.Now() req := parseQuery(r.RequestURI) resapp := t.selectApp(req.URL) t.log.LogInfo("Request", req, "app", resapp.name) if res, ok, expired := resapp.cacheCheck(req, now); ok { - if len(expired) > 0 { - t.log.LogDebug("Expired links", expired) - } + printExpired(expired) t.log.LogDebug("Link already cached", res) resapp.play(w, r, req, res, t.log) } else { - if len(expired) > 0 { - t.log.LogDebug("Expired links", expired) - } + printExpired(expired) res, err := resapp.extractor.Extract(req) if err != nil { t.log.LogError("URL extract error", err) resapp.playError(w, req, err, t.log) + return } t.log.LogDebug("Extractor returned", res) resapp.cacheAdd(req, res, now, t.log) diff --git a/lib/logger/logger.go b/lib/logger/logger.go index 0411b93..1f99b5c 100644 --- a/lib/logger/logger.go +++ b/lib/logger/logger.go @@ -1,6 +1,7 @@ package logger import ( + "fmt" config "lib/logger/config" logger_default "lib/logger/impl/default" logger_empty "lib/logger/impl/empty" @@ -24,3 +25,32 @@ type T interface { LogDebug(string, ...any) LogInfo(string, ...any) } + +type loggerLayer struct { + impl T + logger_name string +} + +func NewLayer(impl T, name_str string) T { + return &loggerLayer{ + impl: impl, + logger_name: name_str, + } +} + +func (t *loggerLayer) f(s string) string { + return fmt.Sprintf("%s. %s", t.logger_name, s) +} + +func (t *loggerLayer) LogError(s string, i ...any) { + t.impl.LogError(t.f(s), i) +} +func (t *loggerLayer) LogWarning(s string, i ...any) { + t.impl.LogWarning(t.f(s), i) +} +func (t *loggerLayer) LogDebug(s string, i ...any) { + t.impl.LogDebug(t.f(s), i) +} +func (t *loggerLayer) LogInfo(s string, i ...any) { + t.impl.LogInfo(t.f(s), i) +} From 4eb66cdb886827bebe26715ddbd71e19234d3a91 Mon Sep 17 00:00:00 2001 From: mejgun Date: Thu, 19 Sep 2024 17:41:15 +0300 Subject: [PATCH 23/29] refactoring app creating --- .gitignore | 2 +- cmd/main.go | 103 +++++++++++++++++++++---------------- lib/app/app.go | 27 +++++----- lib/cache/cache.go | 4 +- lib/config/config.go | 20 +++---- lib/extractor/extractor.go | 2 +- lib/logger/logger.go | 15 ++++-- lib/streamer/streamer.go | 6 +-- 8 files changed, 101 insertions(+), 78 deletions(-) diff --git a/.gitignore b/.gitignore index ce68f86..028726f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ bin/ ytproxy log.txt -config.json +config.jsonc diff --git a/cmd/main.go b/cmd/main.go index 17372e3..2ce4429 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -34,69 +34,61 @@ const ( func parseCLIFlags() flagsT { var f flagsT flag.BoolVar(&f.version, "version", false, "prints current yt-proxy version") - flag.StringVar(&f.config, "config", "config.json", "config file path") + flag.StringVar(&f.config, "config", "config.jsonc", "config file path") flag.Parse() return f } func main() { - stdout := func(s string) { os.Stdout.WriteString(fmt.Sprintf("%s\n", s)) } - stderr := func(s string) { os.Stderr.WriteString(fmt.Sprintf("ERROR %s\n", s)) } flags := parseCLIFlags() if flags.version { stdout(appVersion) os.Exit(NoError) } - checkOrExit := func(err error, name string, errorcode int) { - if err != nil { - stderr(fmt.Sprintf("%s error.", name)) - stderr(err.Error()) - os.Exit(errorcode) - } - } - conf, err := config.Read(flags.config) - checkOrExit(err, "Config", ConfigError) + startApp(flags.config) +} - log, err := logger.New(conf.Log) - checkOrExit(err, "Logger", LoggerError) - status := func(s string) { - log.LogDebug("App starting", "status", s) +var stdout = func(s string) { os.Stdout.WriteString(fmt.Sprintf("%s\n", s)) } +var stderr = func(s string) { os.Stderr.WriteString(fmt.Sprintf("ERROR %s\n", s)) } +var checkOrExit = func(err error, name string, errorcode int) { + if err != nil { + stderr(fmt.Sprintf("%s error. %s", name, err)) + os.Exit(errorcode) } - status("logger created") +} +var texts = [5]string{ + "Config", + "Logger", + "Extractor", + "Cache", + "Streamer", +} - extr, err := extractor.New(conf.Extractor, logger.NewLayer(log, "Extractor")) - checkOrExit(err, "Extractor", ExtractorError) - status("extractor created") +func startApp(conf_file string) { - cache_, err := cache.New(conf.Cache, logger.NewLayer(log, "Cache")) - checkOrExit(err, "Cache", CacheError) - status("cache created") + conf, err := config.Read(conf_file) + checkOrExit(err, texts[0], ConfigError) - restreamer, err := streamer.New(conf.Streamer, logger.NewLayer(log, "Streamer"), extr) - checkOrExit(err, "Streamer", StreamerError) - status("streamer created") + log, err := logger.New(conf.Log) + checkOrExit(err, texts[1], LoggerError) opts := make([]app.Option, 0) for _, v := range conf.SubConfig { - subcheck := func(err error, name string, errorcode int) { - checkOrExit(err, v.Name+" "+name, errorcode) - } - xtr, err := extractor.New(v.Extractor, log) - subcheck(err, "Extractor", ExtractorError) - cch, err := cache.New(v.Cache, log) - subcheck(err, "Cache", CacheError) - strm, err := streamer.New(v.Streamer, log, extr) - subcheck(err, "Streamer", StreamerError) - opts = append(opts, app.Option{ - Name: v.Name, - Sites: v.Sites, - X: xtr, - S: strm, - C: cch, - }) - + opt := getNewApp(log, v) + opts = append(opts, opt) } - app := app.New(log, cache_, extr, restreamer, opts) + defapp := getNewApp(log, config.SubConfigT{ + ConfigT: config.ConfigT{ + Streamer: conf.Streamer, + Extractor: conf.Extractor, + Cache: conf.Cache, + }, + Name: "default", + }) + app := app.New( + logger.NewLayer(log, "App"), + defapp, + opts) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { log.LogInfo("Bad request", r.RemoteAddr, r.RequestURI) @@ -119,3 +111,26 @@ func main() { os.Exit(WebServerError) } } + +func getNewApp(log logger.T, v config.SubConfigT) app.Option { + subcheck := func(err error, name string, errorcode int) { + checkOrExit(err, v.Name+" "+name, errorcode) + } + newname := func(name string) string { + return fmt.Sprintf("[%s] %s", v.Name, name) + } + xtr, err := extractor.New(v.Extractor, logger.NewLayer(log, newname(texts[2]))) + subcheck(err, texts[2], ExtractorError) + cch, err := cache.New(v.Cache, logger.NewLayer(log, newname(texts[3]))) + subcheck(err, texts[3], CacheError) + strm, err := streamer.New(v.Streamer, logger.NewLayer(log, newname(texts[4])), xtr) + subcheck(err, texts[4], StreamerError) + return app.Option{ + Name: v.Name, + Sites: v.Sites, + X: xtr, + S: strm, + C: cch, + L: logger.NewLayer(log, fmt.Sprintf("[%s] app", v.Name)), + } +} diff --git a/lib/app/app.go b/lib/app/app.go index 2897a86..4401725 100644 --- a/lib/app/app.go +++ b/lib/app/app.go @@ -25,6 +25,7 @@ type appT struct { streamer streamer.T name string sites []string + log logger.T } type T struct { @@ -39,21 +40,20 @@ type Option struct { X extractor.T S streamer.T C cache.T + L logger.T } func New( - l logger.T, - c cache.T, - x extractor.T, - s streamer.T, + log logger.T, + def Option, opts []Option) *T { t := T{ - log: l, defaultApp: appT{ + log: def.L, name: "default", - cache: c, - extractor: x, - streamer: s, + cache: def.C, + extractor: def.X, + streamer: def.S, }, } t.appList = make([]appT, 0) @@ -64,16 +64,19 @@ func New( streamer: v.S, name: v.Name, sites: v.Sites, + log: v.L, }) } - t.log.LogDebug("t=", "y", t) return &t } func (t *T) selectApp(rawURL string) appT { - for _, v := range t.appList { - if slices.Contains(v.sites, rawURL) { - return v + host, err := parseUrlHost(rawURL) + if err == nil { + for _, v := range t.appList { + if slices.Contains(v.sites, host) { + return v + } } } return t.defaultApp diff --git a/lib/cache/cache.go b/lib/cache/cache.go index 9303538..65d6e2b 100644 --- a/lib/cache/cache.go +++ b/lib/cache/cache.go @@ -26,9 +26,9 @@ func New(conf ConfigT, log logger.T) (T, error) { return cache_default.New(0), err } if t.Seconds() < 1 { - log.LogDebug("cache", "disabled by config") + log.LogDebug("", "disabled by config") return cache_empty.New(), nil } - log.LogDebug("cache", fmt.Sprintf("expire time set to %s", t)) + log.LogDebug("", fmt.Sprintf("expire time set to %s", t)) return cache_default.New(0), nil } diff --git a/lib/config/config.go b/lib/config/config.go index 505a225..ee6da16 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -12,23 +12,23 @@ import ( streamer "lib/streamer" ) -type configT struct { +type ConfigT struct { PortInt uint16 `json:"port"` Host string `json:"host"` Streamer streamer.ConfigT `json:"streamer"` Extractor extractor_config.ConfigT `json:"extractor"` Log logger_config.ConfigT `json:"log"` Cache cache.ConfigT `json:"cache"` - SubConfig []subConfigT `json:"sub-config"` + SubConfig []SubConfigT `json:"sub-config"` } -type subConfigT struct { +type SubConfigT struct { Name string `json:"name"` Sites []string `json:"sites"` - configT + ConfigT } -func defaultConfig() configT { +func defaultConfig() ConfigT { fls := false tru := true ext := streamer.Extractor @@ -48,7 +48,7 @@ func defaultConfig() configT { lo := logger_config.Stdout lf := "log.txt" exp := "3h" - return configT{ + return ConfigT{ PortInt: 8080, Host: "0.0.0.0", Streamer: streamer.ConfigT{ @@ -83,7 +83,7 @@ func defaultConfig() configT { } // add second config options to first -func appendConfig(src configT, dst configT) configT { +func appendConfig(src ConfigT, dst ConfigT) ConfigT { // general options if dst.PortInt == 0 { dst.PortInt = src.PortInt @@ -158,8 +158,8 @@ func appendConfig(src configT, dst configT) configT { return dst } -func Read(path string) (configT, error) { - var c configT +func Read(path string) (ConfigT, error) { + var c ConfigT b, err := os.ReadFile(path) if err != nil { return c, err @@ -189,7 +189,7 @@ func Read(path string) (configT, error) { if len(v.Sites) == 0 { return c, fmt.Errorf("sub-config sites empty") } - c.SubConfig[k].configT = appendConfig(c, v.configT) + c.SubConfig[k].ConfigT = appendConfig(c, v.ConfigT) } return c, nil } diff --git a/lib/extractor/extractor.go b/lib/extractor/extractor.go index d0dfe7e..9768d8f 100644 --- a/lib/extractor/extractor.go +++ b/lib/extractor/extractor.go @@ -23,7 +23,7 @@ func New(c extractor_config.ConfigT, log logger.T) (T, error) { ) ext.force_http = *c.ForceHttp if ext.force_http { - log.LogDebug("extractor", "force-http", true) + log.LogDebug("", "force-http", true) } ext.impl, err = real_new(c, log) return &ext, err diff --git a/lib/logger/logger.go b/lib/logger/logger.go index 1f99b5c..c771cbf 100644 --- a/lib/logger/logger.go +++ b/lib/logger/logger.go @@ -39,18 +39,23 @@ func NewLayer(impl T, name_str string) T { } func (t *loggerLayer) f(s string) string { - return fmt.Sprintf("%s. %s", t.logger_name, s) + switch s { + case "": + return t.logger_name + default: + return fmt.Sprintf("%s. %s", t.logger_name, s) + } } func (t *loggerLayer) LogError(s string, i ...any) { - t.impl.LogError(t.f(s), i) + t.impl.LogError(t.f(s), i...) } func (t *loggerLayer) LogWarning(s string, i ...any) { - t.impl.LogWarning(t.f(s), i) + t.impl.LogWarning(t.f(s), i...) } func (t *loggerLayer) LogDebug(s string, i ...any) { - t.impl.LogDebug(t.f(s), i) + t.impl.LogDebug(t.f(s), i...) } func (t *loggerLayer) LogInfo(s string, i ...any) { - t.impl.LogInfo(t.f(s), i) + t.impl.LogInfo(t.f(s), i...) } diff --git a/lib/streamer/streamer.go b/lib/streamer/streamer.go index 2fbe532..b42ac86 100644 --- a/lib/streamer/streamer.go +++ b/lib/streamer/streamer.go @@ -319,19 +319,19 @@ func makeSetHeaders(conf ConfigT) func(http.ResponseWriter, *http.Response) erro func makeSetStreamerUserAgent(conf ConfigT, xt extractor.T, log logger.T) (func(*http.Request) string, error) { switch *conf.SetUserAgent { case Request: - log.LogDebug("Streamer User-Agent set to request-set") + log.LogDebug("", "User-Agent", "request-set") return func(r *http.Request) string { return r.UserAgent() }, nil case Extractor: ua, err := xt.GetUserAgent() - log.LogDebug("Streamer User-Agent set to", ua) + log.LogDebug("", "User-Agent", ua) return func(r *http.Request) string { return ua }, err case Config: ua := conf.UserAgent - log.LogDebug("Streamer User-Agent set to", ua) + log.LogDebug("", "User-Agent", ua) return func(r *http.Request) string { return *ua }, nil From 2fa63773414b8d430dc9ee4d8f3e455e59d5fbbb Mon Sep 17 00:00:00 2001 From: mejgun Date: Thu, 19 Sep 2024 17:47:06 +0300 Subject: [PATCH 24/29] add sync mutex to app --- lib/app/app.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/app/app.go b/lib/app/app.go index 4401725..8a8aa43 100644 --- a/lib/app/app.go +++ b/lib/app/app.go @@ -7,6 +7,7 @@ import ( logger "lib/logger" streamer "lib/streamer" "slices" + "sync" "net/http" "net/url" @@ -29,6 +30,7 @@ type appT struct { } type T struct { + mu sync.RWMutex log logger.T defaultApp appT appList []appT @@ -48,6 +50,7 @@ func New( def Option, opts []Option) *T { t := T{ + log: log, defaultApp: appT{ log: def.L, name: "default", @@ -88,6 +91,8 @@ func parseUrlHost(rawURL string) (string, error) { } func (t *T) Run(w http.ResponseWriter, r *http.Request) { + t.mu.RLock() + defer t.mu.RUnlock() printExpired := func(links []extractor_config.RequestT) { if len(links) > 0 { t.log.LogDebug("Expired links", links) From faf6ad0111b1cdfbe4f0da3e71303c55922c94ad Mon Sep 17 00:00:00 2001 From: mejgun Date: Thu, 19 Sep 2024 18:03:41 +0300 Subject: [PATCH 25/29] config comments edit --- README.md | 2 +- cmd/main.go | 1 - config.example.jsonc | 32 ++++++++++++++++++++++++++------ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1f55e3e..1ec7310 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is yt-dlp based video restreamer, part of another project: https://github.c ### Build ### -`cd src && go build` +`cd cmd && go build` ### Options ### diff --git a/cmd/main.go b/cmd/main.go index 2ce4429..fedd950 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -65,7 +65,6 @@ var texts = [5]string{ } func startApp(conf_file string) { - conf, err := config.Read(conf_file) checkOrExit(err, texts[0], ConfigError) diff --git a/config.example.jsonc b/config.example.jsonc index 4fe0d28..9f2d667 100644 --- a/config.example.jsonc +++ b/config.example.jsonc @@ -1,32 +1,52 @@ +// example config for the app. +// only full string comments are supported, +// do not use /* comment */ and comment after keys. +// delete string to set key to the DEFAULT value. { - // web server listen port + // web server listen host. + // DEFAULT "0.0.0.0" + "host": "127.0.0.1", + // web server listen port. + // DEFAULT 8080 "port": 8080, - // restreamer config + // restreamer config. + // restreamer takes https stream and restream it as http. "streamer": { - // show errors in headers (insecure) + // show errors in headers (insecure). + // streaming errors will be sent as Error-Header-xx header. + // DEFAULT false "error-headers": false, - // do not strictly check video headers + // do not strictly check video headers. + // if true, streamer will ignore incorrect "Content-Length" and "Content-Type" header. + // DEFAULT false "ignore-missing-headers": false, // do not check video server certificate (insecure) + // DEFAULT false "ignore-ssl-errors": false, - // video file that will be shown on errors + // video file that will be shown on video stream errors + // DEFAULT "corrupted.mp4" "error-video": "corrupted.mp4", - // audio file that will be played on errors + // audio file that will be played on audio stream errors // dwnlded here youtu.be/_b8KPiT1PxI (suggest your options) + // DEFAULT "failed.m4a" "error-audio": "failed.m4a", // how to set streamer's user-agent // request - set from user's request (old default) // extractor - set from extractor on app start (default) // config - set from config + // DEFAULT "extractor" "set-user-agent": "extractor", // custom user agent used if "set-user-agent" set to "config" + // DEFAULT "Mozilla" "user-agent": "Mozilla", // proxy for restreamer // empty - no proxy // "env" - read proxy from environment variables (e.g. HTTP_PROXY="http://127.0.0.1:3128") // proxy url - e.g. "socks5://127.0.0.1:9999" + // DEFAULT "env" "proxy": "env", // min TLS version: "TLS 1.3", "TLS 1.2", etc. + // DEFAULT "TLS 1.2" "min-tls-version": "TLS 1.2" }, // media extractor config From d173b0ef5af90f649ceb2a4622b4700b6d5c4099 Mon Sep 17 00:00:00 2001 From: mejgun Date: Thu, 19 Sep 2024 22:41:11 +0300 Subject: [PATCH 26/29] add signals catching add http server shutdown add log file closing --- cmd/main.go | 35 ++++++++++++++++++++++--- lib/app/app.go | 6 +++++ lib/logger/impl/default/log.go | 33 +++++++++++++++++++---- lib/logger/impl/empty/empty.go | 1 + lib/logger/impl/slog/slog.go | 48 +++++++++++++++++++++++++++------- lib/logger/logger.go | 4 +++ 6 files changed, 110 insertions(+), 17 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index fedd950..4d6f2c9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,10 +1,13 @@ package main import ( + "context" "flag" "fmt" "net/http" "os" + "os/signal" + "syscall" app "lib/app" cache "lib/cache" @@ -103,14 +106,40 @@ func startApp(conf_file string) { s := &http.Server{ Addr: fmt.Sprintf("%s:%d", conf.Host, conf.PortInt), } + go signalsCatcher(log, app, s) + log.LogInfo("Starting web server", "host", conf.Host, "port", conf.PortInt) - err = s.ListenAndServe() - if err != nil { - log.LogError("HTTP server start failed: ", err) + if err = s.ListenAndServe(); err == http.ErrServerClosed { + log.LogInfo("HTTP server closed") + os.Exit(NoError) + } else { + log.LogError("HTTP server", err) + log.Close() os.Exit(WebServerError) } } +func signalsCatcher(log logger.T, app *app.T, s *http.Server) { + sigint := make(chan os.Signal, 1) + signal.Notify(sigint, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + for { + switch <-sigint { + case syscall.SIGHUP: + log.LogWarning("Config reloading") + case syscall.SIGINT: + fallthrough + case syscall.SIGTERM: + log.LogWarning("Exiting") + app.Shutdown() + if err := s.Shutdown(context.Background()); err != nil { + // Error from closing listeners, or context timeout: + log.LogError("HTTP server Shutdown", err) + } + return + } + } +} + func getNewApp(log logger.T, v config.SubConfigT) app.Option { subcheck := func(err error, name string, errorcode int) { checkOrExit(err, v.Name+" "+name, errorcode) diff --git a/lib/app/app.go b/lib/app/app.go index 8a8aa43..0da67f8 100644 --- a/lib/app/app.go +++ b/lib/app/app.go @@ -192,3 +192,9 @@ func parseQuery(query string) extractor_config.RequestT { } return req } + +func (t *T) Shutdown() { + t.mu.Lock() // locking app forever + t.log.LogInfo("Exiting") + t.log.Close() +} diff --git a/lib/logger/impl/default/log.go b/lib/logger/impl/default/log.go index 7e53529..eb88b37 100644 --- a/lib/logger/impl/default/log.go +++ b/lib/logger/impl/default/log.go @@ -5,13 +5,16 @@ import ( "io" "log" "os" + "sync" l "lib/logger/config" ) type loggerT struct { - lvl *l.LevelT - lgr *log.Logger + mu sync.RWMutex + lvl *l.LevelT + lgr *log.Logger + outputs []*os.File } func (t *loggerT) print(str string, s string, args []any) { @@ -44,19 +47,36 @@ func (t *loggerT) checkAndPrint(lvl l.LevelT, str string, s string, i []any) { } func (t *loggerT) LogError(s string, i ...any) { + t.mu.RLock() + defer t.mu.RUnlock() t.checkAndPrint(l.Error, "ERROR", s, i) } func (t *loggerT) LogWarning(s string, i ...any) { + t.mu.RLock() + defer t.mu.RUnlock() t.checkAndPrint(l.Warning, "WARNING", s, i) } func (t *loggerT) LogDebug(s string, i ...any) { + t.mu.RLock() + defer t.mu.RUnlock() t.checkAndPrint(l.Debug, "DEBUG", s, i) } func (t *loggerT) LogInfo(s string, i ...any) { + t.mu.RLock() + defer t.mu.RUnlock() t.checkAndPrint(l.Info, "INFO", s, i) } +func (t *loggerT) Close() { + t.mu.Lock() + defer t.mu.Unlock() + for _, v := range t.outputs { + v.Close() + } + t.lgr = log.Default() +} + func New(conf l.ConfigT) (*loggerT, error) { var ( logger loggerT @@ -64,10 +84,11 @@ func New(conf l.ConfigT) (*loggerT, error) { ) open := func() (*os.File, error) { return os.OpenFile( - // will never close this file :| - // should trap exit - *conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) + *conf.FileName, + os.O_APPEND|os.O_WRONLY|os.O_CREATE, + 0664) } + logger.outputs = make([]*os.File, 0) switch *conf.Output { case l.Stdout: lgr.SetOutput(os.Stdout) @@ -76,12 +97,14 @@ func New(conf l.ConfigT) (*loggerT, error) { if err != nil { return &logger, err } + logger.outputs = append(logger.outputs, f) lgr.SetOutput(f) case l.Both: f, err := open() if err != nil { return &logger, err } + logger.outputs = append(logger.outputs, f) out := io.MultiWriter(os.Stdout, f) lgr.SetOutput(out) } diff --git a/lib/logger/impl/empty/empty.go b/lib/logger/impl/empty/empty.go index a9b3d32..47da36e 100644 --- a/lib/logger/impl/empty/empty.go +++ b/lib/logger/impl/empty/empty.go @@ -7,6 +7,7 @@ func (t *loggerT) LogError(string, ...any) {} func (t *loggerT) LogWarning(string, ...any) {} func (t *loggerT) LogDebug(string, ...any) {} func (t *loggerT) LogInfo(string, ...any) {} +func (t *loggerT) Close() {} func New() (*loggerT, error) { return &loggerT{}, nil diff --git a/lib/logger/impl/slog/slog.go b/lib/logger/impl/slog/slog.go index f356918..b56ceb4 100644 --- a/lib/logger/impl/slog/slog.go +++ b/lib/logger/impl/slog/slog.go @@ -4,34 +4,54 @@ import ( "io" "log/slog" "os" + "sync" l "lib/logger/config" ) type loggerT struct { - lgr *slog.Logger + mu sync.RWMutex + lgr *slog.Logger + outputs []*os.File } func (t *loggerT) LogError(s string, i ...any) { + t.mu.RLock() + defer t.mu.RUnlock() t.lgr.Error(s, i...) } func (t *loggerT) LogWarning(s string, i ...any) { + t.mu.RLock() + defer t.mu.RUnlock() t.lgr.Warn(s, i...) } func (t *loggerT) LogDebug(s string, i ...any) { + t.mu.RLock() + defer t.mu.RUnlock() t.lgr.Debug(s, i...) } func (t *loggerT) LogInfo(s string, i ...any) { + t.mu.RLock() + defer t.mu.RUnlock() t.lgr.Info(s, i...) } +func (t *loggerT) Close() { + t.mu.Lock() + defer t.mu.Unlock() + for _, v := range t.outputs { + v.Close() + } + t.lgr = slog.Default() +} + func New(conf l.ConfigT) (*loggerT, error) { open := func() (*os.File, error) { return os.OpenFile( - // will never close this file :| - // should trap exit - *conf.FileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0664) + *conf.FileName, + os.O_APPEND|os.O_WRONLY|os.O_CREATE, + 0664) } var ( lvl slog.Level @@ -47,28 +67,38 @@ func New(conf l.ConfigT) (*loggerT, error) { case l.Error: lvl = slog.LevelError } - mkLogger := func(dst io.Writer) { + mkLogger := func(dst1, dst2 *os.File) { + var dst io.Writer + if dst2 == nil { + dst = dst2 + } else { + dst = io.MultiWriter(dst1, dst2) + } lgr = slog.New( slog.NewJSONHandler(dst, &slog.HandlerOptions{Level: lvl})) } + outputs := make([]*os.File, 0) switch *conf.Output { case l.Stdout: - mkLogger(os.Stdout) + mkLogger(os.Stdout, nil) case l.File: f, err := open() if err != nil { return &loggerT{}, err } - mkLogger(f) + outputs = append(outputs, f) + mkLogger(f, nil) case l.Both: f, err := open() if err != nil { return &loggerT{}, err } - mkLogger(io.MultiWriter(os.Stdout, f)) + outputs = append(outputs, f) + mkLogger(os.Stdout, f) } return &loggerT{ - lgr: lgr, + lgr: lgr, + outputs: outputs, }, nil } diff --git a/lib/logger/logger.go b/lib/logger/logger.go index c771cbf..b77e0d3 100644 --- a/lib/logger/logger.go +++ b/lib/logger/logger.go @@ -24,6 +24,7 @@ type T interface { LogWarning(string, ...any) LogDebug(string, ...any) LogInfo(string, ...any) + Close() } type loggerLayer struct { @@ -59,3 +60,6 @@ func (t *loggerLayer) LogDebug(s string, i ...any) { func (t *loggerLayer) LogInfo(s string, i ...any) { t.impl.LogInfo(t.f(s), i...) } +func (t *loggerLayer) Close() { + t.impl.Close() +} From 1d71cdaeb06acae7598d809a56cfa70e8f98df4f Mon Sep 17 00:00:00 2001 From: mejgun Date: Fri, 20 Sep 2024 14:20:16 +0300 Subject: [PATCH 27/29] rework app starting & add config reloading --- cmd/main.go | 133 ++++++++++++++++++++++++++++--------------------- lib/app/app.go | 65 ++++++++++++++---------- 2 files changed, 115 insertions(+), 83 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4d6f2c9..255586a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -26,12 +26,7 @@ type flagsT struct { const ( NoError = iota - ConfigError - LoggerError - ExtractorError - StreamerError - WebServerError - CacheError + SomeError ) func parseCLIFlags() flagsT { @@ -45,52 +40,21 @@ func parseCLIFlags() flagsT { func main() { flags := parseCLIFlags() if flags.version { - stdout(appVersion) + os.Stdout.WriteString(fmt.Sprintf("%s\n", appVersion)) os.Exit(NoError) } startApp(flags.config) } -var stdout = func(s string) { os.Stdout.WriteString(fmt.Sprintf("%s\n", s)) } -var stderr = func(s string) { os.Stderr.WriteString(fmt.Sprintf("ERROR %s\n", s)) } -var checkOrExit = func(err error, name string, errorcode int) { - if err != nil { - stderr(fmt.Sprintf("%s error. %s", name, err)) - os.Exit(errorcode) - } -} -var texts = [5]string{ - "Config", - "Logger", - "Extractor", - "Cache", - "Streamer", -} - func startApp(conf_file string) { - conf, err := config.Read(conf_file) - checkOrExit(err, texts[0], ConfigError) - - log, err := logger.New(conf.Log) - checkOrExit(err, texts[1], LoggerError) - - opts := make([]app.Option, 0) - for _, v := range conf.SubConfig { - opt := getNewApp(log, v) - opts = append(opts, opt) + conf, def, opts, log, err := readConfig(conf_file) + if err != nil { + os.Stderr.WriteString(fmt.Sprintf("Config read error: %s\n", err)) + os.Exit(SomeError) } - defapp := getNewApp(log, config.SubConfigT{ - ConfigT: config.ConfigT{ - Streamer: conf.Streamer, - Extractor: conf.Extractor, - Cache: conf.Cache, - }, - Name: "default", - }) app := app.New( logger.NewLayer(log, "App"), - defapp, - opts) + def, opts) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { log.LogInfo("Bad request", r.RemoteAddr, r.RequestURI) @@ -106,7 +70,7 @@ func startApp(conf_file string) { s := &http.Server{ Addr: fmt.Sprintf("%s:%d", conf.Host, conf.PortInt), } - go signalsCatcher(log, app, s) + go signalsCatcher(conf_file, log, app, s) log.LogInfo("Starting web server", "host", conf.Host, "port", conf.PortInt) if err = s.ListenAndServe(); err == http.ErrServerClosed { @@ -115,17 +79,59 @@ func startApp(conf_file string) { } else { log.LogError("HTTP server", err) log.Close() - os.Exit(WebServerError) + os.Exit(SomeError) } } -func signalsCatcher(log logger.T, app *app.T, s *http.Server) { +func readConfig(conf_file string) (config.ConfigT, app.Option, []app.Option, + logger.T, error) { + conf, err := config.Read(conf_file) + if err != nil { + return config.ConfigT{}, app.Option{}, nil, nil, err + } + + log, err := logger.New(conf.Log) + if err != nil { + return config.ConfigT{}, app.Option{}, nil, nil, err + } + + defapp, err := getNewApp(log, config.SubConfigT{ + ConfigT: config.ConfigT{ + Streamer: conf.Streamer, + Extractor: conf.Extractor, + Cache: conf.Cache, + }, + Name: "default", + }) + if err != nil { + return config.ConfigT{}, app.Option{}, nil, nil, err + } + + opts := make([]app.Option, 0) + for _, v := range conf.SubConfig { + opt, err := getNewApp(log, v) + if err != nil { + return config.ConfigT{}, app.Option{}, nil, nil, err + } + opts = append(opts, opt) + } + return conf, defapp, opts, log, nil +} + +func signalsCatcher(conf_file string, log logger.T, app *app.AppLogic, s *http.Server) { sigint := make(chan os.Signal, 1) signal.Notify(sigint, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) for { switch <-sigint { case syscall.SIGHUP: log.LogWarning("Config reloading") + _, def, opts, lognew, err := readConfig(conf_file) + if err != nil { + log.LogError("Config reload error", err) + } else { + app.ReloadConfig(logger.NewLayer(lognew, "App"), def, opts) + log = lognew + } case syscall.SIGINT: fallthrough case syscall.SIGTERM: @@ -140,19 +146,34 @@ func signalsCatcher(log logger.T, app *app.T, s *http.Server) { } } -func getNewApp(log logger.T, v config.SubConfigT) app.Option { - subcheck := func(err error, name string, errorcode int) { - checkOrExit(err, v.Name+" "+name, errorcode) +func getNewApp(log logger.T, v config.SubConfigT) (app.Option, error) { + texts := [3]string{ + "Extractor", + "Cache", + "Streamer", } + newname := func(name string) string { return fmt.Sprintf("[%s] %s", v.Name, name) } - xtr, err := extractor.New(v.Extractor, logger.NewLayer(log, newname(texts[2]))) - subcheck(err, texts[2], ExtractorError) - cch, err := cache.New(v.Cache, logger.NewLayer(log, newname(texts[3]))) - subcheck(err, texts[3], CacheError) - strm, err := streamer.New(v.Streamer, logger.NewLayer(log, newname(texts[4])), xtr) - subcheck(err, texts[4], StreamerError) + nameerr := func(name string, err error) error { + return fmt.Errorf("%s: %s", newname(name), err) + } + xtr, err := extractor.New(v.Extractor, + logger.NewLayer(log, newname(texts[0]))) + if err != nil { + return app.Option{}, nameerr(texts[0], err) + } + cch, err := cache.New(v.Cache, + logger.NewLayer(log, newname(texts[1]))) + if err != nil { + return app.Option{}, nameerr(texts[1], err) + } + strm, err := streamer.New(v.Streamer, + logger.NewLayer(log, newname(texts[2])), xtr) + if err != nil { + return app.Option{}, nameerr(texts[2], err) + } return app.Option{ Name: v.Name, Sites: v.Sites, @@ -160,5 +181,5 @@ func getNewApp(log logger.T, v config.SubConfigT) app.Option { S: strm, C: cch, L: logger.NewLayer(log, fmt.Sprintf("[%s] app", v.Name)), - } + }, nil } diff --git a/lib/app/app.go b/lib/app/app.go index 0da67f8..2994fe3 100644 --- a/lib/app/app.go +++ b/lib/app/app.go @@ -20,7 +20,7 @@ const ( defaultVideoFormat = "mp4" ) -type appT struct { +type app struct { cache cache.T extractor extractor.T streamer streamer.T @@ -29,11 +29,11 @@ type appT struct { log logger.T } -type T struct { +type AppLogic struct { mu sync.RWMutex log logger.T - defaultApp appT - appList []appT + defaultApp app + appList []app } type Option struct { @@ -45,23 +45,25 @@ type Option struct { L logger.T } -func New( - log logger.T, - def Option, - opts []Option) *T { - t := T{ - log: log, - defaultApp: appT{ - log: def.L, - name: "default", - cache: def.C, - extractor: def.X, - streamer: def.S, - }, +func New(log logger.T, def Option, opts []Option) *AppLogic { + var t AppLogic + t.set(log, def, opts) + return &t +} + +func (t *AppLogic) set(log logger.T, def Option, opts []Option) { + t.log = log + t.defaultApp = app{ + log: def.L, + name: "default", + cache: def.C, + extractor: def.X, + streamer: def.S, } - t.appList = make([]appT, 0) + + t.appList = make([]app, 0) for _, v := range opts { - t.appList = append(t.appList, appT{ + t.appList = append(t.appList, app{ cache: v.C, extractor: v.X, streamer: v.S, @@ -70,10 +72,9 @@ func New( log: v.L, }) } - return &t } -func (t *T) selectApp(rawURL string) appT { +func (t *AppLogic) selectApp(rawURL string) app { host, err := parseUrlHost(rawURL) if err == nil { for _, v := range t.appList { @@ -90,7 +91,7 @@ func parseUrlHost(rawURL string) (string, error) { return u.Host, err } -func (t *T) Run(w http.ResponseWriter, r *http.Request) { +func (t *AppLogic) Run(w http.ResponseWriter, r *http.Request) { t.mu.RLock() defer t.mu.RUnlock() printExpired := func(links []extractor_config.RequestT) { @@ -120,7 +121,7 @@ func (t *T) Run(w http.ResponseWriter, r *http.Request) { } } -func (t *appT) play( +func (t *app) play( w http.ResponseWriter, r *http.Request, req extractor_config.RequestT, @@ -133,7 +134,7 @@ func (t *appT) play( } } -func (t *appT) playError( +func (t *app) playError( w http.ResponseWriter, req extractor_config.RequestT, err error, @@ -144,13 +145,13 @@ func (t *appT) playError( } } -func (t *appT) cacheCheck(req extractor_config.RequestT, now time.Time) (extractor_config.ResultT, bool, []extractor_config.RequestT) { +func (t *app) cacheCheck(req extractor_config.RequestT, now time.Time) (extractor_config.ResultT, bool, []extractor_config.RequestT) { expired := t.cache.CleanExpired(now) res, ok := t.cache.Get(req) return res, ok, expired } -func (t *appT) cacheAdd( +func (t *app) cacheAdd( req extractor_config.RequestT, res extractor_config.ResultT, now time.Time, @@ -193,8 +194,18 @@ func parseQuery(query string) extractor_config.RequestT { return req } -func (t *T) Shutdown() { +func (t *AppLogic) Shutdown() { t.mu.Lock() // locking app forever t.log.LogInfo("Exiting") t.log.Close() } + +func (t *AppLogic) ReloadConfig(log logger.T, def Option, opts []Option) { + t.log.LogDebug("Waiting for clients disconnect to reload app") + t.mu.Lock() + defer t.mu.Unlock() + t.log.LogInfo("Reloading app") + t.log.Close() + t.set(log, def, opts) + t.log.LogInfo("Reloading complete") +} From 338d6499fb9c6011761105a5c1e29dca32fa7ee8 Mon Sep 17 00:00:00 2001 From: mejgun Date: Fri, 20 Sep 2024 14:47:17 +0300 Subject: [PATCH 28/29] edit changelog & default config fix https option --- CHANGELOG.md | 11 ++++- README.md | 12 +---- config.example.jsonc | 86 +++++++++++++++++++++++++++------- lib/config/config.go | 6 +-- lib/extractor/config/config.go | 2 +- lib/extractor/extractor.go | 2 +- 6 files changed, 84 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfeb984..0d333d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,21 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] -- app refactored & reworked + +## 2.0.0 - 2024-09-20 +app refactored & reworked ### Added -- direct extractor (returning same url) +- per site settings +- direct/no extractor (returning same url) - json format logs - disabling logs - force https links to extractor options - host setting in config - stripping (bad) http(s) prefix in url +- os signals catching +- config reload (SIGHUP) +### Changed +- default config name from "config.json" to "config.jsonc" ## 1.6.0 - 2024-09-13 ### Added diff --git a/README.md b/README.md index 1ec7310..cda6c3e 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,4 @@ This is yt-dlp based video restreamer, part of another project: https://github.c ### Options ### -Run with `--help` - -### Exit codes ### - - - 0 - OK - - 1 - config read/parse error - - 2 - logger create error - - 3 - extractor create error - - 4 - streamer create error - - 5 - web server error - - 6 - links cache error \ No newline at end of file +Run with `--help` \ No newline at end of file diff --git a/config.example.jsonc b/config.example.jsonc index 9f2d667..068707b 100644 --- a/config.example.jsonc +++ b/config.example.jsonc @@ -9,7 +9,24 @@ // web server listen port. // DEFAULT 8080 "port": 8080, - // restreamer config. + // logger config + "log": { + // log level + // debug/info/warning/error/nothing + // DEFAULT "info" + "level": "info", + // log destination + // stdout/file/both + // DEFAULT "stdout" + "output": "stdout", + // filename if writing to file + // DEFAULT "log.txt" + "filename": "log.txt", + // set output to json format + // DEFAULT false + "json": false + }, + // default restreamer config. // restreamer takes https stream and restream it as http. "streamer": { // show errors in headers (insecure). @@ -49,48 +66,83 @@ // DEFAULT "TLS 1.2" "min-tls-version": "TLS 1.2" }, - // media extractor config + // default media extractor config "extractor": { // file path + // "direct" - do not use extractor, just pass url to streamer + // DEFAULT "yt-dlp" "path": "yt-dlp", // arguments for extractor // args separator is ",,", not space // {{.HEIGHT}} will be replaced with requested height (360/480/720) // {{.URL}} will be replace with requested url // also you can use {{.FORMAT}} - requested format (now - only mp4 or m4a) + // DEFAULT "-f,,(mp4)[height<={{.HEIGHT}}],,-g,,{{.URL}}", "mp4": "-f,,(mp4)[height<={{.HEIGHT}}],,-g,,{{.URL}}", // same for m4a + // DEFAULT "-f,,(m4a),,-g,,{{.URL}}", "m4a": "-f,,(m4a),,-g,,{{.URL}}", // args for getting user-agent + // DEFAULT "--dump-user-agent" "get-user-agent": "--dump-user-agent", + // add "https://" to links passed to extractor + // DEFAULT true + "force-https": true, // custom options list to extractor, like proxy, etc. // same rules as mp4/m4a // HEIGHT/URL/.. templates also can be used - // "custom-options": [] + // DEFAULT [] "custom-options": [ "--option1,,value1", "--option2", "value2", "--option3", - "very long value 3" + "very long value 3", + "--option4,,very long value 4" ] }, - // logger config - "log": { - // log level - // debug/info/warning/error/nothing - "level": "info", - // log destination - // stdout/file/both - "output": "stdout", - // filename if writing to file - "filename": "log.txt" - }, - // links cache config + // default links cache config "cache": { // links expire time // time units are "s", "m", "h", e.g. "1h10m10s", "10h", "1s" // "0s" will disable cache + // DEFAULT "3h" "expire-time": "3h" - } + }, + // per site configs for streamer, extractor and cache. + // absent options will be set from default part. + // only exact matching domains will be affected. + // e.g. "site.com/video" matching "site.com" + // but "www.site.com/video" is not + // DEFAULT [] + "sub-config": [ + { + // sub config name. displayed in logs + // cannot be empty + "name": "some site", + // sites list + "sites": [ + "site.com", + "a.site.com" + ], + "extractor": { + "path": "my-extractor", + "mp4": "{{.URL}}" + } + }, + { + "name": "my stream", + "sites": [ + "my.streamer.example" + ], + "extractor": { + "path": "direct" + }, + "streamer": { + "error-headers": true, + "ignore-missing-headers": true, + "ignore-ssl-errors": true + } + } + ] } \ No newline at end of file diff --git a/lib/config/config.go b/lib/config/config.go index ee6da16..5e60e48 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -68,7 +68,7 @@ func defaultConfig() ConfigT { M4A: &e[2], GetUserAgent: &e[3], CustomOptions: &co, - ForceHttp: &tru, + ForceHttps: &tru, }, Log: logger_config.ConfigT{ Level: &ll, @@ -135,8 +135,8 @@ func appendConfig(src ConfigT, dst ConfigT) ConfigT { if dst.Extractor.CustomOptions == nil { dst.Extractor.CustomOptions = src.Extractor.CustomOptions } - if dst.Extractor.ForceHttp == nil { - dst.Extractor.ForceHttp = src.Extractor.ForceHttp + if dst.Extractor.ForceHttps == nil { + dst.Extractor.ForceHttps = src.Extractor.ForceHttps } // logger if dst.Log.Level == nil { diff --git a/lib/extractor/config/config.go b/lib/extractor/config/config.go index 7f384f6..218fd60 100644 --- a/lib/extractor/config/config.go +++ b/lib/extractor/config/config.go @@ -8,7 +8,7 @@ type ConfigT struct { M4A *string `json:"m4a"` GetUserAgent *string `json:"get-user-agent"` CustomOptions *[]string `json:"custom-options"` - ForceHttp *bool `json:"force-http"` + ForceHttps *bool `json:"force-https"` } type ResultT struct { diff --git a/lib/extractor/extractor.go b/lib/extractor/extractor.go index 9768d8f..ee82f6a 100644 --- a/lib/extractor/extractor.go +++ b/lib/extractor/extractor.go @@ -21,7 +21,7 @@ func New(c extractor_config.ConfigT, log logger.T) (T, error) { ext layer err error ) - ext.force_http = *c.ForceHttp + ext.force_http = *c.ForceHttps if ext.force_http { log.LogDebug("", "force-http", true) } From d9ca744bcc56062d990c790417572767fbc1949f Mon Sep 17 00:00:00 2001 From: mejgun Date: Fri, 20 Sep 2024 14:49:44 +0300 Subject: [PATCH 29/29] fix build script --- .gitignore | 1 + build.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 028726f..caffbd9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ bin/ ytproxy log.txt +all.md5 config.jsonc diff --git a/build.sh b/build.sh index 49237e5..b7754b6 100755 --- a/build.sh +++ b/build.sh @@ -9,7 +9,7 @@ FilePrefix="yt-proxy" date >${Md5File} mkdir -p ${BinDir} rm ${BinDir}/${FilePrefix}* -rf -cd src +cd cmd go run ../build.go ${FilePrefix} cd ../$BinDir md5sum -b ${FilePrefix}* >${Md5File}