From 54b8b60edd5933116cab4c6130cdcb7bfc87a327 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 09:18:31 -0400 Subject: [PATCH 01/10] add initial local spam filter --- contrib/backends/srndv2/src/srnd/config.go | 36 +++++++++++++++++++ contrib/backends/srndv2/src/srnd/daemon.go | 10 ++++++ .../backends/srndv2/src/srnd/frontend_http.go | 2 +- contrib/backends/srndv2/src/srnd/nntp.go | 2 +- contrib/backends/srndv2/src/srnd/store.go | 8 +++-- 5 files changed, 54 insertions(+), 4 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/config.go b/contrib/backends/srndv2/src/srnd/config.go index 9eca5fa..645d30e 100644 --- a/contrib/backends/srndv2/src/srnd/config.go +++ b/contrib/backends/srndv2/src/srnd/config.go @@ -5,18 +5,45 @@ package srnd import ( + "bufio" + "bytes" "encoding/base32" "fmt" "github.com/majestrate/configparser" "github.com/majestrate/nacl" + "io/ioutil" "log" "net" "os" "path/filepath" + "regexp" "strings" "time" ) +type FilterConfig struct { + globalFilters []*regexp.Regexp +} + +func (fc *FilterConfig) LoadFile(fname string) (err error) { + var data []byte + data, err = ioutil.ReadFile(fname) + if err == nil { + r := bytes.NewReader(data) + sc := bufio.NewScanner(r) + for sc.Scan() { + txt := sc.Text() + idx := strings.Index(txt, "#") + if idx >= 0 { + txt = txt[:idx] + } + fc.globalFilters = append(fc.globalFilters, regexp.MustCompile(txt)) + } + + } + return +} + type FeedConfig struct { policy FeedPolicy quarks map[string]string @@ -70,6 +97,7 @@ type SRNdConfig struct { pprof *ProfilingConfig hooks []*HookConfig inboundPolicy *FeedPolicy + filter FilterConfig } // check for config files @@ -417,6 +445,14 @@ func ReadConfig() *SRNdConfig { } } + filterFile := "filters.txt" + + if CheckFile(filterFile) { + err = sconf.filter.LoadFile(filterFile) + if err != nil { + log.Fatalf("failed to load %s: %s", filterFile, err) + } + } return &sconf } diff --git a/contrib/backends/srndv2/src/srnd/daemon.go b/contrib/backends/srndv2/src/srnd/daemon.go index 60742be..0166510 100644 --- a/contrib/backends/srndv2/src/srnd/daemon.go +++ b/contrib/backends/srndv2/src/srnd/daemon.go @@ -131,6 +131,16 @@ type NNTPDaemon struct { article_lifetime time.Duration } +// return true if text passes all checks and is okay for posting +func (self *NNTPDaemon) CheckText(text string) bool { + for _, re := range self.conf.filter.globalFilters { + if re.MatchString(text) { + return false + } + } + return true +} + func (self NNTPDaemon) End() { if self.listener != nil { self.listener.Close() diff --git a/contrib/backends/srndv2/src/srnd/frontend_http.go b/contrib/backends/srndv2/src/srnd/frontend_http.go index 1f411a8..4ac4f15 100644 --- a/contrib/backends/srndv2/src/srnd/frontend_http.go +++ b/contrib/backends/srndv2/src/srnd/frontend_http.go @@ -393,7 +393,7 @@ func (self *httpFrontend) poll() { R: msg.Body, N: self.daemon.messageSizeLimitFor(nntp.Newsgroup()), } - err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), body) + err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), body, self.daemon.CheckText) } } } diff --git a/contrib/backends/srndv2/src/srnd/nntp.go b/contrib/backends/srndv2/src/srnd/nntp.go index 55d6572..e8821bf 100644 --- a/contrib/backends/srndv2/src/srnd/nntp.go +++ b/contrib/backends/srndv2/src/srnd/nntp.go @@ -565,7 +565,7 @@ func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEH // now store attachments and article err = writeMIMEHeader(f, hdr) if err == nil { - err = daemon.store.ProcessMessageBody(f, hdr, body) + err = daemon.store.ProcessMessageBody(f, hdr, body, daemon.CheckText) if err == nil { // tell daemon daemon.loadFromInfeed(msgid) diff --git a/contrib/backends/srndv2/src/srnd/store.go b/contrib/backends/srndv2/src/srnd/store.go index 06a0d0b..2686338 100644 --- a/contrib/backends/srndv2/src/srnd/store.go +++ b/contrib/backends/srndv2/src/srnd/store.go @@ -65,7 +65,7 @@ type ArticleStore interface { // process body of nntp message, register attachments and the article // write the body into writer as we go through the body // does NOT write mime header - ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) error + ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader, spamfilter func(string) bool) error // register this post with the daemon RegisterPost(nntp NNTPMessage) error // register signed message @@ -437,8 +437,12 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea return hdr } -func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) { +func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader, spamfilter func(string) bool) (err error) { err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) { + if !spamfilter(nntp.Message()) { + err = errors.New("spam message") + return + } err = self.RegisterPost(nntp) if err == nil { pk := hdr.Get("X-PubKey-Ed25519") From a60ecff7e317b3509ce0de3d64b90ade99967084 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 09:22:21 -0400 Subject: [PATCH 02/10] add log --- contrib/backends/srndv2/src/srnd/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/backends/srndv2/src/srnd/config.go b/contrib/backends/srndv2/src/srnd/config.go index 645d30e..69bc151 100644 --- a/contrib/backends/srndv2/src/srnd/config.go +++ b/contrib/backends/srndv2/src/srnd/config.go @@ -452,6 +452,7 @@ func ReadConfig() *SRNdConfig { if err != nil { log.Fatalf("failed to load %s: %s", filterFile, err) } + log.Println("loaded filters") } return &sconf } From 32e9e4b3ebaf5890ce5c6a9ea21f95d91e9abd0f Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 09:24:44 -0400 Subject: [PATCH 03/10] add log --- contrib/backends/srndv2/src/srnd/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/backends/srndv2/src/srnd/config.go b/contrib/backends/srndv2/src/srnd/config.go index 69bc151..46b25db 100644 --- a/contrib/backends/srndv2/src/srnd/config.go +++ b/contrib/backends/srndv2/src/srnd/config.go @@ -452,7 +452,7 @@ func ReadConfig() *SRNdConfig { if err != nil { log.Fatalf("failed to load %s: %s", filterFile, err) } - log.Println("loaded filters") + log.Printf("loaded %d filters", len(sconf.filter.globalFilters)) } return &sconf } From d48b585fcf0967b91f4bd8b4f0202dc957ce82d9 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 09:32:18 -0400 Subject: [PATCH 04/10] eh whatever this should work --- contrib/backends/srndv2/src/srnd/frontend_http.go | 5 +++++ contrib/backends/srndv2/src/srnd/store.go | 1 + 2 files changed, 6 insertions(+) diff --git a/contrib/backends/srndv2/src/srnd/frontend_http.go b/contrib/backends/srndv2/src/srnd/frontend_http.go index 4ac4f15..d2c276f 100644 --- a/contrib/backends/srndv2/src/srnd/frontend_http.go +++ b/contrib/backends/srndv2/src/srnd/frontend_http.go @@ -803,6 +803,11 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er return } + if !self.daemon.CheckText(pr.Message) { + e(errors.New("spam")) + return + } + if len(pr.Frontend) == 0 { // :-DDD pr.Frontend = "mongo.db.is.web.scale" diff --git a/contrib/backends/srndv2/src/srnd/store.go b/contrib/backends/srndv2/src/srnd/store.go index 2686338..902affc 100644 --- a/contrib/backends/srndv2/src/srnd/store.go +++ b/contrib/backends/srndv2/src/srnd/store.go @@ -439,6 +439,7 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader, spamfilter func(string) bool) (err error) { err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) { + log.Println(nntp.Message()) if !spamfilter(nntp.Message()) { err = errors.New("spam message") return From 5e66346662c4b8f0924108476e0a0c68eecdbbf9 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 09:32:42 -0400 Subject: [PATCH 05/10] eh whatever this should work --- contrib/backends/srndv2/src/srnd/store.go | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/backends/srndv2/src/srnd/store.go b/contrib/backends/srndv2/src/srnd/store.go index 902affc..2686338 100644 --- a/contrib/backends/srndv2/src/srnd/store.go +++ b/contrib/backends/srndv2/src/srnd/store.go @@ -439,7 +439,6 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader, spamfilter func(string) bool) (err error) { err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) { - log.Println(nntp.Message()) if !spamfilter(nntp.Message()) { err = errors.New("spam message") return From f079a1fee4512049bb1e79a39e56df0399ce6b00 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 18:26:59 -0400 Subject: [PATCH 06/10] fix history.html --- contrib/templates/placebo/graph_history.mustache | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/templates/placebo/graph_history.mustache b/contrib/templates/placebo/graph_history.mustache index f61405d..4c8efed 100644 --- a/contrib/templates/placebo/graph_history.mustache +++ b/contrib/templates/placebo/graph_history.mustache @@ -8,7 +8,8 @@ template parameters: - + + {{#i18n.Translations}}{{post_history_title}}{{/i18n.Translations}} From fa5e250595960e2597b2626aed08234db640c582 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 18:30:59 -0400 Subject: [PATCH 07/10] fix history.html --- contrib/backends/srndv2/src/srnd/varnish_cache.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/backends/srndv2/src/srnd/varnish_cache.go b/contrib/backends/srndv2/src/srnd/varnish_cache.go index 91797b7..a57f420 100644 --- a/contrib/backends/srndv2/src/srnd/varnish_cache.go +++ b/contrib/backends/srndv2/src/srnd/varnish_cache.go @@ -66,6 +66,8 @@ func (self *VarnishCache) invalidateUkko() { self.invalidate(fmt.Sprintf("%s%so/", self.varnish_url, self.prefix)) // TODO: this is lazy af self.RegenFrontPage() + // TODO: this is also lazy af + self.invalidate(fmt.Sprintf("%s%shistory.html", self.varnish_url, self.prefix)) } func (self *VarnishCache) pollRegen() { From 23ae28bc719844df648b7bb411cd9e2fe821fa66 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 18:38:48 -0400 Subject: [PATCH 08/10] fix bump bug --- contrib/backends/srndv2/src/srnd/expiration.go | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/backends/srndv2/src/srnd/expiration.go b/contrib/backends/srndv2/src/srnd/expiration.go index 7600a25..3d3dfd0 100644 --- a/contrib/backends/srndv2/src/srnd/expiration.go +++ b/contrib/backends/srndv2/src/srnd/expiration.go @@ -77,6 +77,7 @@ func (self expire) ExpireThread(group, rootMsgid string) { } } self.database.DeleteThread(rootMsgid) + self.database.DeleteArticle(rootMsgid) self.expireCache(group, rootMsgid, rootMsgid) } From f61470468ba0c98f70ef2cd5426a00c4c6f48fc6 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 18:45:51 -0400 Subject: [PATCH 09/10] add expire tool --- contrib/backends/srndv2/src/srnd/daemon.go | 6 ++++++ contrib/backends/srndv2/srnd.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/contrib/backends/srndv2/src/srnd/daemon.go b/contrib/backends/srndv2/src/srnd/daemon.go index 0166510..9445631 100644 --- a/contrib/backends/srndv2/src/srnd/daemon.go +++ b/contrib/backends/srndv2/src/srnd/daemon.go @@ -488,6 +488,12 @@ func (self *NNTPDaemon) syncPull(proxy_type, proxy_addr, remote_addr string) { } } +func (self *NNTPDaemon) ExpireAll() { + log.Println("expiring all orphans") + self.expire = createExpirationCore(self.database, self.store, self.informHooks) + self.expire.ExpireOrphans() +} + // run daemon func (self *NNTPDaemon) Run() { diff --git a/contrib/backends/srndv2/srnd.go b/contrib/backends/srndv2/srnd.go index d2cbc23..f8c5848 100644 --- a/contrib/backends/srndv2/srnd.go +++ b/contrib/backends/srndv2/srnd.go @@ -94,6 +94,9 @@ func main() { } else { fmt.Fprintf(os.Stdout, "usage: %s tool mod [[add|del] pubkey]|[do modactiongoeshere]\n", os.Args[0]) } + } else if tool == "expire" { + daemon.Setup() + daemon.ExpireAll() } else if tool == "rethumb" { if len(os.Args) >= 4 { threads := runtime.NumCPU() From 41a85416603dfc5a5dcf6e7d7a48f56aad4af57c Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 8 Aug 2017 18:48:07 -0400 Subject: [PATCH 10/10] add expire tool --- contrib/backends/srndv2/src/srnd/store.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/backends/srndv2/src/srnd/store.go b/contrib/backends/srndv2/src/srnd/store.go index 2686338..8491f76 100644 --- a/contrib/backends/srndv2/src/srnd/store.go +++ b/contrib/backends/srndv2/src/srnd/store.go @@ -428,7 +428,9 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea var msg *mail.Message msg, err = readMIMEHeader(r) f.Close() - hdr = textproto.MIMEHeader(msg.Header) + if msg != nil { + hdr = textproto.MIMEHeader(msg.Header) + } } if err != nil { log.Println("failed to load article headers for", messageID, err)