Merge pull request #190 from cargoedit/master

chore: remove unused min/max func

.github/CONTRIBUTING.md

@@ -0,0 +1,5 @@
+
+* be awesome to each other
+* fun is required, don't be a killjoy
+* **vendor everything**
+

.github/issue_template.md

@@ -0,0 +1,21 @@
+## info
+
+git revision / version: [git revision or version here]
+
+OS: [os here]
+
+Architecture: [architecture here]
+
+## problem
+
+[insert description of problem here]
+
+## backtrace / error messages
+
+Error messages: [yes/no]
+
+[insert any error messages here]
+
+Backtrace: [yes/no]
+
+[insert any backtraces here]

.gitignore

@@ -19,6 +19,7 @@ webroot
 
 # built binaries
 go
+gopherjs_go
 ./srndv2
 
 # private key
@@ -36,5 +37,6 @@ contrib/static/nntpchan.js
 contrib/static/js/nntpchan.js
 contrib/static/miner-js.js
 
+
 #docs trash
 doc/.trash

CODE_OF_CONDUCT.md

@@ -0,0 +1,30 @@
+
+**You have been visited by the CoC(K)**
+
+
+                                           _.+._              
+                                         (^\/^\/^)            
+                                          \@☆@☆@/             
+           CoC(k)                         {_____}             
+        Code Of Conduct Killer           _oOPPYbo.            
+                                   _,ooO8O'    `Ob            
+                             _,ooOPP"'  Ob      dO            
+                        _oooOP"''       `Oo.   ,O[            
+       Ob         _,ooOPP''               `YYboOP             
+      `O[     _ooOP"''                  _,oOPP"'              
+       YOooooOP'                    _ooOP"'                   
+         ''                     ,ooOP''                       
+                            ,odPP''                           
+                        _,oOP'                                
+                     ooOP"'                                   
+                  _oOP'o                                      
+                 ,OP   YOL                                    
+    ,O.         ,OP      Yb     contribute code or you're a racist    
+    dO'          "        Yb    get offended as a responsible adult
+    ]O.                   dO    spread this like happy herpes    
+     Ob        _,o.      dOP    
+     `Ooo___ooOP'`YbooodPP      just imagine what would happen
+       '`"""''     `'''         if we all decided to understand
+
+
+Sweet blessing be upon you but **ONLY** if comment `ebin` to this commit.

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2018 Jeff Becker
+Copyright (c) 2015-2020 Jeff Becker
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,7 +1,9 @@
 REPO=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 REPO_GOPATH=$(REPO)/go
 MINIFY=$(REPO_GOPATH)/bin/minify
-JS=$(REPO)/contrib/static/nntpchan.js
+STATIC_DIR=$(REPO)/contrib/static
+JS=$(STATIC_DIR)/nntpchan.js
+MINER_JS=$(STATIC_DIR)/miner-js.js
 CONTRIB_JS=$(REPO)/contrib/js/contrib
 LOCAL_JS=$(REPO)/contrib/js/nntpchan
 VENDOR_JS=$(REPO)/contrib/js/vendor
@@ -15,6 +17,12 @@ NNTPD=$(REPO)/nntpd
 GOROOT=$(shell go env GOROOT)
 GO=$(GOROOT)/bin/go
 
+GOPHERJS_GOROOT ?= $(GOROOT)
+GOPHERJS_GO = $(GOPHERJS_GOROOT)/bin/go
+
+GOPHERJS_GOPATH=$(REPO)/gopherjs_go
+GOPHERJS=$(GOPHERJS_GOPATH)/bin/gopherjs
+
 all: clean build
 
 build: js srnd
@@ -23,21 +31,27 @@ full: clean full-build
 
 full-build: srnd beta native
 
-js: $(JS)
+js: $(JS) 
 
 srnd: $(SRND)
 
 $(MINIFY):
-	GOPATH=$(REPO_GOPATH) go get -v github.com/tdewolff/minify/cmd/minify
+	GO111MODULE=on GOPATH=$(REPO_GOPATH) go get -u github.com/tdewolff/minify
+
+
+$(GOPHERJS):
+	GOROOT=$(GOPHERJS_GOROOT) GOPATH=$(GOPHERJS_GOPATH) $(GOPHERJS_GO) get -v github.com/gopherjs/gopherjs
 
 js-deps: $(MINIFY)
 
+$(MINER_JS): $(GOPHERJS) $(MINIFY)
+	rm -rf $(GOPHERJS_GOPATH)/pkg/
+	cp -rf $(SRND_DIR)/src/github.com $(GOPHERJS_GOPATH)/src/
+	GOROOT=$(GOPHERJS_GOROOT) GOPATH=$(GOPHERJS_GOPATH) $(GOPHERJS) -m -v build github.com/ZiRo-/cuckgo/miner_js -o miner.js
+	$(MINIFY) --mime=text/javascript > $(STATIC_DIR)/miner-js.js < miner.js
+	rm -f miner.js.map miner.js
+
 $(JS): js-deps
-	rm -f $(JS)
-	for f in $(CONTRIB_JS)/*.js ; do $(MINIFY) --mime=text/javascript >> $(JS) < $$f ; done
-	$(MINIFY) --mime=text/javascript >> $(JS) < $(REPO)/contrib/js/entry.js
-	for f in $(LOCAL_JS)/*.js ; do $(MINIFY) --mime=text/javascript >> $(JS) < $$f ; done
-	for f in $(VENDOR_JS)/*.js ; do $(MINIFY) --mime=text/javascript >> $(JS) < $$f ; done
 
 
 $(SRND):
@@ -70,16 +84,16 @@ test-native:
 	GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAN_DAEMON_DIR) test
 
 
-clean: clean-js clean-srnd
+clean: clean-srnd clean-js
 
-clean-full: clean clean-beta clean-native
+clean-full: clean clean-beta clean-native clean-js
 
 clean-srnd:
 	rm -f $(SRND)
 	GOROOT=$(GOROOT) $(MAKE) -C $(SRND_DIR) clean
 
 clean-js:
-	rm -f $(JS)
+	rm -f $(JS) $(MINER_JS)
 
 clean-beta:
 	rm -f $(NNTPCHAND)
@@ -91,3 +105,4 @@ clean-native:
 
 distclean: clean
 	rm -rf $(REPO_GOPATH)
+	rm -rf $(GOPHERJS_GOPATH)

README.md

@@ -1,21 +1,79 @@
 
 ![le ebin logo](nntpchan.png "ebin logo")
 
+![MIT License](https://img.shields.io/github/license/majestrate/nntpchan.svg)
+![Logo is ebin](https://img.shields.io/badge/logo-ebin-brightgreen.svg)
+
+
 **NNTPChan** (previously known as overchan) is a decentralized imageboard that uses the [NNTP protocol](https://en.wikipedia.org/wiki/Network_News_Transfer_Protocol) (network-news transfer protocol) to synchronize content between many different servers. It utilizes cryptographically signed posts to perform optional/opt-in decentralized moderation.
 
-## Getting started
+## Getting started Ubuntu 24.04 installation guide
 
-[This](doc) is a step-by-step guide for getting up-and-running with NNTPChan as well as documentation for developers who want to either work on NNTPChan directly or use NNTPChan in their aplications with the API.
+[This](doc) is a step-by-step guide for getting up-and-running with NNTPChan as well as documentation for developers who want to either work on NNTPChan directly or use NNTPChan in their aplications with the API. It works fine on ubuntu 24.04
 
-TL;DR edition:
+### Step 1: Download Go 1.15 Tarball
+```
+wget https://dl.google.com/go/go1.15.linux-amd64.tar.gz
+```
+
+### Step 2: Extract the Tarball
+```
+sudo tar -C /usr/local -xvzf go1.15.linux-amd64.tar.gz
+```
+
+### Set Up Go Environment Variables
+```
+export PATH=$PATH:/usr/local/go/bin
+export GOPATH=$HOME/go
+export PATH=$PATH:$GOPATH/bin
+```
+
+### Install the dependancies
+
+    sudo apt-get update
+    sudo apt-get --no-install-recommends install imagemagick ffmpeg sox build-essential git ca-certificates postgresql postgresql-client golang
+    
+
+### Get the NNTPChan source
+    git clone https://github.com/konamicode9/nntpchan
+    cd nntpchan
+
+### Now compile!
+
+Run `make`:
+
+    make
+
+## now its time to create the database in postgres
+```
+CREATE DATABASE root;
+CREATE USER root WITH PASSWORD 'root';
+GRANT ALL PRIVILEGES ON DATABASE root TO root;
+```
+note this only allows db root and username root
+not sure why but i will investigate oh and the default port is 5432
+your gonna need it in the installation
+
+Running NNTPChan
+================
+
+Once you have [built NNTPChan](building.md) and done [the initial setup you](setting-up.md) you can start NNTPChan.
+
+Before running make sure you run the setup command, you only need to do this one time:
+
+    ./srndv2 setup
+
+You can now start the NNTPChan node (srndv2) by running:
+
+    ./srndv2 run
+
+Now you can check out the web-interface by navigating to 127.0.0.1:18000 (default address - unless you have changed it in your `srnd.ini`) or you can [configure your newsreader](extras/configure-newsreader.md).
 
 
-    $ sudo apt update
-    $ sudo apt install --no-install-recommends install imagemagick ffmpeg sox build-essential git ca-certificates postgresql postgresql-client
-    $ git clone https://github.com/majestrate/nntpchan
-    $ cd nntpchan
-    $ make
-    $ SRND_INSTALLER=0 ./srndv2 setup 
+## Support chat 
+
+https://discord.gg/Ydss9wTk7G
+
 
 ## Bugs and issues
 
@@ -31,14 +89,6 @@ Web:
 
 * [Yukko](https://github.com/faissaloo/Yukko): ncurses based nntpchan web ui reader
 
-
-## Support
-
-Need help? Join us on IRC.
-
-1. [freenode: #nntpchan](https://webchat.freenode.net/?channels=#nntpchan)
-2. [rizon: #nntpchan](https://qchat.rizon.net/?channels=#nntpchan) - Most active
-
 ## History
 
 * started in mid 2013 on anonet
@@ -51,13 +101,6 @@ This is a graph of the post flow of the `overchan.test` newsgroup over 4 years,
 
 [source code for map generation](https://github.com/nilesr/nntpchan-mapper)
 
-## Donations
-
-Like this project? Why not help by funding it? This address pays for the server that runs `2hu-ch.org`
-
-Bitcoin: 15yuMzuueV8y5vPQQ39ZqQVz5Ey98DNrjE
-
-Monero: 46thSVXSPNhJkCgUsFD9WuCjW4K41DAHGL9khni2VEqmZZhfEZVvcukCp357rfhngZdviZMaeNdj5CLqhLyeK2qZRBCyL7Q
 
 ## Acknowledgements
 

contrib/backends/nntpchan-daemon/Makefile

@@ -6,13 +6,6 @@ NNTPCHAN_SRC := $(wildcard $(NNTPCHAN_PATH)/*.cpp)
 NNTPCHAN_HDR := $(wildcard $(NNTPCHAN_PATH)/*.hpp)
 NNTPCHAN_OBJ := $(NNTPCHAN_SRC:.cpp=.o)
 
-MUSTACHE_PATH = $(REPO)/libmustache
-
-MUSTACHE_SRC := $(wildcard $(MUSTACHE_PATH)/*.cpp)
-MUSTACHE_SRC += $(wildcard $(MUSTACHE_PATH)/*/*.cpp)
-MUSTACHE_HDR := $(wildcard $(MUSTACHE_PATH)/*.hpp)
-MUSTACHE_OBJ := $(MUSTACHE_SRC:.cpp=.o)
-
 HEADERS_PATH=$(REPO)/include
 
 TOOL_PATH := $(REPO)/tools
@@ -20,17 +13,28 @@ TOOL_PATH := $(REPO)/tools
 TOOL_SRC := $(wildcard $(TOOL_PATH)/*.cpp)
 TOOLS := $(TOOL_SRC:.cpp=)
 
+SRCS = $(NNTPCHAN_SRC) $(TOOL_SRC)
+
 OBJ := $(NNTPCHAN_OBJ)
-OBJ += $(MUSTACHE_OBJ)
 
 TEST = $(REPO)/test
 
 DAEMON_SRC = $(REPO)/daemon
 
-PKGS := libuv libsodium
+LD_FLAGS ?= 
+
+INC_FLAGS = -I$(HEADERS_PATH)
+
+ifeq ($(shell uname -s),FreeBSD)
+	LD_FLAGS += $(shell dirname $(CXX))/../lib/libc++experimental.a
+	INC_FLAGS += -I/usr/local/include
+	LD_FLAGS += /usr/local/lib/libsodium.a
+else 
+	LD_FLAGS += -lstdc++fs
+	INC_FLAGS += $(shell pkg-config --cflags libsodium)
+	LD_FLAGS += $(shell pkg-config --libs --static libsodium)
+endif
 
-LD_FLAGS := $(shell pkg-config --libs $(PKGS)) -lstdc++fs
-INC_FLAGS := $(shell pkg-config --cflags $(PKGS)) -I$(HEADERS_PATH)
 REQUIRED_CXXFLAGS = -std=c++17 -Wall -Wextra -Werror -pedantic $(INC_FLAGS)
 
 DEBUG = 1
@@ -42,19 +46,18 @@ endif
 CXXFLAGS += $(REQUIRED_CXXFLAGS)
 
 NNTPCHAN_LIB = $(REPO)/libnntpchan.a
-MUSTACHE_LIB = $(REPO)/libmustache.a
 
-LIBS = $(NNTPCHAN_LIB) $(MUSTACHE_LIB)
+LIBS = $(NNTPCHAN_LIB)
 
 EXE = $(REPO)/nntpd
 
 
 all: build
 
-build: $(EXE) $(TOOLS)
+format:
+	clang-format -i $(SRCS)
 
-$(MUSTACHE_LIB): $(MUSTACHE_OBJ)
-	$(AR) -r $(MUSTACHE_LIB) $(MUSTACHE_OBJ)
+build: $(EXE) tools
 
 $(NNTPCHAN_LIB): $(NNTPCHAN_OBJ)
 	$(AR) -r $(NNTPCHAN_LIB) $(NNTPCHAN_OBJ)
@@ -62,8 +65,10 @@ $(NNTPCHAN_LIB): $(NNTPCHAN_OBJ)
 $(EXE): $(LIBS)
 	$(CXX) $(CXXFLAGS) $(DAEMON_SRC)/main.cpp $(LIBS) $(LD_FLAGS) -o $(EXE)
 
-$(TOOLS): $(TOOL_SRC) $(LIBS)
-	$(CXX) $(CXXFLAGS) $< $(LIBS) $(LD_FLAGS) -o $@
+tools: $(TOOLS)
+
+$(TOOLS): $(LIBS)
+	$(CXX) $(CXXFLAGS) $@.cpp $(LIBS) $(LD_FLAGS) -o $@
 
 build-test: $(LIBS)
 	$(CXX) -o $(TEST) $(CXXFLAGS) test.cpp $(LIBS) $(LD_FLAGS)

contrib/backends/nntpchan-daemon/README.md

@@ -4,18 +4,16 @@ C++ rewrite
 
 requirements:
 
-* g++ 7.2.0 >= or clang 5.x >=
-
-* pkg-config
+* C++17 compiler
 
 * libsodium 1.x
 
-* libuv 1.x
-
-* boost variant (for now)
-
 * GNU Make
 
-building:
+building on freebsd:
 
-    $ make
+    $ gmake
+
+building on Linux:
+
+    $ make

contrib/backends/nntpchan-daemon/daemon/main.cpp

@@ -10,7 +10,8 @@
 #include <string>
 #include <vector>
 
-int main(int argc, char *argv[])
+
+int main(int argc, char *argv[], char * argenv[])
 {
   if (argc != 2)
   {
@@ -20,9 +21,9 @@ int main(int argc, char *argv[])
 
   nntpchan::Crypto crypto;
 
-  nntpchan::Mainloop loop;
+  std::unique_ptr<nntpchan::ev::Loop> loop(nntpchan::NewMainLoop());
 
-  nntpchan::NNTPServer nntp(loop);
+  std::unique_ptr<nntpchan::NNTPServer> nntp = std::make_unique<nntpchan::NNTPServer>(loop.get());
 
   std::string fname(argv[1]);
 
@@ -54,7 +55,7 @@ int main(int argc, char *argv[])
       return 1;
     }
 
-    nntp.SetStoragePath(storeconf["store_path"]);
+    nntp->SetStoragePath(storeconf["store_path"]);
 
     auto &nntpconf = level.sections["nntp"].values;
 
@@ -70,11 +71,11 @@ int main(int argc, char *argv[])
       return 1;
     }
 
-    nntp.SetInstanceName(nntpconf["instance_name"]);
+    nntp->SetInstanceName(nntpconf["instance_name"]);
 
     if (nntpconf.find("authdb") != nntpconf.end())
     {
-      nntp.SetLoginDB(nntpconf["authdb"]);
+      nntp->SetLoginDB(nntpconf["authdb"]);
     }
 
     if (level.sections.find("frontend") != level.sections.end())
@@ -86,7 +87,7 @@ int main(int argc, char *argv[])
         std::cerr << "frontend section provided but 'type' value not provided" << std::endl;
         return 1;
       }
-      auto ftype = frontconf["type"];
+      auto &ftype = frontconf["type"];
       if (ftype == "exec")
       {
         if (frontconf.find("exec") == frontconf.end())
@@ -94,7 +95,7 @@ int main(int argc, char *argv[])
           std::cerr << "exec frontend specified but no 'exec' value provided" << std::endl;
           return 1;
         }
-        nntp.SetFrontend(new nntpchan::ExecFrontend(frontconf["exec"]));
+        nntp->SetFrontend(new nntpchan::ExecFrontend(frontconf["exec"], argenv));
       }
       else if (ftype == "staticfile")
       {
@@ -113,8 +114,14 @@ int main(int argc, char *argv[])
           std::cerr << "max_pages invalid value '" << frontconf["max_pages"] << "'" << std::endl;
           return 1;
         }
-        nntp.SetFrontend(new nntpchan::StaticFileFrontend(nntpchan::CreateTemplateEngine(frontconf["template_dialect"]),
-                                                          frontconf["template_dir"], frontconf["out_dir"], maxPages));
+        auto & dialect = frontconf["template_dialect"];
+        auto templateEngine = nntpchan::CreateTemplateEngine(dialect);
+        if(templateEngine == nullptr)
+        {
+          std::cerr << "invalid template dialect '" << dialect << "'" << std::endl;
+          return 1;
+        }
+        nntp->SetFrontend(new nntpchan::StaticFileFrontend(templateEngine, frontconf["template_dir"], frontconf["out_dir"], maxPages));
       }
       else
       {
@@ -122,21 +129,32 @@ int main(int argc, char *argv[])
         return 1;
       }
     }
+    else 
+    {
+      std::cerr << "no frontend configured, running without generating markup" << std::endl;
+    }
 
     auto &a = nntpconf["bind"];
 
     try
     {
-      nntp.Bind(a);
+      if(nntp->Bind(a))
+      {
+        std::cerr << "nntpd for " << nntp->InstanceName() << " bound to " << a << std::endl;
+      }
+      else 
+      {
+        std::cerr << "nntpd for " << nntp->InstanceName() << " failed to bind to " << a << ": "<< strerror(errno) << std::endl;
+        return 1;
+      }
     } catch (std::exception &ex)
     {
       std::cerr << "failed to bind: " << ex.what() << std::endl;
       return 1;
     }
 
-    std::cerr << "nntpd for " << nntp.InstanceName() << " bound to " << a << std::endl;
-
-    loop.Run();
+    loop->Run();
+    std::cerr << "Exiting" << std::endl;
   }
   else
   {

contrib/backends/nntpchan-daemon/include/mstch/mstch.hpp

@@ -1,113 +0,0 @@
-#pragma once
-
-#include <vector>
-#include <map>
-#include <string>
-#include <memory>
-#include <functional>
-
-#include <boost/variant.hpp>
-
-namespace mstch {
-
-struct config {
-  static std::function<std::string(const std::string&)> escape;
-};
-
-namespace internal {
-
-template<class N>
-class object_t {
- public:
-  const N& at(const std::string& name) const {
-    cache[name] = (methods.at(name))();
-    return cache[name];
-  }
-
-  bool has(const std::string name) const {
-    return methods.count(name) != 0;
-  }
-
- protected:
-  template<class S>
-  void register_methods(S* s, std::map<std::string,N(S::*)()> methods) {
-    for(auto& item: methods)
-      this->methods.insert({item.first, std::bind(item.second, s)});
-  }
-
- private:
-  std::map<std::string, std::function<N()>> methods;
-  mutable std::map<std::string, N> cache;
-};
-
-template<class T, class N>
-class is_fun {
- private:
-  using not_fun = char;
-  using fun_without_args = char[2];
-  using fun_with_args = char[3];
-  template <typename U, U> struct really_has;
-  template <typename C> static fun_without_args& test(
-      really_has<N(C::*)() const, &C::operator()>*);
-  template <typename C> static fun_with_args& test(
-      really_has<N(C::*)(const std::string&) const,
-      &C::operator()>*);
-  template <typename> static not_fun& test(...);
-
- public:
-  static bool const no_args = sizeof(test<T>(0)) == sizeof(fun_without_args);
-  static bool const has_args = sizeof(test<T>(0)) == sizeof(fun_with_args);
-};
-
-template<class N>
-using node_renderer = std::function<std::string(const N& n)>;
-
-template<class N>
-class lambda_t {
- public:
-  template<class F>
-  lambda_t(F f, typename std::enable_if<is_fun<F, N>::no_args>::type* = 0):
-      fun([f](node_renderer<N> renderer, const std::string&) {
-        return renderer(f());
-      })
-  {
-  }
-
-  template<class F>
-  lambda_t(F f, typename std::enable_if<is_fun<F, N>::has_args>::type* = 0):
-      fun([f](node_renderer<N> renderer, const std::string& text) {
-        return renderer(f(text));
-      })
-  {
-  }
-
-  std::string operator()(node_renderer<N> renderer,
-      const std::string& text = "") const
-  {
-    return fun(renderer, text);
-  }
-
- private:
-  std::function<std::string(node_renderer<N> renderer, const std::string&)> fun;
-};
-
-}
-
-using node = boost::make_recursive_variant<
-    std::nullptr_t, std::string, int, double, bool,
-    internal::lambda_t<boost::recursive_variant_>,
-    std::shared_ptr<internal::object_t<boost::recursive_variant_>>,
-    std::map<const std::string, boost::recursive_variant_>,
-    std::vector<boost::recursive_variant_>>::type;
-using object = internal::object_t<node>;
-using lambda = internal::lambda_t<node>;
-using map = std::map<const std::string, node>;
-using array = std::vector<node>;
-
-std::string render(
-    const std::string& tmplt,
-    const node& root,
-    const std::map<std::string,std::string>& partials =
-        std::map<std::string,std::string>());
-
-}

contrib/backends/nntpchan-daemon/include/nntpchan/buffer.hpp

@@ -1,19 +0,0 @@
-#ifndef NNTPCHAN_BUFFER_HPP
-#define NNTPCHAN_BUFFER_HPP
-#include <string>
-#include <uv.h>
-
-namespace nntpchan
-{
-struct WriteBuffer
-{
-  uv_write_t w;
-  uv_buf_t b;
-
-  WriteBuffer(const std::string &s);
-  WriteBuffer(const char *b, const size_t s);
-  ~WriteBuffer();
-};
-}
-
-#endif

contrib/backends/nntpchan-daemon/include/nntpchan/event.hpp

@@ -1,23 +1,52 @@
 #ifndef NNTPCHAN_EVENT_HPP
 #define NNTPCHAN_EVENT_HPP
-#include <uv.h>
+
+#include <unistd.h>
+#include <cstdint>
+#include <string>
+#include <sys/socket.h>
 
 namespace nntpchan
 {
-class Mainloop
+namespace ev
 {
-public:
-  Mainloop();
-  ~Mainloop();
+  struct io
+  {
+    int fd;
 
-  operator uv_loop_t *() const { return m_loop; }
+    io(int f) : fd(f) {};
+    virtual ~io() {};
+    virtual bool readable() const { return true; };
+    virtual int read(char * buf, size_t sz) = 0;
+    virtual bool writeable() const { return true; };
+    virtual int write(size_t avail) = 0;
+    virtual bool keepalive() = 0;
+    virtual void close() 
+    { 
+      if(fd!=-1)
+      {
+        ::close(fd);
+      }
+    };
+    virtual bool acceptable() const { return false; };
+    virtual int accept() { return -1; };
+  };
 
-  void Run(uv_run_mode mode = UV_RUN_DEFAULT);
-  void Stop();
+  struct Loop
+  {
+  public:
+    virtual ~Loop() {};
+
+    bool BindTCP(const sockaddr * addr, ev::io * handler);
+    virtual bool TrackConn(ev::io * handler) = 0;
+    virtual void UntrackConn(ev::io * handler) = 0;
+    virtual void Run() = 0;
+    bool SetNonBlocking(ev::io *handler);
+  };
+}
+
+ev::Loop * NewMainLoop();
 
-private:
-  uv_loop_t *m_loop;
-};
 }
 
 #endif

contrib/backends/nntpchan-daemon/include/nntpchan/exec_frontend.hpp

@@ -8,7 +8,7 @@ namespace nntpchan
 class ExecFrontend : public Frontend
 {
 public:
-  ExecFrontend(const std::string &exe);
+  ExecFrontend(const std::string &exe, char * const* env);
 
   ~ExecFrontend();
 
@@ -20,6 +20,7 @@ private:
   int Exec(std::deque<std::string> args);
 
 private:
+  char * const* m_Environ;
   std::string m_exec;
 };
 }

contrib/backends/nntpchan-daemon/include/nntpchan/frontend.hpp

@@ -11,6 +11,7 @@ namespace fs = std::experimental::filesystem;
 class Frontend
 {
 public:
+  virtual ~Frontend() {};
   /** @brief process an inbound message stored at fpath that we have accepted. */
   virtual void ProcessNewMessage(const fs::path &fpath) = 0;
 

contrib/backends/nntpchan-daemon/include/nntpchan/line.hpp

@@ -2,6 +2,8 @@
 #define NNTPCHAN_LINE_HPP
 #include "server.hpp"
 #include <stdint.h>
+#include <sstream>
+
 namespace nntpchan
 {
 
@@ -9,23 +11,18 @@ namespace nntpchan
 class LineReader
 {
 public:
-  LineReader(size_t lineLimit);
 
   /** @brief queue inbound data from connection */
   void Data(const char *data, ssize_t s);
 
-  /** implements IConnHandler */
-  virtual bool ShouldClose();
-
 protected:
   /** @brief handle a line from the client */
-  virtual void HandleLine(const std::string &line) = 0;
+  virtual void HandleLine(const std::string line) = 0;
 
 private:
-  void OnLine(const char *d, const size_t l);
-  std::string m_leftovers;
-  bool m_close;
-  const size_t lineLimit;
+  
+  std::stringstream m_line;
+  std::string m_leftover;
 };
 }
 

contrib/backends/nntpchan-daemon/include/nntpchan/message.hpp

@@ -10,6 +10,7 @@ struct MessageDB
 {
   using BoardPage = nntpchan::model::BoardPage;
   using Thread = nntpchan::model::Thread;
+  virtual ~MessageDB() {};
   virtual bool LoadBoardPage(BoardPage &board, const std::string &newsgroup, uint32_t perpage, uint32_t page) const = 0;
   virtual bool FindThreadByHash(const std::string &hashhex, std::string &msgid) const = 0;
   virtual bool LoadThread(Thread &thread, const std::string &rootmsgid) const = 0;

contrib/backends/nntpchan-daemon/include/nntpchan/model.hpp

@@ -26,7 +26,13 @@ typedef std::tuple<PostHeader, PostBody, Attachments> Post;
 // a thread (many posts in post order)
 typedef std::vector<Post> Thread;
 // a board page is many threads in bump order
-typedef std::vector<Thread> BoardPage;
+struct BoardPage
+{
+  std::vector<Thread> threads = {};
+  std::string name = "";
+  uint32_t pageno = 0;
+};
+
 
 static inline const std::string &GetFilename(const PostAttachment &att) { return std::get<0>(att); }
 

contrib/backends/nntpchan-daemon/include/nntpchan/net.hpp

@@ -4,6 +4,7 @@
 #include <netinet/in.h>
 #include <string>
 #include <sys/types.h>
+#include <sys/socket.h>
 
 namespace nntpchan
 {

contrib/backends/nntpchan-daemon/include/nntpchan/nntp_auth.hpp

@@ -35,7 +35,7 @@ protected:
   void SetStream(std::istream *i);
 
   std::string Hash(const std::string &data, const std::string &salt);
-  void HandleLine(const std::string &line);
+  void HandleLine(const std::string line);
 
 private:
   bool ProcessLine(const std::string &line);

contrib/backends/nntpchan-daemon/include/nntpchan/nntp_handler.hpp

@@ -11,7 +11,7 @@ namespace nntpchan
 class NNTPServerHandler : public LineReader, public IConnHandler
 {
 public:
-  NNTPServerHandler(const fs::path &storage);
+  NNTPServerHandler(fs::path storage);
   ~NNTPServerHandler();
 
   virtual bool ShouldClose();
@@ -23,7 +23,7 @@ public:
   void Greet();
 
 protected:
-  void HandleLine(const std::string &line);
+  void HandleLine(const std::string line);
   void HandleCommand(const std::deque<std::string> &command);
 
 private:
@@ -51,7 +51,7 @@ private:
   std::string m_articleName;
   FileHandle_ptr m_article;
   CredDB_ptr m_auth;
-  ArticleStorage_ptr m_store;
+  ArticleStorage m_store;
   std::string m_mode;
   bool m_authed;
   State m_state;

contrib/backends/nntpchan-daemon/include/nntpchan/nntp_server.hpp

@@ -4,7 +4,6 @@
 #include "server.hpp"
 #include <deque>
 #include <string>
-#include <uv.h>
 
 namespace nntpchan
 {
@@ -12,7 +11,7 @@ namespace nntpchan
 class NNTPServer : public Server
 {
 public:
-  NNTPServer(uv_loop_t *loop);
+  NNTPServer(ev::Loop * loop);
 
   virtual ~NNTPServer();
 
@@ -24,9 +23,7 @@ public:
 
   std::string InstanceName() const;
 
-  void Close();
-
-  virtual IServerConn *CreateConn(uv_stream_t *s);
+  virtual IServerConn *CreateConn(int fd);
 
   virtual void OnAcceptError(int status);
 
@@ -43,13 +40,10 @@ private:
 class NNTPServerConn : public IServerConn
 {
 public:
-  NNTPServerConn(uv_loop_t *l, uv_stream_t *s, Server *parent, IConnHandler *h) : IServerConn(l, s, parent, h) {}
+  NNTPServerConn(int fd, Server *parent, IConnHandler *h) : IServerConn(fd, parent, h) {}
 
   virtual bool IsTimedOut() { return false; };
 
-  /** @brief send next queued reply */
-  virtual void SendNextReply();
-
   virtual void Greet();
 };
 }

contrib/backends/nntpchan-daemon/include/nntpchan/server.hpp

@@ -3,7 +3,7 @@
 #include <deque>
 #include <functional>
 #include <string>
-#include <uv.h>
+#include <nntpchan/event.hpp>
 
 namespace nntpchan
 {
@@ -37,38 +37,45 @@ private:
 };
 
 /** server connection handler interface */
-struct IServerConn
+struct IServerConn : public ev::io
 {
-  IServerConn(uv_loop_t *l, uv_stream_t *s, Server *parent, IConnHandler *h);
+  IServerConn(int fd, Server *parent, IConnHandler *h);
   virtual ~IServerConn();
-  virtual void Close();
+  virtual int read(char * buf, size_t sz);
+  virtual int write(size_t avail);
+  virtual void close();
   virtual void Greet() = 0;
-  virtual void SendNextReply() = 0;
   virtual bool IsTimedOut() = 0;
-  void SendString(const std::string &str);
+  virtual bool keepalive() ;
   Server *Parent() { return m_parent; };
   IConnHandler *GetHandler() { return m_handler; };
-  uv_loop_t *GetLoop() { return m_loop; };
 
 private:
-  uv_tcp_t m_conn;
-  uv_loop_t *m_loop;
   Server *m_parent;
   IConnHandler *m_handler;
+  std::string m_writeLeftover;
 };
 
-class Server
+class Server : public ev::io
 {
 public:
-  Server(uv_loop_t *loop);
-  /** called after socket close, NEVER call directly */
-  virtual ~Server() {}
+  Server(ev::Loop * loop);
+  virtual ~Server() {};
+
+  virtual bool acceptable() const { return true; };
+  virtual void close();
+  virtual bool readable() const { return false; };
+  virtual int read(char *,size_t) { return -1; };
+  virtual bool writeable() const { return false; };
+  virtual int write(size_t) {return -1; };
+  virtual int accept();
+  virtual bool keepalive() { return true; };
+
+
   /** create connection handler from open stream */
-  virtual IServerConn *CreateConn(uv_stream_t *s) = 0;
-  /** close all sockets and stop */
-  void Close();
+  virtual IServerConn *CreateConn(int fd) = 0;
   /** bind to address */
-  void Bind(const std::string &addr);
+  bool Bind(const std::string &addr);
 
   typedef std::function<void(IServerConn *)> ConnVisitor;
 
@@ -78,19 +85,11 @@ public:
   /** remove connection from server, called after proper close */
   void RemoveConn(IServerConn *conn);
 
-protected:
-  uv_loop_t *GetLoop() { return m_loop; }
-  virtual void OnAcceptError(int status) = 0;
-
 private:
-  operator uv_handle_t *() { return (uv_handle_t *)&m_server; }
-  operator uv_tcp_t *() { return &m_server; }
-  operator uv_stream_t *() { return (uv_stream_t *)&m_server; }
 
-  void OnAccept(uv_stream_t *s, int status);
+  void OnAccept(int fd);
+  ev::Loop * m_Loop;
   std::deque<IServerConn *> m_conns;
-  uv_tcp_t m_server;
-  uv_loop_t *m_loop;
 };
 }
 

contrib/backends/nntpchan-daemon/include/nntpchan/staticfile_frontend.hpp

@@ -16,11 +16,11 @@ class StaticFileFrontend : public Frontend
 public:
   StaticFileFrontend(TemplateEngine *tmpl, const std::string &templateDir, const std::string &outDir, uint32_t pages);
 
-  ~StaticFileFrontend();
+  virtual ~StaticFileFrontend();
 
-  void ProcessNewMessage(const fs::path &fpath);
-  bool AcceptsNewsgroup(const std::string &newsgroup);
-  bool AcceptsMessage(const std::string &msgid);
+  virtual void ProcessNewMessage(const fs::path &fpath);
+  virtual bool AcceptsNewsgroup(const std::string &newsgroup);
+  virtual bool AcceptsMessage(const std::string &msgid);
 
 private:
   MessageDB_ptr m_MessageDB;

contrib/backends/nntpchan-daemon/include/nntpchan/template_engine.hpp

@@ -12,13 +12,16 @@ namespace nntpchan
 
 struct TemplateEngine
 {
-  typedef std::map<std::string, std::variant<nntpchan::model::Model, std::string>> Args_t;
-  virtual bool WriteTemplate(const fs::path &template_fpath, const Args_t &args, const FileHandle_ptr &out) = 0;
+  virtual ~TemplateEngine() {};
+  virtual bool WriteBoardPage(const nntpchan::model::BoardPage & page, const FileHandle_ptr &out) = 0;
+  virtual bool WriteThreadPage(const nntpchan::model::Thread & thread, const FileHandle_ptr &out) = 0;
 };
 
-TemplateEngine *CreateTemplateEngine(const std::string &dialect);
+
 
 typedef std::unique_ptr<TemplateEngine> TemplateEngine_ptr;
+
+TemplateEngine * CreateTemplateEngine(const std::string &dialect);
 }
 
 #endif

contrib/backends/nntpchan-daemon/libmustache/mstch.cpp

@@ -1,20 +0,0 @@
-#include <iostream>
-
-#include "mstch/mstch.hpp"
-#include "render_context.hpp"
-
-using namespace mstch;
-
-std::function<std::string(const std::string&)> mstch::config::escape;
-
-std::string mstch::render(
-    const std::string& tmplt,
-    const node& root,
-    const std::map<std::string,std::string>& partials)
-{
-  std::map<std::string, template_type> partial_templates;
-  for (auto& partial: partials)
-    partial_templates.insert({partial.first, {partial.second}});
-
-  return render_context(root, partial_templates).render(tmplt);
-}

contrib/backends/nntpchan-daemon/libmustache/render_context.cpp

@@ -1,62 +0,0 @@
-#include "render_context.hpp"
-#include "state/outside_section.hpp"
-#include "visitor/get_token.hpp"
-
-using namespace mstch;
-
-const mstch::node render_context::null_node;
-
-render_context::push::push(render_context &context, const mstch::node &node) : m_context(context)
-{
-  context.m_nodes.emplace_front(node);
-  context.m_node_ptrs.emplace_front(&node);
-  context.m_state.push(std::unique_ptr<render_state>(new outside_section));
-}
-
-render_context::push::~push()
-{
-  m_context.m_nodes.pop_front();
-  m_context.m_node_ptrs.pop_front();
-  m_context.m_state.pop();
-}
-
-std::string render_context::push::render(const template_type &templt) { return m_context.render(templt); }
-
-render_context::render_context(const mstch::node &node, const std::map<std::string, template_type> &partials)
-    : m_partials(partials), m_nodes(1, node), m_node_ptrs(1, &node)
-{
-  m_state.push(std::unique_ptr<render_state>(new outside_section));
-}
-
-const mstch::node &render_context::find_node(const std::string &token, std::list<node const *> current_nodes)
-{
-  if (token != "." && token.find('.') != std::string::npos)
-    return find_node(token.substr(token.rfind('.') + 1),
-                     {&find_node(token.substr(0, token.rfind('.')), current_nodes)});
-  else
-    for (auto &node : current_nodes)
-      if (visit(has_token(token), *node))
-        return visit(get_token(token, *node), *node);
-  return null_node;
-}
-
-const mstch::node &render_context::get_node(const std::string &token) { return find_node(token, m_node_ptrs); }
-
-std::string render_context::render(const template_type &templt, const std::string &prefix)
-{
-  std::string output;
-  bool prev_eol = true;
-  for (auto &token : templt)
-  {
-    if (prev_eol && prefix.length() != 0)
-      output += m_state.top()->render(*this, {prefix});
-    output += m_state.top()->render(*this, token);
-    prev_eol = token.eol();
-  }
-  return output;
-}
-
-std::string render_context::render_partial(const std::string &partial_name, const std::string &prefix)
-{
-  return m_partials.count(partial_name) ? render(m_partials.at(partial_name), prefix) : "";
-}

contrib/backends/nntpchan-daemon/libmustache/render_context.hpp

@@ -1,47 +0,0 @@
-#pragma once
-
-#include <deque>
-#include <list>
-#include <sstream>
-#include <stack>
-#include <string>
-
-#include "state/render_state.hpp"
-#include "template_type.hpp"
-#include <mstch/mstch.hpp>
-
-namespace mstch
-{
-
-class render_context
-{
-public:
-  class push
-  {
-  public:
-    push(render_context &context, const mstch::node &node = {});
-    ~push();
-    std::string render(const template_type &templt);
-
-  private:
-    render_context &m_context;
-  };
-
-  render_context(const mstch::node &node, const std::map<std::string, template_type> &partials);
-  const mstch::node &get_node(const std::string &token);
-  std::string render(const template_type &templt, const std::string &prefix = "");
-  std::string render_partial(const std::string &partial_name, const std::string &prefix);
-  template <class T, class... Args> void set_state(Args &&... args)
-  {
-    m_state.top() = std::unique_ptr<render_state>(new T(std::forward<Args>(args)...));
-  }
-
-private:
-  static const mstch::node null_node;
-  const mstch::node &find_node(const std::string &token, std::list<node const *> current_nodes);
-  std::map<std::string, template_type> m_partials;
-  std::deque<mstch::node> m_nodes;
-  std::list<const mstch::node *> m_node_ptrs;
-  std::stack<std::unique_ptr<render_state>> m_state;
-};
-}

contrib/backends/nntpchan-daemon/libmustache/state/in_section.cpp

@@ -1,36 +0,0 @@
-#include "in_section.hpp"
-#include "../visitor/is_node_empty.hpp"
-#include "../visitor/render_section.hpp"
-#include "outside_section.hpp"
-
-using namespace mstch;
-
-in_section::in_section(type type, const token &start_token)
-    : m_type(type), m_start_token(start_token), m_skipped_openings(0)
-{
-}
-
-std::string in_section::render(render_context &ctx, const token &token)
-{
-  if (token.token_type() == token::type::section_close)
-    if (token.name() == m_start_token.name() && m_skipped_openings == 0)
-    {
-      auto &node = ctx.get_node(m_start_token.name());
-      std::string out;
-
-      if (m_type == type::normal && !visit(is_node_empty(), node))
-        out = visit(render_section(ctx, m_section, m_start_token.delims()), node);
-      else if (m_type == type::inverted && visit(is_node_empty(), node))
-        out = render_context::push(ctx).render(m_section);
-
-      ctx.set_state<outside_section>();
-      return out;
-    }
-    else
-      m_skipped_openings--;
-  else if (token.token_type() == token::type::inverted_section_open || token.token_type() == token::type::section_open)
-    m_skipped_openings++;
-
-  m_section << token;
-  return "";
-}

contrib/backends/nntpchan-daemon/libmustache/state/in_section.hpp

@@ -1,29 +0,0 @@
-#pragma once
-
-#include <sstream>
-#include <vector>
-
-#include "../template_type.hpp"
-#include "render_state.hpp"
-
-namespace mstch
-{
-
-class in_section : public render_state
-{
-public:
-  enum class type
-  {
-    inverted,
-    normal
-  };
-  in_section(type type, const token &start_token);
-  std::string render(render_context &context, const token &token) override;
-
-private:
-  const type m_type;
-  const token &m_start_token;
-  template_type m_section;
-  int m_skipped_openings;
-};
-}

contrib/backends/nntpchan-daemon/libmustache/state/outside_section.cpp

@@ -1,32 +0,0 @@
-#include "outside_section.hpp"
-
-#include "../render_context.hpp"
-#include "../visitor/render_node.hpp"
-#include "in_section.hpp"
-
-using namespace mstch;
-
-std::string outside_section::render(render_context &ctx, const token &token)
-{
-  using flag = render_node::flag;
-  switch (token.token_type())
-  {
-  case token::type::section_open:
-    ctx.set_state<in_section>(in_section::type::normal, token);
-    break;
-  case token::type::inverted_section_open:
-    ctx.set_state<in_section>(in_section::type::inverted, token);
-    break;
-  case token::type::variable:
-    return visit(render_node(ctx, flag::escape_html), ctx.get_node(token.name()));
-  case token::type::unescaped_variable:
-    return visit(render_node(ctx, flag::none), ctx.get_node(token.name()));
-  case token::type::text:
-    return token.raw();
-  case token::type::partial:
-    return ctx.render_partial(token.name(), token.partial_prefix());
-  default:
-    break;
-  }
-  return "";
-}

contrib/backends/nntpchan-daemon/libmustache/state/outside_section.hpp

@@ -1,13 +0,0 @@
-#pragma once
-
-#include "render_state.hpp"
-
-namespace mstch
-{
-
-class outside_section : public render_state
-{
-public:
-  std::string render(render_context &context, const token &token) override;
-};
-}

contrib/backends/nntpchan-daemon/libmustache/state/render_state.hpp

@@ -1,18 +0,0 @@
-#pragma once
-
-#include <memory>
-
-#include "../token.hpp"
-
-namespace mstch
-{
-
-class render_context;
-
-class render_state
-{
-public:
-  virtual ~render_state() {}
-  virtual std::string render(render_context &context, const token &token) = 0;
-};
-}

contrib/backends/nntpchan-daemon/libmustache/template_type.cpp

@@ -1,110 +0,0 @@
-#include "template_type.hpp"
-
-using namespace mstch;
-
-template_type::template_type(const std::string &str, const delim_type &delims)
-    : m_open(delims.first), m_close(delims.second)
-{
-  tokenize(str);
-  strip_whitespace();
-}
-
-template_type::template_type(const std::string &str) : m_open("{{"), m_close("}}")
-{
-  tokenize(str);
-  strip_whitespace();
-}
-
-void template_type::process_text(citer begin, citer end)
-{
-  if (begin == end)
-    return;
-  auto start = begin;
-  for (auto it = begin; it != end; ++it)
-    if (*it == '\n' || it == end - 1)
-    {
-      m_tokens.push_back({{start, it + 1}});
-      start = it + 1;
-    }
-}
-
-void template_type::tokenize(const std::string &tmp)
-{
-  citer beg = tmp.begin();
-  auto npos = std::string::npos;
-
-  for (std::size_t cur_pos = 0; cur_pos < tmp.size();)
-  {
-    auto open_pos = tmp.find(m_open, cur_pos);
-    auto close_pos = tmp.find(m_close, open_pos == npos ? open_pos : open_pos + 1);
-
-    if (close_pos != npos && open_pos != npos)
-    {
-      if (*(beg + open_pos + m_open.size()) == '{' && *(beg + close_pos + m_close.size()) == '}')
-        ++close_pos;
-
-      process_text(beg + cur_pos, beg + open_pos);
-      cur_pos = close_pos + m_close.size();
-      m_tokens.push_back({{beg + open_pos, beg + close_pos + m_close.size()}, m_open.size(), m_close.size()});
-
-      if (cur_pos == tmp.size())
-      {
-        m_tokens.push_back({{""}});
-        m_tokens.back().eol(true);
-      }
-
-      if (*(beg + open_pos + m_open.size()) == '=' && *(beg + close_pos - 1) == '=')
-      {
-        auto tok_beg = beg + open_pos + m_open.size() + 1;
-        auto tok_end = beg + close_pos - 1;
-        auto front_skip = first_not_ws(tok_beg, tok_end);
-        auto back_skip = first_not_ws(reverse(tok_end), reverse(tok_beg));
-        m_open = {front_skip, beg + tmp.find(' ', front_skip - beg)};
-        m_close = {beg + tmp.rfind(' ', back_skip - beg) + 1, back_skip + 1};
-      }
-    }
-    else
-    {
-      process_text(beg + cur_pos, tmp.end());
-      cur_pos = close_pos;
-    }
-  }
-}
-
-void template_type::strip_whitespace()
-{
-  auto line_begin = m_tokens.begin();
-  bool has_tag = false, non_space = false;
-
-  for (auto it = m_tokens.begin(); it != m_tokens.end(); ++it)
-  {
-    auto type = (*it).token_type();
-    if (type != token::type::text && type != token::type::variable && type != token::type::unescaped_variable)
-      has_tag = true;
-    else if (!(*it).ws_only())
-      non_space = true;
-
-    if ((*it).eol())
-    {
-      if (has_tag && !non_space)
-      {
-        store_prefixes(line_begin);
-
-        auto c = line_begin;
-        for (bool end = false; !end; (*c).ws_only() ? c = m_tokens.erase(c) : ++c)
-          if ((end = (*c).eol()))
-            it = c - 1;
-      }
-
-      non_space = has_tag = false;
-      line_begin = it + 1;
-    }
-  }
-}
-
-void template_type::store_prefixes(std::vector<token>::iterator beg)
-{
-  for (auto cur = beg; !(*cur).eol(); ++cur)
-    if ((*cur).token_type() == token::type::partial && cur != beg && (*(cur - 1)).ws_only())
-      (*cur).partial_prefix((*(cur - 1)).raw());
-}

contrib/backends/nntpchan-daemon/libmustache/template_type.hpp

@@ -1,31 +0,0 @@
-#pragma once
-
-#include <string>
-#include <vector>
-
-#include "token.hpp"
-#include "utils.hpp"
-
-namespace mstch
-{
-
-class template_type
-{
-public:
-  template_type() = default;
-  template_type(const std::string &str);
-  template_type(const std::string &str, const delim_type &delims);
-  std::vector<token>::const_iterator begin() const { return m_tokens.begin(); }
-  std::vector<token>::const_iterator end() const { return m_tokens.end(); }
-  void operator<<(const token &token) { m_tokens.push_back(token); }
-
-private:
-  std::vector<token> m_tokens;
-  std::string m_open;
-  std::string m_close;
-  void strip_whitespace();
-  void process_text(citer beg, citer end);
-  void tokenize(const std::string &tmp);
-  void store_prefixes(std::vector<token>::iterator beg);
-};
-}

contrib/backends/nntpchan-daemon/libmustache/token.cpp

@@ -1,57 +0,0 @@
-#include "token.hpp"
-#include "utils.hpp"
-
-using namespace mstch;
-
-token::type token::token_info(char c)
-{
-  switch (c)
-  {
-  case '>':
-    return type::partial;
-  case '^':
-    return type::inverted_section_open;
-  case '/':
-    return type::section_close;
-  case '&':
-    return type::unescaped_variable;
-  case '#':
-    return type::section_open;
-  case '!':
-    return type::comment;
-  default:
-    return type::variable;
-  }
-}
-
-token::token(const std::string &str, std::size_t left, std::size_t right) : m_raw(str), m_eol(false), m_ws_only(false)
-{
-  if (left != 0 && right != 0)
-  {
-    if (str[left] == '=' && str[str.size() - right - 1] == '=')
-    {
-      m_type = type::delimiter_change;
-    }
-    else if (str[left] == '{' && str[str.size() - right - 1] == '}')
-    {
-      m_type = type::unescaped_variable;
-      m_name = {first_not_ws(str.begin() + left + 1, str.end() - right),
-                first_not_ws(str.rbegin() + 1 + right, str.rend() - left) + 1};
-    }
-    else
-    {
-      auto c = first_not_ws(str.begin() + left, str.end() - right);
-      m_type = token_info(*c);
-      if (m_type != type::variable)
-        c = first_not_ws(c + 1, str.end() - right);
-      m_name = {c, first_not_ws(str.rbegin() + right, str.rend() - left) + 1};
-      m_delims = {{str.begin(), str.begin() + left}, {str.end() - right, str.end()}};
-    }
-  }
-  else
-  {
-    m_type = type::text;
-    m_eol = (str.size() > 0 && str[str.size() - 1] == '\n');
-    m_ws_only = (str.find_first_not_of(" \r\n\t") == std::string::npos);
-  }
-}

contrib/backends/nntpchan-daemon/libmustache/token.hpp

@@ -1,46 +0,0 @@
-#pragma once
-
-#include <string>
-
-namespace mstch
-{
-
-using delim_type = std::pair<std::string, std::string>;
-
-class token
-{
-public:
-  enum class type
-  {
-    text,
-    variable,
-    section_open,
-    section_close,
-    inverted_section_open,
-    unescaped_variable,
-    comment,
-    partial,
-    delimiter_change
-  };
-  token(const std::string &str, std::size_t left = 0, std::size_t right = 0);
-  type token_type() const { return m_type; };
-  const std::string &raw() const { return m_raw; };
-  const std::string &name() const { return m_name; };
-  const std::string &partial_prefix() const { return m_partial_prefix; };
-  const delim_type &delims() const { return m_delims; };
-  void partial_prefix(const std::string &p_partial_prefix) { m_partial_prefix = p_partial_prefix; };
-  bool eol() const { return m_eol; }
-  void eol(bool eol) { m_eol = eol; }
-  bool ws_only() const { return m_ws_only; }
-
-private:
-  type m_type;
-  std::string m_name;
-  std::string m_raw;
-  std::string m_partial_prefix;
-  delim_type m_delims;
-  bool m_eol;
-  bool m_ws_only;
-  type token_info(char c);
-};
-}

contrib/backends/nntpchan-daemon/libmustache/utils.cpp

@@ -1,61 +0,0 @@
-#include "utils.hpp"
-#include "mstch/mstch.hpp"
-
-mstch::citer mstch::first_not_ws(mstch::citer begin, mstch::citer end)
-{
-  for (auto it = begin; it != end; ++it)
-    if (*it != ' ')
-      return it;
-  return end;
-}
-
-mstch::citer mstch::first_not_ws(mstch::criter begin, mstch::criter end)
-{
-  for (auto rit = begin; rit != end; ++rit)
-    if (*rit != ' ')
-      return --(rit.base());
-  return --(end.base());
-}
-
-mstch::criter mstch::reverse(mstch::citer it) { return std::reverse_iterator<mstch::citer>(it); }
-
-std::string mstch::html_escape(const std::string &str)
-{
-  if (mstch::config::escape)
-    return mstch::config::escape(str);
-
-  std::string out;
-  citer start = str.begin();
-
-  auto add_escape = [&out, &start](const std::string &escaped, citer &it) {
-    out += std::string{start, it} + escaped;
-    start = it + 1;
-  };
-
-  for (auto it = str.begin(); it != str.end(); ++it)
-    switch (*it)
-    {
-    case '&':
-      add_escape("&amp;", it);
-      break;
-    case '\'':
-      add_escape("&#39;", it);
-      break;
-    case '"':
-      add_escape("&quot;", it);
-      break;
-    case '<':
-      add_escape("&lt;", it);
-      break;
-    case '>':
-      add_escape("&gt;", it);
-      break;
-    case '/':
-      add_escape("&#x2F;", it);
-      break;
-    default:
-      break;
-    }
-
-  return out + std::string{start, str.end()};
-}

contrib/backends/nntpchan-daemon/libmustache/utils.hpp

@@ -1,21 +0,0 @@
-#pragma once
-
-#include <boost/variant/apply_visitor.hpp>
-#include <string>
-
-namespace mstch
-{
-
-using citer = std::string::const_iterator;
-using criter = std::string::const_reverse_iterator;
-
-citer first_not_ws(citer begin, citer end);
-citer first_not_ws(criter begin, criter end);
-std::string html_escape(const std::string &str);
-criter reverse(citer it);
-
-template <class... Args> auto visit(Args &&... args) -> decltype(boost::apply_visitor(std::forward<Args>(args)...))
-{
-  return boost::apply_visitor(std::forward<Args>(args)...);
-}
-}

contrib/backends/nntpchan-daemon/libmustache/visitor/get_token.hpp

@@ -1,26 +0,0 @@
-#pragma once
-
-#include <boost/variant/static_visitor.hpp>
-
-#include "has_token.hpp"
-#include "mstch/mstch.hpp"
-
-namespace mstch
-{
-
-class get_token : public boost::static_visitor<const mstch::node &>
-{
-public:
-  get_token(const std::string &token, const mstch::node &node) : m_token(token), m_node(node) {}
-
-  template <class T> const mstch::node &operator()(const T &) const { return m_node; }
-
-  const mstch::node &operator()(const map &map) const { return map.at(m_token); }
-
-  const mstch::node &operator()(const std::shared_ptr<object> &object) const { return object->at(m_token); }
-
-private:
-  const std::string &m_token;
-  const mstch::node &m_node;
-};
-}

contrib/backends/nntpchan-daemon/libmustache/visitor/has_token.hpp

@@ -1,24 +0,0 @@
-#pragma once
-
-#include <boost/variant/static_visitor.hpp>
-
-#include "mstch/mstch.hpp"
-
-namespace mstch
-{
-
-class has_token : public boost::static_visitor<bool>
-{
-public:
-  has_token(const std::string &token) : m_token(token) {}
-
-  template <class T> bool operator()(const T &) const { return m_token == "."; }
-
-  bool operator()(const map &map) const { return map.count(m_token) == 1; }
-
-  bool operator()(const std::shared_ptr<object> &object) const { return object->has(m_token); }
-
-private:
-  const std::string &m_token;
-};
-}

contrib/backends/nntpchan-daemon/libmustache/visitor/is_node_empty.hpp

@@ -1,27 +0,0 @@
-#pragma once
-
-#include <boost/variant/static_visitor.hpp>
-
-#include "mstch/mstch.hpp"
-
-namespace mstch
-{
-
-class is_node_empty : public boost::static_visitor<bool>
-{
-public:
-  template <class T> bool operator()(const T &) const { return false; }
-
-  bool operator()(const std::nullptr_t &) const { return true; }
-
-  bool operator()(const int &value) const { return value == 0; }
-
-  bool operator()(const double &value) const { return value == 0; }
-
-  bool operator()(const bool &value) const { return !value; }
-
-  bool operator()(const std::string &value) const { return value == ""; }
-
-  bool operator()(const array &array) const { return array.size() == 0; }
-};
-}

contrib/backends/nntpchan-daemon/libmustache/visitor/render_node.hpp

@@ -1,52 +0,0 @@
-#pragma once
-
-#include <boost/variant/static_visitor.hpp>
-#include <sstream>
-
-#include "../render_context.hpp"
-#include "../utils.hpp"
-#include "mstch/mstch.hpp"
-
-namespace mstch
-{
-
-class render_node : public boost::static_visitor<std::string>
-{
-public:
-  enum class flag
-  {
-    none,
-    escape_html
-  };
-  render_node(render_context &ctx, flag p_flag = flag::none) : m_ctx(ctx), m_flag(p_flag) {}
-
-  template <class T> std::string operator()(const T &) const { return ""; }
-
-  std::string operator()(const int &value) const { return std::to_string(value); }
-
-  std::string operator()(const double &value) const
-  {
-    std::stringstream ss;
-    ss << value;
-    return ss.str();
-  }
-
-  std::string operator()(const bool &value) const { return value ? "true" : "false"; }
-
-  std::string operator()(const lambda &value) const
-  {
-    template_type interpreted{value([this](const mstch::node &n) { return visit(render_node(m_ctx), n); })};
-    auto rendered = render_context::push(m_ctx).render(interpreted);
-    return (m_flag == flag::escape_html) ? html_escape(rendered) : rendered;
-  }
-
-  std::string operator()(const std::string &value) const
-  {
-    return (m_flag == flag::escape_html) ? html_escape(value) : value;
-  }
-
-private:
-  render_context &m_ctx;
-  flag m_flag;
-};
-}

contrib/backends/nntpchan-daemon/libmustache/visitor/render_section.hpp

@@ -1,58 +0,0 @@
-#pragma once
-
-#include <boost/variant/static_visitor.hpp>
-
-#include "../render_context.hpp"
-#include "../utils.hpp"
-#include "mstch/mstch.hpp"
-#include "render_node.hpp"
-
-namespace mstch
-{
-
-class render_section : public boost::static_visitor<std::string>
-{
-public:
-  enum class flag
-  {
-    none,
-    keep_array
-  };
-  render_section(render_context &ctx, const template_type &section, const delim_type &delims, flag p_flag = flag::none)
-      : m_ctx(ctx), m_section(section), m_delims(delims), m_flag(p_flag)
-  {
-  }
-
-  template <class T> std::string operator()(const T &t) const
-  {
-    return render_context::push(m_ctx, t).render(m_section);
-  }
-
-  std::string operator()(const lambda &fun) const
-  {
-    std::string section_str;
-    for (auto &token : m_section)
-      section_str += token.raw();
-    template_type interpreted{fun([this](const mstch::node &n) { return visit(render_node(m_ctx), n); }, section_str),
-                              m_delims};
-    return render_context::push(m_ctx).render(interpreted);
-  }
-
-  std::string operator()(const array &array) const
-  {
-    std::string out;
-    if (m_flag == flag::keep_array)
-      return render_context::push(m_ctx, array).render(m_section);
-    else
-      for (auto &item : array)
-        out += visit(render_section(m_ctx, m_section, m_delims, flag::keep_array), item);
-    return out;
-  }
-
-private:
-  render_context &m_ctx;
-  const template_type &m_section;
-  const delim_type &m_delims;
-  flag m_flag;
-};
-}

contrib/backends/nntpchan-daemon/libnntpchan/base64.cpp

@@ -16,12 +16,12 @@ const char *GetBase32SubstitutionTable() { return T32; }
 static void iT64Build(void);
 
 /*
-*
-* BASE64 Substitution Table
-* -------------------------
-*
-* Direct Substitution Table
-*/
+ *
+ * BASE64 Substitution Table
+ * -------------------------
+ *
+ * Direct Substitution Table
+ */
 
 static const char T64[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
                              'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
@@ -31,33 +31,33 @@ static const char T64[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', '
 const char *GetBase64SubstitutionTable() { return T64; }
 
 /*
-* Reverse Substitution Table (built in run time)
-*/
+ * Reverse Substitution Table (built in run time)
+ */
 
 static char iT64[256];
 static int isFirstTime = 1;
 
 /*
-* Padding
-*/
+ * Padding
+ */
 
 static char P64 = '=';
 
 /*
-*
-* ByteStreamToBase64
-* ------------------
-*
-* Converts binary encoded data to BASE64 format.
-*
-*/
+ *
+ * ByteStreamToBase64
+ * ------------------
+ *
+ * Converts binary encoded data to BASE64 format.
+ *
+ */
 
-size_t                                          /* Number of bytes in the encoded buffer */
-    ByteStreamToBase64(const uint8_t *InBuffer, /* Input buffer, binary data */
-                       size_t InCount,          /* Number of bytes in the input buffer */
-                       char *OutBuffer,         /* output buffer */
-                       size_t len               /* length of output buffer */
-                       )
+size_t                                      /* Number of bytes in the encoded buffer */
+ByteStreamToBase64(const uint8_t *InBuffer, /* Input buffer, binary data */
+                   size_t InCount,          /* Number of bytes in the input buffer */
+                   char *OutBuffer,         /* output buffer */
+                   size_t len               /* length of output buffer */
+)
 
 {
   unsigned char *ps;
@@ -125,21 +125,21 @@ size_t                                          /* Number of bytes in the encode
 }
 
 /*
-*
-* Base64ToByteStream
-* ------------------
-*
-* Converts BASE64 encoded data to binary format. If input buffer is
-* not properly padded, buffer of negative length is returned
-*
-*/
+ *
+ * Base64ToByteStream
+ * ------------------
+ *
+ * Converts BASE64 encoded data to binary format. If input buffer is
+ * not properly padded, buffer of negative length is returned
+ *
+ */
 
-size_t                                       /* Number of output bytes */
-    Base64ToByteStream(const char *InBuffer, /* BASE64 encoded buffer */
-                       size_t InCount,       /* Number of input bytes */
-                       uint8_t *OutBuffer,   /* output buffer length */
-                       size_t len            /* length of output buffer */
-                       )
+size_t                                   /* Number of output bytes */
+Base64ToByteStream(const char *InBuffer, /* BASE64 encoded buffer */
+                   size_t InCount,       /* Number of input bytes */
+                   uint8_t *OutBuffer,   /* output buffer length */
+                   size_t len            /* length of output buffer */
+)
 {
   unsigned char *ps;
   unsigned char *pd;
@@ -212,13 +212,13 @@ size_t Base32EncodingBufferSize(const size_t input_size)
   return 8 * d.quot;
 }
 /*
-*
-* iT64
-* ----
-* Reverse table builder. P64 character is replaced with 0
-*
-*
-*/
+ *
+ * iT64
+ * ----
+ * Reverse table builder. P64 character is replaced with 0
+ *
+ *
+ */
 
 static void iT64Build()
 {

contrib/backends/nntpchan-daemon/libnntpchan/buffer.cpp

@@ -1,17 +0,0 @@
-#include <cstring>
-#include <nntpchan/buffer.hpp>
-
-namespace nntpchan
-{
-WriteBuffer::WriteBuffer(const char *b, const size_t s)
-{
-  char *buf = new char[s];
-  std::memcpy(buf, b, s);
-  this->b = uv_buf_init(buf, s);
-  w.data = this;
-}
-
-WriteBuffer::WriteBuffer(const std::string &s) : WriteBuffer(s.c_str(), s.size()) {}
-
-WriteBuffer::~WriteBuffer() { delete[] b.base; }
-}

contrib/backends/nntpchan-daemon/libnntpchan/crypto.cpp

@@ -7,16 +7,18 @@ namespace nntpchan
 {
 void SHA512(const uint8_t *d, const std::size_t l, SHA512Digest &h) { crypto_hash(h.data(), d, l); }
 
-  void Blake2B(const uint8_t *d, std::size_t l, Blake2BDigest & h) { crypto_generichash(h.data(), h.size(), d, l, nullptr, 0); }
+void Blake2B(const uint8_t *d, std::size_t l, Blake2BDigest &h)
+{
+  crypto_generichash(h.data(), h.size(), d, l, nullptr, 0);
+}
 
-  std::string Blake2B_base32(const std::string & str)
-  {
-    Blake2BDigest d;
-    Blake2B(reinterpret_cast<const uint8_t*>(str.c_str()), str.size(), d);
-    return B32Encode(d.data(), d.size());
-  }
+std::string Blake2B_base32(const std::string &str)
+{
+  Blake2BDigest d;
+  Blake2B(reinterpret_cast<const uint8_t *>(str.c_str()), str.size(), d);
+  return B32Encode(d.data(), d.size());
+}
 
-  
 Crypto::Crypto() { assert(sodium_init() == 0); }
 
 Crypto::~Crypto() {}

contrib/backends/nntpchan-daemon/libnntpchan/epoll.hpp

@@ -0,0 +1,183 @@
+#include <cassert>
+#include <nntpchan/event.hpp>
+#include <sys/epoll.h>
+#include <unistd.h>
+#include <netinet/in.h>
+#include <sys/un.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <sys/signalfd.h>
+
+#include <iostream>
+
+namespace nntpchan
+{
+namespace ev
+{
+  template<size_t bufsz>
+  struct EpollLoop : public Loop
+  {
+    size_t conns;
+    int epollfd;
+    char readbuf[bufsz];
+    EpollLoop() : conns(0), epollfd(epoll_create1(EPOLL_CLOEXEC))
+    {
+    }
+    
+    virtual ~EpollLoop()
+    {
+        ::close(epollfd);
+    }
+
+    
+
+    virtual bool TrackConn(ev::io * handler)
+    {
+        epoll_event ev;
+        ev.data.ptr = handler;
+        ev.events = EPOLLET;
+        if(handler->readable() || handler->acceptable())
+        {
+            ev.events |= EPOLLIN;
+        }
+        if(handler->writeable())
+        {
+            ev.events |= EPOLLOUT;
+        }
+        if ( epoll_ctl(epollfd, EPOLL_CTL_ADD, handler->fd, &ev) == -1)
+        {
+            return false;
+        }
+        ++conns;
+        return true;
+    }
+
+    virtual void UntrackConn(ev::io * handler)
+    {
+        if(epoll_ctl(epollfd, EPOLL_CTL_DEL, handler->fd, nullptr) != -1)
+            --conns;
+    }
+
+
+    virtual void Run() 
+    {
+        epoll_event evs[512];
+        epoll_event * ev;
+        ev::io * handler;
+        int res = -1;
+        int idx ;
+
+        sigset_t mask;
+        
+        sigemptyset(&mask);
+        sigaddset(&mask, SIGWINCH);
+
+        int sfd = signalfd(-1, &mask, SFD_NONBLOCK | SFD_CLOEXEC);
+        epoll_event sig_ev;
+        sig_ev.data.fd = sfd;
+        sig_ev.events = EPOLLIN;
+        epoll_ctl(epollfd, EPOLL_CTL_ADD, sfd, &sig_ev);
+        do
+        {
+            res = epoll_wait(epollfd, evs, 512, -1);
+            idx = 0;
+            while(idx < res)
+            {
+                errno = 0;
+                ev = &evs[idx++];
+                if(ev->data.fd == sfd)
+                {
+                    read(sfd, readbuf, sizeof(readbuf));
+                    continue;
+                }
+                
+                handler = static_cast<ev::io *>(ev->data.ptr);
+
+                if(ev->events & EPOLLERR || ev->events & EPOLLHUP)
+                {
+                    handler->close();
+                    delete handler;
+                    continue;
+                }
+
+                if (handler->acceptable())
+                {
+                    int acceptfd;
+                    bool errored = false;
+                    while(true)
+                    {
+                    acceptfd = handler->accept();
+                    if(acceptfd == -1)
+                    {
+                        if (errno == EAGAIN || errno == EWOULDBLOCK)
+                        {
+                        break;
+                        }
+                        perror("accept()");
+                        errored = true;
+                        break;
+                    }
+                    }
+                    if(errored)
+                    {
+                    handler->close();
+                    delete handler;
+                    continue;
+                    }
+                }
+                if(ev->events & EPOLLIN && handler->readable())
+                {
+                    bool errored = false;
+                    while(true)
+                    {
+                    int readed = handler->read(readbuf, sizeof(readbuf));
+                    if(readed == -1)
+                    {
+                        if(errno != EAGAIN)
+                        {
+                        perror("read()");
+                        handler->close();
+                        delete handler;
+                        errored = true;
+                        }
+                        break;
+                    }
+                    else if (readed == 0)
+                    {
+                        handler->close();
+                        delete handler;
+                        errored = true;
+                        break;
+                    }
+                    }
+                    if(errored) continue;
+                }
+                if(ev->events & EPOLLOUT && handler->writeable())
+                {
+                    int written = handler->write(1024);
+                    if(written < 0)
+                    {
+                        if (errno == EAGAIN || errno == EWOULDBLOCK)
+                        {
+                            // blocking
+                        }
+                        else
+                        {
+                            perror("write()");
+                            handler->close();
+                            delete handler;
+                        }
+                    }
+                }
+                if (!handler->keepalive())
+                {
+                    handler->close();
+                    delete handler;
+                }
+            }
+        }
+        while(res != -1 && conns);
+    }
+};
+}
+}

contrib/backends/nntpchan-daemon/libnntpchan/event.cpp

@@ -1,17 +1,78 @@
+#include <fcntl.h>
+#include <cstdlib>
 #include <cassert>
-#include <nntpchan/event.hpp>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+
+constexpr std::size_t ev_buffsz = 512;
+
+#ifdef __linux__
+#include "epoll.hpp"
+typedef nntpchan::ev::EpollLoop<ev_buffsz> LoopImpl;
+#else
+#ifdef __FreeBSD__
+#include "kqueue.hpp"
+typedef nntpchan::ev::KqueueLoop<ev_buffsz> LoopImpl;
+#else
+#ifdef __netbsd__
+typedef nntpchan::ev::KqueueLoop<ev_buffsz> LoopImpl;
+#else
+#error "unsupported platform"
+#endif
+#endif
+#endif
 
 namespace nntpchan
 {
-Mainloop::Mainloop()
+namespace ev
 {
-  m_loop = uv_default_loop();
-  assert(uv_loop_init(m_loop) == 0);
+bool ev::Loop::BindTCP(const sockaddr *addr, ev::io *handler)
+{
+  assert(handler->acceptable());
+  socklen_t slen;
+  switch (addr->sa_family)
+  {
+  case AF_INET:
+    slen = sizeof(sockaddr_in);
+    break;
+  case AF_INET6:
+    slen = sizeof(sockaddr_in6);
+    break;
+  case AF_UNIX:
+    slen = sizeof(sockaddr_un);
+    break;
+  default:
+    return false;
+  }
+  int fd = socket(addr->sa_family, SOCK_STREAM | SOCK_NONBLOCK, 0);
+  if (fd == -1)
+  {
+    return false;
+  }
+
+  if (bind(fd, addr, slen) == -1)
+  {
+    ::close(fd);
+    return false;
+  }
+
+  if (listen(fd, 5) == -1)
+  {
+    ::close(fd);
+    return false;
+  }
+
+  handler->fd = fd;
+  return TrackConn(handler);
 }
 
-Mainloop::~Mainloop() { uv_loop_close(m_loop); }
-
-void Mainloop::Stop() { uv_stop(m_loop); }
-
-void Mainloop::Run(uv_run_mode mode) { assert(uv_run(m_loop, mode) == 0); }
+bool Loop::SetNonBlocking(ev::io *handler)
+{
+  return fcntl(handler->fd, F_SETFL, fcntl(handler->fd, F_GETFL, 0) | O_NONBLOCK) != -1;
 }
+}
+
+ev::Loop *NewMainLoop() { return new LoopImpl; }
+}

contrib/backends/nntpchan-daemon/libnntpchan/exec_frontend.cpp

@@ -7,7 +7,7 @@
 
 namespace nntpchan
 {
-ExecFrontend::ExecFrontend(const std::string &fname) : m_exec(fname) {}
+ExecFrontend::ExecFrontend(const std::string &fname, char * const* env) : m_Environ(env), m_exec(fname) {}
 
 ExecFrontend::~ExecFrontend() {}
 
@@ -37,7 +37,7 @@ int ExecFrontend::Exec(std::deque<std::string> args)
   }
   else
   {
-    int r = execvpe(m_exec.c_str(), (char *const *)cargs, environ);
+    int r = execve(m_exec.c_str(), (char *const *)cargs, m_Environ);
     if (r == -1)
     {
       std::cout << strerror(errno) << std::endl;

contrib/backends/nntpchan-daemon/libnntpchan/kqueue.hpp

@@ -0,0 +1,162 @@
+#include <nntpchan/event.hpp>
+#include <sys/types.h>
+#include <sys/event.h>
+
+#include <iostream>
+#include <cstring>
+#include <errno.h>
+
+namespace nntpchan 
+{
+namespace ev 
+{
+    template<size_t bufsz>
+    struct KqueueLoop : public Loop
+    {
+        int kfd;
+        size_t conns;
+        char readbuf[bufsz];
+
+
+        KqueueLoop() : kfd(kqueue()), conns(0)
+        {
+
+        };
+
+        virtual ~KqueueLoop()
+        {
+            ::close(kfd);
+        }
+
+        virtual bool TrackConn(ev::io * handler)
+        {
+            struct kevent event;
+            short filter = 0;
+            if(handler->readable() || handler->acceptable())
+            {   
+                filter |= EVFILT_READ;
+            }
+            if(handler->writeable())
+            {
+                filter |= EVFILT_WRITE;
+            }
+            EV_SET(&event, handler->fd, filter, EV_ADD | EV_CLEAR, 0, 0, handler);
+            int ret = kevent(kfd, &event, 1, nullptr, 0, nullptr);
+            if(ret == -1) return false;
+            if(event.flags & EV_ERROR) 
+            {
+                std::cerr << "KqueueLoop::TrackConn() kevent failed: " << strerror(event.data) << std::endl;
+                return false;
+            }
+            ++conns;
+            return true;
+        }
+
+        virtual void UntrackConn(ev::io * handler)
+        {
+            struct kevent event;
+            short filter = 0;
+            if(handler->readable() || handler->acceptable())
+            {   
+                filter |= EVFILT_READ;
+            }
+            if(handler->writeable())
+            {
+                filter |= EVFILT_WRITE;
+            }
+            EV_SET(&event, handler->fd, filter, EV_DELETE, 0, 0, handler);
+            int ret = kevent(kfd, &event, 1, nullptr, 0, nullptr);
+            if(ret == -1 || event.flags & EV_ERROR) 
+                std::cerr << "KqueueLoop::UntrackConn() kevent failed: " << strerror(event.data) << std::endl;
+            else
+                --conns;
+        }
+
+        virtual void Run()
+        {
+            struct kevent events[512];
+            struct kevent * event;
+            io * handler;
+            int ret, idx;
+            do
+            {
+                idx = 0;
+                ret = kevent(kfd, nullptr, 0, events, 512, nullptr);
+                if(ret > 0)
+                {
+                    while(idx < ret)
+                    {
+                        event = &events[idx++];
+                        handler = static_cast<io *>(event->udata);
+                        if(event->flags & EV_EOF)
+                        {
+                            handler->close();
+                            delete handler;
+                            continue;
+                        }
+                        if(event->filter & EVFILT_READ && handler->acceptable())
+                        {
+                            int backlog = event->data;
+                            while(backlog)
+                            {
+                                handler->accept();
+                                --backlog;
+                            }
+                        }
+
+                        if(event->filter & EVFILT_READ && handler->readable())
+                        {
+                            int readed = 0;
+                            size_t readnum = event->data;
+                            while(readnum > sizeof(readbuf))
+                            {
+                                int r = handler->read(readbuf, sizeof(readbuf));
+                                if(r > 0)
+                                {
+                                    readnum -= r;
+                                    readed += r;
+                                }
+                                else
+                                    readnum = 0;
+                            }
+                            if(readnum && readed != -1)
+                            {
+                               int r = handler->read(readbuf, readnum);
+                               if(r > 0)
+                                 readed += r;
+                               else
+                                 readed = r;
+                            }
+                        }
+                        if(event->filter & EVFILT_WRITE && handler->writeable())
+                        {
+                            int writespace = 1024;
+                            int written = handler->write(writespace);
+                            if(written == -1)
+                            {
+                                if (errno == EAGAIN || errno == EWOULDBLOCK)
+                                {
+                                    // blocking
+                                }
+                                else 
+                                {
+                                    perror("write()");
+                                    handler->close();
+                                    delete handler;
+                                    continue;
+                                }
+                            }
+                        }
+                        if(!handler->keepalive())
+                        {
+                            handler->close();
+                            delete handler;
+                        }
+                    }
+                }
+            }
+            while(ret != -1);
+        }
+    };
+}
+}

contrib/backends/nntpchan-daemon/libnntpchan/line.cpp

@@ -3,43 +3,22 @@
 namespace nntpchan
 {
 
-LineReader::LineReader(size_t limit) : m_close(false), lineLimit(limit) {}
-
 void LineReader::Data(const char *data, ssize_t l)
 {
   if (l <= 0)
     return;
-  // process leftovers
-  std::size_t idx = 0;
-  std::size_t pos = 0;
-  while (l-- > 0)
+  m_line << m_leftover;
+  m_leftover = "";
+  m_line << std::string(data, l);
+
+  for (std::string line; std::getline(m_line, line);)
   {
-    char c = data[idx++];
-    if (c == '\n')
-    {
-      OnLine(data, pos);
-      pos = 0;
-      data += idx;
-    }
-    else if (c == '\r' && data[idx] == '\n')
-    {
-      OnLine(data, pos);
-      data += idx + 1;
-      pos = 0;
-    }
-    else
-    {
-      pos++;
-    }
+    line.erase(std::remove(line.begin(), line.end(), '\r'), line.end());
+    HandleLine(line);
   }
+  if (m_line)
+    m_leftover = m_line.str();
+  m_line.clear();
 }
 
-void LineReader::OnLine(const char *d, const size_t l)
-{
-  std::string line;
-  line += std::string(d, l);
-  HandleLine(line);
-}
-
-bool LineReader::ShouldClose() { return m_close; }
 }

contrib/backends/nntpchan-daemon/libnntpchan/net.cpp

@@ -2,7 +2,8 @@
 #include <nntpchan/net.hpp>
 #include <sstream>
 #include <stdexcept>
-#include <uv.h>
+#include <arpa/inet.h>
+
 
 namespace nntpchan
 {
@@ -11,7 +12,7 @@ std::string NetAddr::to_string()
   std::string str("invalid");
   const size_t s = 128;
   char *buff = new char[s];
-  if (uv_ip6_name(&addr, buff, s) == 0)
+  if (inet_ntop(AF_INET6, &addr, buff, sizeof(sockaddr_in6)))
   {
     str = std::string(buff);
     delete[] buff;
@@ -38,7 +39,9 @@ NetAddr ParseAddr(const std::string &addr)
   auto p = addr.substr(n + 2);
   int port = std::atoi(p.c_str());
   auto a = addr.substr(0, n);
-  uv_ip6_addr(a.c_str(), port, &saddr.addr);
+  saddr.addr.sin6_port = htons(port);
+  saddr.addr.sin6_family = AF_INET6;
+  inet_pton(AF_INET6, a.c_str(), &saddr.addr);
   return saddr;
 }
 }

contrib/backends/nntpchan-daemon/libnntpchan/nntp_auth.cpp

@@ -7,7 +7,7 @@
 
 namespace nntpchan
 {
-HashedCredDB::HashedCredDB() : LineReader(1024) {}
+HashedCredDB::HashedCredDB() : LineReader() {}
 
 bool HashedCredDB::CheckLogin(const std::string &user, const std::string &passwd)
 {
@@ -55,7 +55,7 @@ bool HashedCredDB::ProcessLine(const std::string &line)
   return Hash(m_passwd, salt) == cred;
 }
 
-void HashedCredDB::HandleLine(const std::string &line)
+void HashedCredDB::HandleLine(const std::string line)
 {
   if (m_found)
     return;

contrib/backends/nntpchan-daemon/libnntpchan/nntp_handler.cpp

@@ -9,15 +9,15 @@
 
 namespace nntpchan
 {
-NNTPServerHandler::NNTPServerHandler(const fs::path &storage)
-    : LineReader(1024), m_article(nullptr), m_auth(nullptr), m_store(std::make_unique<ArticleStorage>(storage)),
-      m_authed(false), m_state(eStateReadCommand)
+NNTPServerHandler::NNTPServerHandler(fs::path storage)
+    : LineReader(), m_article(nullptr), m_auth(nullptr), m_store(storage), m_authed(false),
+      m_state(eStateReadCommand)
 {
 }
 
 NNTPServerHandler::~NNTPServerHandler() {}
 
-void NNTPServerHandler::HandleLine(const std::string &line)
+void NNTPServerHandler::HandleLine(const std::string line)
 {
   if (m_state == eStateReadCommand)
   {
@@ -51,6 +51,12 @@ void NNTPServerHandler::OnData(const char *data, ssize_t l)
     return;
   if (m_state == eStateStoreArticle)
   {
+    std::cerr << "storing " << l << " bytes" << std::endl;
+    if (strncmp(data, ".\r\n", l) == 0)
+    {
+      ArticleObtained();
+      return;
+    }
     const char *end = strstr(data, "\r\n.\r\n");
     if (end)
     {
@@ -62,7 +68,8 @@ void NNTPServerHandler::OnData(const char *data, ssize_t l)
       }
       ArticleObtained();
       diff += 5;
-      Data(end + 5, l - diff);
+      if (l - diff)
+        Data(end + 5, l - diff);
       return;
     }
     if (m_article)
@@ -124,7 +131,7 @@ void NNTPServerHandler::HandleCommand(const std::deque<std::string> &command)
     if (cmdlen >= 2)
     {
       const std::string &msgid = command[1];
-      if (IsValidMessageID(msgid) && m_store->Accept(msgid))
+      if (IsValidMessageID(msgid) && m_store.Accept(msgid))
       {
         QueueLine("238 " + msgid);
       }
@@ -139,9 +146,9 @@ void NNTPServerHandler::HandleCommand(const std::deque<std::string> &command)
     if (cmdlen >= 2)
     {
       const std::string &msgid = command[1];
-      if (m_store->Accept(msgid))
+      if (m_store.Accept(msgid))
       {
-        m_article = m_store->OpenWrite(msgid);
+        m_article = m_store.OpenWrite(msgid);
       }
       m_articleName = msgid;
       EnterState(eStateStoreArticle);
@@ -162,6 +169,7 @@ void NNTPServerHandler::ArticleObtained()
   {
     m_article->close();
     m_article = nullptr;
+    m_store.EnsureSymlinks(m_articleName);
     QueueLine("239 " + m_articleName);
     std::cerr << "stored " << m_articleName << std::endl;
   }

contrib/backends/nntpchan-daemon/libnntpchan/nntp_server.cpp

@@ -1,5 +1,6 @@
 
 #include <cassert>
+#include <cstring>
 #include <iostream>
 #include <nntpchan/net.hpp>
 #include <nntpchan/nntp_auth.hpp>
@@ -10,11 +11,11 @@
 namespace nntpchan
 {
 
-NNTPServer::NNTPServer(uv_loop_t *loop) : Server(loop), m_frontend(nullptr) {}
+NNTPServer::NNTPServer(ev::Loop *loop) : Server(loop), m_frontend(nullptr) {}
 
 NNTPServer::~NNTPServer() {}
 
-IServerConn *NNTPServer::CreateConn(uv_stream_t *s)
+IServerConn *NNTPServer::CreateConn(int f)
 {
   CredDB_ptr creds;
 
@@ -27,8 +28,7 @@ IServerConn *NNTPServer::CreateConn(uv_stream_t *s)
   if (creds)
     handler->SetAuth(creds);
 
-  NNTPServerConn *conn = new NNTPServerConn(GetLoop(), s, this, handler);
-  return conn;
+  return new NNTPServerConn(f, this, handler);
 }
 
 void NNTPServer::SetLoginDB(const std::string path) { m_logindbpath = path; }
@@ -41,22 +41,11 @@ void NNTPServer::SetFrontend(Frontend *f) { m_frontend.reset(f); }
 
 std::string NNTPServer::InstanceName() const { return m_servername; }
 
-void NNTPServer::OnAcceptError(int status) { std::cerr << "nntpserver::accept() " << uv_strerror(status) << std::endl; }
-
-void NNTPServerConn::SendNextReply()
-{
-  IConnHandler *handler = GetHandler();
-  while (handler->HasNextLine())
-  {
-    auto line = handler->GetNextLine();
-    SendString(line + "\r\n");
-  }
-}
+void NNTPServer::OnAcceptError(int status) { std::cerr << "nntpserver::accept() " << strerror(status) << std::endl; }
 
 void NNTPServerConn::Greet()
 {
   IConnHandler *handler = GetHandler();
   handler->Greet();
-  SendNextReply();
 }
 }

contrib/backends/nntpchan-daemon/libnntpchan/server.cpp

@@ -1,51 +1,57 @@
 #include <cassert>
 #include <iostream>
-#include <nntpchan/buffer.hpp>
 #include <nntpchan/net.hpp>
 #include <nntpchan/server.hpp>
 
 namespace nntpchan
 {
-Server::Server(uv_loop_t *loop)
-{
-  m_loop = loop;
-  uv_tcp_init(m_loop, &m_server);
-  m_server.data = this;
-}
 
-void Server::Close()
-{
-  std::cout << "Close server" << std::endl;
-  uv_close((uv_handle_t *)&m_server, [](uv_handle_t *s) {
-    Server *self = (Server *)s->data;
-    if (self)
-      delete self;
-    s->data = nullptr;
-  });
-}
+Server::Server(ev::Loop *loop) : ev::io(-1), m_Loop(loop) {}
 
-void Server::Bind(const std::string &addr)
+void Server::close()
+{
+  auto itr = m_conns.begin();
+  while (itr != m_conns.end())
+  {
+    itr = m_conns.erase(itr);
+  }
+  m_Loop->UntrackConn(this);
+  ev::io::close();
+}
+bool Server::Bind(const std::string &addr)
 {
   auto saddr = ParseAddr(addr);
-  assert(uv_tcp_bind(*this, saddr, 0) == 0);
-  auto cb = [](uv_stream_t *s, int status) {
-    Server *self = (Server *)s->data;
-    self->OnAccept(s, status);
-  };
-  assert(uv_listen(*this, 5, cb) == 0);
+  return m_Loop->BindTCP(saddr, this);
 }
 
-void Server::OnAccept(uv_stream_t *s, int status)
+void Server::OnAccept(int f)
 {
-  if (status < 0)
+  IServerConn *conn = CreateConn(f);
+  if (!m_Loop->SetNonBlocking(conn))
   {
-    OnAcceptError(status);
-    return;
+    conn->close();
+    delete conn;
   }
-  IServerConn *conn = CreateConn(s);
-  assert(conn);
-  m_conns.push_back(conn);
-  conn->Greet();
+  else if (m_Loop->TrackConn(conn))
+  {
+    m_conns.push_back(conn);
+    conn->Greet();
+    conn->write(1024);
+  }
+  else
+  {
+    conn->close();
+    delete conn;
+  }
+}
+
+int Server::accept()
+{
+  int res = ::accept(fd, nullptr, nullptr);
+  if (res == -1)
+    return res;
+  OnAccept(res);
+  return res;
 }
 
 void Server::RemoveConn(IServerConn *conn)
@@ -58,9 +64,10 @@ void Server::RemoveConn(IServerConn *conn)
     else
       ++itr;
   }
+  m_Loop->UntrackConn(conn);
 }
 
-void IConnHandler::QueueLine(const std::string &line) { m_sendlines.push_back(line); }
+void IConnHandler::QueueLine(const std::string &line) { m_sendlines.push_back(line + "\r\n"); }
 
 bool IConnHandler::HasNextLine() { return m_sendlines.size() > 0; }
 
@@ -71,72 +78,79 @@ std::string IConnHandler::GetNextLine()
   return line;
 }
 
-IServerConn::IServerConn(uv_loop_t *l, uv_stream_t *st, Server *parent, IConnHandler *h)
-{
-  m_loop = l;
-  m_parent = parent;
-  m_handler = h;
-  uv_tcp_init(l, &m_conn);
-  m_conn.data = this;
-  uv_accept(st, (uv_stream_t *)&m_conn);
-  uv_read_start((uv_stream_t *)&m_conn,
-                [](uv_handle_t *h, size_t s, uv_buf_t *b) {
-                  IServerConn *self = (IServerConn *)h->data;
-                  if (self == nullptr)
-                    return;
-                  b->base = new char[s];
-                },
-                [](uv_stream_t *s, ssize_t nread, const uv_buf_t *b) {
-                  IServerConn *self = (IServerConn *)s->data;
-                  if (self == nullptr)
-                  {
-                    if (b->base)
-                      delete[] b->base;
-                    return;
-                  }
-                  if (nread > 0)
-                  {
-                    self->m_handler->OnData(b->base, nread);
-                    self->SendNextReply();
-                    if (self->m_handler->ShouldClose())
-                      self->Close();
-                    delete[] b->base;
-                  }
-                  else
-                  {
-                    if (nread != UV_EOF)
-                    {
-                      std::cerr << "error in nntp server conn alloc: ";
-                      std::cerr << uv_strerror(nread);
-                      std::cerr << std::endl;
-                    }
-                    // got eof or error
-                    self->Close();
-                  }
-                });
-}
+IServerConn::IServerConn(int fd, Server *parent, IConnHandler *h) : ev::io(fd), m_parent(parent), m_handler(h) {}
 
 IServerConn::~IServerConn() { delete m_handler; }
 
-void IServerConn::SendString(const std::string &str)
+int IServerConn::read(char *buf, size_t sz)
 {
-  WriteBuffer *b = new WriteBuffer(str);
-  uv_write(&b->w, (uv_stream_t *)&m_conn, &b->b, 1, [](uv_write_t *w, int status) {
-    (void)status;
-    WriteBuffer *wb = (WriteBuffer *)w->data;
-    if (wb)
-      delete wb;
-  });
+  ssize_t readsz = ::read(fd, buf, sz);
+  if (readsz > 0)
+  {
+    m_handler->OnData(buf, readsz);
+  }
+  return readsz;
 }
 
-void IServerConn::Close()
+bool IServerConn::keepalive() { return !m_handler->ShouldClose(); }
+
+int IServerConn::write(size_t avail)
+{
+  auto leftovers = m_writeLeftover.size();
+  int written = 0;
+  if (leftovers)
+  {
+    if (leftovers > avail)
+    {
+      leftovers = avail;
+    }
+    written = ::write(fd, m_writeLeftover.c_str(), leftovers);
+    if (written > 0)
+    {
+      avail -= written;
+      m_writeLeftover = m_writeLeftover.substr(written);
+    }
+    else
+    {
+      // too much leftovers
+      return -1;
+    }
+  }
+  do
+  {
+    if (!m_handler->HasNextLine())
+    {
+      return written;
+    }
+    auto line = m_handler->GetNextLine();
+    int wrote;
+    if (line.size() <= avail)
+    {
+      wrote = ::write(fd, line.c_str(), line.size());
+    }
+    else
+    {
+      auto subline = line.substr(0, avail);
+      wrote = ::write(fd, subline.c_str(), subline.size());
+    }
+    if (wrote > 0)
+    {
+      written += wrote;
+      avail -= wrote;
+      m_writeLeftover = line.substr(wrote);
+    }
+    else
+    {
+      m_writeLeftover = line;
+      return -1;
+    }
+  } while (avail > 0);
+  return written;
+}
+
+void IServerConn::close()
 {
   m_parent->RemoveConn(this);
-  uv_close((uv_handle_t *)&m_conn, [](uv_handle_t *s) {
-    IServerConn *self = (IServerConn *)s->data;
-    if (self)
-      delete self;
-    s->data = nullptr;
-  });
+  ev::io::close();
 }
 }

contrib/backends/nntpchan-daemon/libnntpchan/staticfile_frontend.cpp

@@ -17,6 +17,8 @@ StaticFileFrontend::StaticFileFrontend(TemplateEngine *tmpl, const std::string &
 {
 }
 
+StaticFileFrontend::~StaticFileFrontend() {}
+
 void StaticFileFrontend::ProcessNewMessage(const fs::path &fpath)
 {
   std::clog << "process message " << fpath << std::endl;
@@ -118,12 +120,10 @@ void StaticFileFrontend::ProcessNewMessage(const fs::path &fpath)
       std::clog << "cannot find thread with root " << rootmsgid << std::endl;
       return;
     }
-    TemplateEngine::Args_t thread_args;
-    thread_args["posts"] = thread;
     if (m_TemplateEngine)
     {
       FileHandle_ptr out = OpenFile(threadFilePath, eWrite);
-      if (!out || !m_TemplateEngine->WriteTemplate("thread.mustache", thread_args, out))
+      if (!out || !m_TemplateEngine->WriteThreadPage(thread, out))
       {
         std::clog << "failed to write " << threadFilePath << std::endl;
         return;
@@ -135,27 +135,19 @@ void StaticFileFrontend::ProcessNewMessage(const fs::path &fpath)
       uint32_t pageno = 0;
       while (pageno < m_Pages)
       {
-        page.clear();
+        page.threads.clear();
         if (!m_MessageDB->LoadBoardPage(page, name, 10, m_Pages))
         {
           std::clog << "cannot load board page " << pageno << " for " << name << std::endl;
           break;
         }
-        TemplateEngine::Args_t page_args;
-        page_args["group"] = name;
-        page_args["threads"] = page;
-        page_args["pageno"] = std::to_string(pageno);
-        if (pageno)
-          page_args["prev_pageno"] = std::to_string(pageno - 1);
-        if (pageno + 1 < m_Pages)
-          page_args["next_pageno"] = std::to_string(pageno + 1);
         fs::path boardPageFilename(name + "-" + std::to_string(pageno) + ".html");
         if (m_TemplateEngine)
         {
           fs::path outfile = m_OutDir / boardPageFilename;
           FileHandle_ptr out = OpenFile(outfile, eWrite);
           if (out)
-            m_TemplateEngine->WriteTemplate("board.mustache", page_args, out);
+            m_TemplateEngine->WriteBoardPage(page, out);
           else
             std::clog << "failed to open board page " << outfile << std::endl;
         }

contrib/backends/nntpchan-daemon/libnntpchan/storage.cpp

@@ -7,9 +7,9 @@
 namespace nntpchan
 {
 
-  const fs::path posts_skiplist_dir = "posts";
-  const fs::path threads_skiplist_dir = "threads";
-  
+const fs::path posts_skiplist_dir = "posts";
+const fs::path threads_skiplist_dir = "threads";
+
 ArticleStorage::ArticleStorage(const fs::path &fpath) { SetPath(fpath); }
 
 ArticleStorage::~ArticleStorage() {}
@@ -20,17 +20,16 @@ void ArticleStorage::SetPath(const fs::path &fpath)
   fs::create_directories(basedir);
   assert(init_skiplist(posts_skiplist_dir));
   assert(init_skiplist(threads_skiplist_dir));
+  errno = 0;
 }
 
-
 bool ArticleStorage::init_skiplist(const std::string &subdir) const
 {
   fs::path skiplist = skiplist_root(subdir);
-  const auto subdirs = {		'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h',
-		'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
-		'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
-		'y', 'z', '2', '3', '4', '5', '6', '7',
-};
+  const auto subdirs = {
+      'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
+      'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '2', '3', '4', '5', '6', '7',
+  };
   for (const auto &s : subdirs)
     fs::create_directories(skiplist / std::string(&s, 1));
   return true;
@@ -41,7 +40,9 @@ bool ArticleStorage::Accept(const std::string &msgid) const
   if (!IsValidMessageID(msgid))
     return false;
   auto p = MessagePath(msgid);
-  return !fs::exists(p);
+  bool ret = !fs::exists(p);
+  errno = 0;
+  return ret;
 }
 
 fs::path ArticleStorage::MessagePath(const std::string &msgid) const { return basedir / msgid; }
@@ -79,16 +80,15 @@ bool ArticleStorage::LoadThread(Thread &thread, const std::string &rootmsgid) co
 void ArticleStorage::EnsureSymlinks(const std::string &msgid) const
 {
   std::string msgidhash = Blake2B_base32(msgid);
-  skiplist_dir(posts_skiplist_dir, msgidhash);
+  auto skip = skiplist_dir(skiplist_root(posts_skiplist_dir), msgidhash) / msgidhash;
+  auto path = fs::path("..") / fs::path("..") / fs::path("..") / MessagePath(msgid);
+  fs::create_symlink(path, skip);
+  errno = 0;
 }
 
-
-  fs::path ArticleStorage::skiplist_root(const std::string & name ) const
-  {
-    return basedir / name;
-  }
-  fs::path ArticleStorage::skiplist_dir(const fs::path & root, const std::string & name ) const
-  {
-    return root / name.substr(0, 1) ;
-  }
+fs::path ArticleStorage::skiplist_root(const std::string &name) const { return basedir / name; }
+fs::path ArticleStorage::skiplist_dir(const fs::path &root, const std::string &name) const
+{
+  return root / name.substr(0, 1);
+}
 }

contrib/backends/nntpchan-daemon/libnntpchan/template_engine.cpp

@@ -1,165 +1,92 @@
 #include <iostream>
-#include <mstch/mstch.hpp>
 #include <nntpchan/sanitize.hpp>
 #include <nntpchan/template_engine.hpp>
 #include <sstream>
 
 namespace nntpchan
 {
-
-template <class... Ts> struct overloaded : Ts...
-{
-  using Ts::operator()...;
-};
-template <class... Ts> overloaded(Ts...)->overloaded<Ts...>;
-
-namespace mustache = mstch;
-
-static mustache::map post_to_map(const nntpchan::model::Post &post)
-{
-  mustache::map m;
-  mustache::array attachments;
-  mustache::map h;
-
-  for (const auto &att : nntpchan::model::GetAttachments(post))
+  struct StdTemplateEngine : public TemplateEngine
   {
-    mustache::map a;
-    a["filename"] = nntpchan::model::GetFilename(att);
-    a["hexdigest"] = nntpchan::model::GetHexDigest(att);
-    a["thumbnail"] = nntpchan::model::GetThumbnail(att);
-    attachments.push_back(a);
-  }
-
-  for (const auto &item : nntpchan::model::GetHeader(post))
-  {
-    mustache::array vals;
-    for (const auto &v : item.second)
-      vals.push_back(v);
-    h[item.first] = vals;
-  }
-
-  m["attachments"] = attachments;
-  m["message"] = nntpchan::model::GetBody(post);
-  m["header"] = h;
-  return m;
-}
-
-static mustache::map thread_to_map(const nntpchan::model::Thread &t)
-{
-  mustache::map thread;
-  mustache::array posts;
-  for (const auto &post : t)
-  {
-    posts.push_back(post_to_map(post));
-  }
-  auto &opHeader = nntpchan::model::GetHeader(t[0]);
-  thread["title"] = nntpchan::model::HeaderIFind(opHeader, "subject", "None")[0];
-  thread["posts"] = posts;
-  return thread;
-}
-
-struct MustacheTemplateEngine : public TemplateEngine
-{
-  struct Impl
-  {
-
-    Impl(const std::map<std::string, std::string> &partials) : m_partials(partials) {}
-
-    bool ParseTemplate(const FileHandle_ptr &in)
+    struct RenderContext 
     {
-      std::stringstream str;
-      std::string line;
-      while (std::getline(*in, line))
-        str << line << "\n";
-      m_tmplString = str.str();
-      return in->eof();
-    }
 
-    bool RenderFile(const Args_t &args, const FileHandle_ptr &out)
-    {
-      mustache::map obj;
-      for (const auto &item : args)
+      bool Load(const fs::path & path)
       {
-        std::visit(overloaded{[&obj, item](const nntpchan::model::Model &m) {
-                                std::visit(overloaded{[&obj, item](const nntpchan::model::BoardPage &p) {
-                                                        mustache::array threads;
-                                                        for (const auto &thread : p)
-                                                        {
-                                                          threads.push_back(thread_to_map(thread));
-                                                        }
-                                                        obj[item.first] = threads;
-                                                      },
-                                                      [&obj, item](const nntpchan::model::Thread &t) {
-                                                        obj[item.first] = thread_to_map(t);
-                                                      }},
-                                           m);
-                              },
-                              [&obj, item](const std::string &str) { obj[item.first] = str; }},
-                   item.second);
+        // clear out previous data
+        m_Data.clear();
+        // open file
+        std::ifstream f;
+        f.open(path);
+        if(f.is_open())
+        {
+          for(std::string line; std::getline(f, line, '\n');)
+          {
+            m_Data += line + "\n";
+          }
+          return true;
+        }
+        else 
+          return false;
       }
 
-      std::string str = mustache::render(m_tmplString, obj);
-      out->write(str.c_str(), str.size());
-      out->flush();
-      return !out->fail();
-    }
+      virtual bool Render(const FileHandle_ptr & out) const = 0;
 
-    std::string m_tmplString;
-    const std::map<std::string, std::string> &m_partials;
-  };
+      std::string m_Data;
+    };
 
-  virtual bool WriteTemplate(const fs::path &fpath, const Args_t &args, const FileHandle_ptr &out)
-  {
-    auto templFile = OpenFile(fpath, eRead);
-    if (!templFile)
+    struct BoardRenderContext : public RenderContext
     {
-      std::clog << "no such template at " << fpath << std::endl;
-      return false;
-    }
+      const nntpchan::model::BoardPage & m_Page;
 
-    std::map<std::string, std::string> partials;
-    if (!LoadPartials(fpath.parent_path(), partials))
-    {
-      std::clog << "failed to load partials" << std::endl;
-      return false;
-    }
+      BoardRenderContext(const nntpchan::model::BoardPage & page) : m_Page(page) {};
 
-    Impl impl(partials);
-    if (impl.ParseTemplate(templFile))
-    {
-      return impl.RenderFile(args, out);
-    }
-
-    std::clog << "failed to parse template " << fpath << std::endl;
-    return false;
-  }
-
-  bool LoadPartials(fs::path dir, std::map<std::string, std::string> &partials)
-  {
-    const auto partial_files = {"header", "footer"};
-    for (const auto &fname : partial_files)
-    {
-      auto file = OpenFile(dir / fs::path(fname + std::string(".html")), eRead);
-      if (!file)
+      virtual bool Render(const FileHandle_ptr & out) const
       {
-        std::clog << "no such partial: " << fname << std::endl;
+        *out << m_Data;
         return false;
       }
-      std::string line;
-      std::stringstream input;
-      while (std::getline(*file, line))
-        input << line << "\n";
-      partials[fname] = input.str();
+    };
+
+    struct ThreadRenderContext : public RenderContext
+    {
+      const nntpchan::model::Thread & m_Thread;
+
+      ThreadRenderContext(const nntpchan::model::Thread & thread) : m_Thread(thread) {};
+
+      virtual bool Render(const FileHandle_ptr & out) const 
+      {
+        *out << m_Data;
+        return false;
+      }
+
+    };
+
+    bool WriteBoardPage(const nntpchan::model::BoardPage & page, const FileHandle_ptr & out)
+    {
+      BoardRenderContext ctx(page);
+      if(ctx.Load("board.html"))
+      {
+        return ctx.Render(out);
+      }
+      return false;
     }
-    return true;
-  }
-};
+
+    bool WriteThreadPage(const nntpchan::model::Thread & thread, const FileHandle_ptr & out)
+    {
+      ThreadRenderContext ctx(thread);
+      if(ctx.Load("thread.html"))
+      { 
+        return ctx.Render(out);
+      }
+      return false;
+    }
+  };
 
 TemplateEngine *CreateTemplateEngine(const std::string &dialect)
 {
   auto d = ToLower(dialect);
-  if (d == "mustache")
-    return new MustacheTemplateEngine;
+  if (d == "std")
+    return new StdTemplateEngine;
   else
     return nullptr;
 }

contrib/backends/nntpchan-daemon/test.cpp

@@ -3,9 +3,9 @@
 #include <nntpchan/exec_frontend.hpp>
 #include <nntpchan/sanitize.hpp>
 
-int main(int, char *[])
+int main(int, char *[], char * argenv[])
 {
-  nntpchan::Frontend_ptr f(new nntpchan::ExecFrontend("./contrib/nntpchan.sh"));
+  nntpchan::Frontend_ptr f(new nntpchan::ExecFrontend("./contrib/nntpchan.sh", argenv));
   assert(f->AcceptsMessage("<test@server>"));
   assert(f->AcceptsNewsgroup("overchan.test"));
   assert(nntpchan::IsValidMessageID("<test@test>"));

contrib/backends/nntpchan-daemon/tools/authtool.cpp

@@ -1,5 +1,5 @@
-#include "base64.hpp"
-#include "crypto.hpp"
+#include <nntpchan/base64.hpp>
+#include <nntpchan/crypto.hpp>
 
 #include <cassert>
 #include <cstring>
@@ -12,8 +12,6 @@ static void print_help(const std::string &exename)
   std::cout << "usage: " << exename << " [help|gen|check]" << std::endl;
 }
 
-static void print_long_help() {}
-
 static void gen_passwd(const std::string &username, const std::string &passwd)
 {
   std::array<uint8_t, 8> random;
@@ -52,12 +50,12 @@ int main(int argc, char *argv[])
   if (argc == 1)
   {
     print_help(argv[0]);
-    return 1;
+    return 0;
   }
   std::string cmd(argv[1]);
   if (cmd == "help")
   {
-    print_long_help();
+    print_help(argv[0]);
     return 0;
   }
   if (cmd == "gen")
@@ -70,7 +68,7 @@ int main(int argc, char *argv[])
     else
     {
       std::cout << "usage: " << argv[0] << " gen username password" << std::endl;
-      return 1;
+      return 0;
     }
   }
   if (cmd == "check")
@@ -79,12 +77,14 @@ int main(int argc, char *argv[])
     std::cout << "credential: ";
     if (!std::getline(std::cin, cred))
     {
+      std::cout << "read error" << std::endl;
       return 1;
     }
     std::string passwd;
     std::cout << "password: ";
     if (!std::getline(std::cin, passwd))
     {
+      std::cout << "read error" << std::endl;
       return 1;
     }
     if (check_cred(cred, passwd))
@@ -95,7 +95,6 @@ int main(int argc, char *argv[])
     std::cout << "bad login" << std::endl;
     return 1;
   }
-
   print_help(argv[0]);
   return 1;
 }

contrib/backends/nntpchan-daemon/tools/testtool.cpp

@@ -3,9 +3,9 @@
 #include <nntpchan/exec_frontend.hpp>
 #include <nntpchan/sanitize.hpp>
 
-int main(int, char *[])
+int main(int, char *[], char * argenv[])
 {
-  nntpchan::Frontend_ptr f(new nntpchan::ExecFrontend("./contrib/nntpchan.sh"));
+  nntpchan::Frontend_ptr f(new nntpchan::ExecFrontend("./contrib/nntpchan.sh", argenv));
   assert(nntpchan::IsValidMessageID("<a28a71493831188@web.oniichan.onion>"));
   assert(f->AcceptsNewsgroup("overchan.test"));
   std::cout << "all good" << std::endl;

contrib/backends/nntpchand/files/templates/overchan/error.html.tmpl

@@ -1,11 +0,0 @@
-<!doctype html>
-<html>
-
-  <head>
-    <title> Error </title>
-  </head>
-
-  <body>
-    <pre> {{ .Error}} </pre>
-  </body>
-</html>

contrib/backends/nntpchand/files/templates/overchan/index.html.tmpl

@@ -1,11 +0,0 @@
-<!doctype html>
-<html>
-
-  <head>
-    <title> Overchan </title>
-  </head>
-
-  <body>
-    <pre>ebin</pre>
-  </body>
-</html>

contrib/backends/nntpchand/src/nntpchan/lib/config/middleware.go

@@ -6,9 +6,12 @@ type MiddlewareConfig struct {
 	Type string `json:"type"`
 	// directory for our html templates
 	Templates string `json:"templates_dir"`
+	// directory for static files
+	StaticDir string `json:"static_dir"`
 }
 
 var DefaultMiddlewareConfig = MiddlewareConfig{
 	Type:      "overchan",
 	Templates: "./files/templates/overchan/",
+	StaticDir: "./files/",
 }

contrib/backends/nntpchand/src/nntpchan/lib/frontend/overchan.go

@@ -14,10 +14,11 @@ import (
 
 // standard overchan imageboard middleware
 type overchanMiddleware struct {
-	templ   *template.Template
-	captcha *CaptchaServer
-	store   *sessions.CookieStore
-	db      database.Database
+	templ     *template.Template
+	captcha   *CaptchaServer
+	store     *sessions.CookieStore
+	db        database.Database
+	staticDir string
 }
 
 func (m *overchanMiddleware) SetupRoutes(mux *mux.Router) {
@@ -34,6 +35,8 @@ func (m *overchanMiddleware) SetupRoutes(mux *mux.Router) {
 	m.captcha = NewCaptchaServer(200, 400, captchaPrefix, m.store)
 	// setup captcha endpoint
 	m.captcha.SetupRoutes(mux.PathPrefix(captchaPrefix).Subrouter())
+	mux.Path("/static/").Handler(http.FileServer(http.Dir(m.staticDir)))
+
 }
 
 // reload middleware
@@ -57,9 +60,9 @@ func (m *overchanMiddleware) ServeBoardPage(w http.ResponseWriter, r *http.Reque
 		var obj interface{}
 		obj, err = m.db.BoardPage(board, pageno, 10)
 		if err == nil {
-			m.serveTemplate(w, r, "board.html.tmpl", obj)
+			m.serveTemplate(w, r, "board.html", obj)
 		} else {
-			m.serveTemplate(w, r, "error.html.tmpl", err)
+			m.serveTemplate(w, r, "error.html", err)
 		}
 	} else {
 		// 404
@@ -72,15 +75,15 @@ func (m *overchanMiddleware) ServeThread(w http.ResponseWriter, r *http.Request)
 	param := mux.Vars(r)
 	obj, err := m.db.ThreadByHash(param["id"])
 	if err == nil {
-		m.serveTemplate(w, r, "thread.html.tmpl", obj)
+		m.serveTemplate(w, r, "thread.html", obj)
 	} else {
-		m.serveTemplate(w, r, "error.html.tmpl", err)
+		m.serveTemplate(w, r, "error.html", err)
 	}
 }
 
 // serve index page
 func (m *overchanMiddleware) ServeIndex(w http.ResponseWriter, r *http.Request) {
-	m.serveTemplate(w, r, "index.html.tmpl", nil)
+	m.serveTemplate(w, r, "index.html", nil)
 }
 
 // serve a template

contrib/backends/srndv2/Makefile

@@ -2,13 +2,15 @@ GOROOT ?= $(shell go env GOROOT)
 GO ?= $(GOROOT)/bin/go
 REPO=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 VERSION=$(shell $(GO) version | cut -d' ' -f3)
-
+GOARCH ?= $(shell go env GOARCH)
+GOOS ?= $(shell go env GOOS)
+GOARM ?= $(shell go env GOARM)
 all: build
 
 build: srndv2
 
 srndv2:
-	GOPATH=$(REPO) GOROOT=$(GOROOT) $(GO) build -ldflags "-X srnd.GitVersion=-$(shell git rev-parse --short HEAD)" -v
+	GOARM=$(GOARM) GOOS=$(GOOS) GOARCH=$(GOARCH) GOPATH=$(REPO) GOROOT=$(GOROOT) $(GO) build -ldflags "-X srnd.GitVersion=-$(shell git rev-parse --short HEAD)" -v
 
 srndv2-lua:
 	GOPATH=$(REPO) GOROOT=$(GOROOT) $(GO) build -ldflags "-X srnd.GitVersion=-$(shell git rev-parse --short HEAD)" -tags lua -v

contrib/backends/srndv2/src/srnd/attachment.go

@@ -122,7 +122,7 @@ func (self *nntpAttachment) Save(dir string) (err error) {
 		fpath := filepath.Join(dir, self.filepath)
 		if !CheckFile(fpath) {
 			var f io.WriteCloser
-			// does not exist so will will write it
+			// does not exist so will write it
 			f, err = os.Create(fpath)
 			if err == nil {
 				_, err = f.Write(self.Bytes())

contrib/backends/srndv2/src/srnd/cache_interface.go

@@ -3,6 +3,7 @@ package srnd
 import (
 	"log"
 	"net/http"
+	"strconv"
 )
 
 type CacheHandler interface {
@@ -12,7 +13,7 @@ type CacheHandler interface {
 
 type CacheInterface interface {
 	RegenAll()
-	RegenFrontPage()
+	RegenFrontPage(pagesstart int)
 	RegenOnModEvent(newsgroup, msgid, root string, page int)
 	RegenerateBoard(group string)
 	Regen(msg ArticleEntry)
@@ -25,6 +26,7 @@ type CacheInterface interface {
 	GetHandler() CacheHandler
 
 	SetRequireCaptcha(required bool)
+	InvertPagination()
 }
 
 //TODO only pass needed config
@@ -45,7 +47,11 @@ func NewCache(cache_type, host, port, user, password string, cache_config, confi
 	if cache_type == "varnish" {
 		url := cache_config["url"]
 		bind_addr := cache_config["bind"]
-		return NewVarnishCache(url, bind_addr, prefix, webroot, name, translations, attachments, db, store)
+		workers, _ := strconv.Atoi(cache_config["workers"])
+		if workers <= 0 {
+			workers = 4
+		}
+		return NewVarnishCache(url, bind_addr, prefix, webroot, name, translations, workers, attachments, db, store)
 	}
 
 	log.Fatalf("invalid cache type: %s", cache_type)

contrib/backends/srndv2/src/srnd/config.go

@@ -8,6 +8,7 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/base32"
+	"encoding/hex"
 	"fmt"
 	"github.com/majestrate/configparser"
 	"io/ioutil"
@@ -176,7 +177,7 @@ func GenSRNdConfig() *configparser.Configuration {
 	// nntp related section
 	sect := conf.NewSection("nntp")
 	sect.Add("instance_name", "test.srndv2.tld")
-	sect.Add("bind", "127.0.0.1:1199")
+	sect.Add("bind", ":1199")
 	sect.Add("sync_on_start", "1")
 	sect.Add("allow_anon", "0")
 	sect.Add("allow_anon_attachments", "0")
@@ -187,6 +188,7 @@ func GenSRNdConfig() *configparser.Configuration {
 	sect.Add("archive", "0")
 	sect.Add("article_lifetime", "0")
 	sect.Add("filters_file", "filters.txt")
+	sect.Add("secretkey", hex.EncodeToString(randbytes(32)))
 
 	// spamd settings
 	sect = conf.NewSection("spamd")
@@ -267,7 +269,7 @@ func GenSRNdConfig() *configparser.Configuration {
 	sect.Add("minimize_html", "0")
 	sect.Add("prefix", "/")
 	sect.Add("static_files", "contrib")
-	sect.Add("templates", "contrib/templates/default")
+	sect.Add("templates", "contrib/templates/placebo")
 	sect.Add("translations", "contrib/translations")
 	sect.Add("markup_script", "contrib/lua/memeposting.lua")
 	sect.Add("locale", "en")

contrib/backends/srndv2/src/srnd/daemon.go

@@ -443,7 +443,7 @@ func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
 
 			if mode == "sync" {
 				// yeh, do it
-				self.syncPull(conf.proxy_type, conf.proxy_addr, conf.Addr)
+				self.syncPull(conf)
 				// sleep for the sleep interval and continue
 				log.Println(conf.Name, "waiting for", conf.sync_interval, "before next sync")
 				time.Sleep(conf.sync_interval)
@@ -492,15 +492,16 @@ func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
 }
 
 // do a oneshot pull based sync with another server
-func (self *NNTPDaemon) syncPull(proxy_type, proxy_addr, remote_addr string) {
-	c, err := self.dialOut(proxy_type, proxy_addr, remote_addr)
+func (self *NNTPDaemon) syncPull(conf *FeedConfig) {
+	c, err := self.dialOut(conf.proxy_type, conf.proxy_addr, conf.Addr)
 	if err == nil {
 		conn := textproto.NewConn(c)
 		// we connected
-		nntp := createNNTPConnection(remote_addr)
-		nntp.name = remote_addr + "-sync"
+		nntp := createNNTPConnection(conf.Addr)
+		nntp.name = conf.Addr + "-sync"
+		nntp.feedname = conf.Name
 		// do handshake
-		_, reader, _, err := nntp.outboundHandshake(conn, nil)
+		_, reader, _, err := nntp.outboundHandshake(conn, conf)
 
 		if err != nil {
 			log.Println("failed to scrape server", err)
@@ -534,9 +535,17 @@ func (self *NNTPDaemon) ExpireAll() {
 	self.expire.ExpireOrphans()
 }
 
+func (self *NNTPDaemon) MarkSpam(msgid string) {
+	if ValidMessageID(msgid) {
+		err := self.mod.MarkSpam(msgid)
+		if err != nil {
+			log.Println(err)
+		}
+	}
+}
+
 // run daemon
 func (self *NNTPDaemon) Run() {
-	self.spamFilter.Configure(self.conf.spamconf)
 	self.bind_addr = self.conf.daemon["bind"]
 
 	listener, err := net.Listen("tcp", self.bind_addr)
@@ -572,7 +581,7 @@ func (self *NNTPDaemon) Run() {
 
 	self.register_connection = make(chan *nntpConnection)
 	self.deregister_connection = make(chan *nntpConnection)
-	self.send_all_feeds = make(chan ArticleEntry)
+	self.send_all_feeds = make(chan ArticleEntry, 128)
 	self.activeConnections = make(map[string]*nntpConnection)
 	self.loadedFeeds = make(map[string]*feedState)
 	self.register_feed = make(chan FeedConfig)
@@ -580,12 +589,13 @@ func (self *NNTPDaemon) Run() {
 	self.get_feeds = make(chan chan []*feedStatus)
 	self.get_feed = make(chan *feedStatusQuery)
 	self.modify_feed_policy = make(chan *modifyFeedPolicyEvent)
-	self.ask_for_article = make(chan string)
+	self.ask_for_article = make(chan string, 128)
 
 	self.pump_ticker = time.NewTicker(time.Millisecond * 100)
 	if self.conf.daemon["archive"] == "1" {
 		log.Println("running in archive mode")
 		self.expire = nil
+		self.frontend.ArchiveMode()
 	} else {
 		self.expire = createExpirationCore(self.database, self.store, self.informHooks)
 	}
@@ -893,17 +903,17 @@ func (self *NNTPDaemon) processMessage(msgid string) {
 		if self.expire != nil {
 			// expire posts
 			log.Println("expire", group, "for", rollover, "threads")
-			self.expire.ExpireGroup(group, rollover)
+			go self.expire.ExpireGroup(group, rollover)
 		}
 		// send to mod panel
 		if group == "ctl" {
 			log.Println("process mod message", msgid)
-			self.mod.HandleMessage(msgid)
+			go self.mod.HandleMessage(msgid)
 		}
 		// inform callback hooks
 		go self.informHooks(group, msgid, ref)
 		// federate
-		self.sendAllFeeds(ArticleEntry{msgid, group})
+		go self.sendAllFeeds(ArticleEntry{msgid, group})
 		// send to frontend
 		if self.frontend != nil {
 			if self.frontend.AllowNewsgroup(group) {
@@ -1142,10 +1152,17 @@ func (self *NNTPDaemon) Setup() {
 		self.frontend = NewHTTPFrontend(self, self.cache, self.conf.frontend, self.conf.worker["url"])
 	}
 
+	self.spamFilter.Configure(self.conf.spamconf)
+
+	regen := func(string, string, string, int) {}
+	if self.frontend != nil {
+		regen = self.frontend.RegenOnModEvent
+	}
 	self.mod = &modEngine{
+		//spam:     &self.spamFilter,
 		store:    self.store,
 		database: self.database,
-		regen:    self.frontend.RegenOnModEvent,
+		regen:    regen,
 	}
 	// inject DB into template engine
 	template.DB = self.database

contrib/backends/srndv2/src/srnd/database.go

@@ -47,14 +47,12 @@ func (self PostEntry) Count() int64 {
 	return self[1]
 }
 
+type PostEntryList []PostEntry
+
 // stats about newsgroup postings
 type NewsgroupStats struct {
-	Posted []PostEntry
-	Delted []PostEntry
-	Hits   []PostEntry
-	Start  time.Time
-	End    time.Time
-	Name   string
+	PPD  int64
+	Name string
 }
 
 type PostingStatsEntry struct {
@@ -124,6 +122,9 @@ type Database interface {
 	// if N <= 0 then count all we have now
 	CountPostsInGroup(group string, time_frame int64) int64
 
+	// get the stats for the overview page
+	GetNewsgroupStats() ([]NewsgroupStats, error)
+
 	// get all replies to a thread
 	// if last > 0 then get that many of the last replies
 	// start at reply number start
@@ -135,6 +136,9 @@ type Database interface {
 	// get all attachments for this message
 	GetPostAttachments(message_id string) []string
 
+	// get all attachments in a thread
+	GetThreadAttachments(rootmsgid string) ([]string, error)
+
 	// get all attachments for this message
 	GetPostAttachmentModels(prefix, message_id string) []AttachmentModel
 
@@ -240,6 +244,9 @@ type Database interface {
 	// delete an article from the database
 	DeleteArticle(msg_id string) error
 
+	// remove an article from the database
+	RemoveArticle(msg_id string) error
+
 	// detele the existance of a thread from the threads table, does NOT remove replies
 	DeleteThread(root_msg_id string) error
 
@@ -302,10 +309,12 @@ type Database interface {
 	GetMessageIDByEncryptedIP(encaddr string) ([]string, error)
 
 	// check if this public key is banned from posting
-	PubkeyIsBanned(pubkey string) (bool, error)
+	PubkeyRejected(pubkey string) (bool, error)
 
 	// ban a public key from posting
-	BanPubkey(pubkey string) error
+	BlacklistPubkey(pubkey string) error
+	WhitelistPubkey(pubkey string) error
+	DeletePubkey(pubkey string) error
 
 	// get all message-id posted before a time
 	GetPostsBefore(t time.Time) ([]string, error)
@@ -314,10 +323,10 @@ type Database interface {
 	GetPostingStats(granularity, begin, end int64) (PostingStats, error)
 
 	// peform search query
-	SearchQuery(prefix, group, text string, chnl chan PostModel) error
+	SearchQuery(prefix, group, text string, chnl chan PostModel, limit int) error
 
 	// find posts with similar hash
-	SearchByHash(prefix, group, posthash string, chnl chan PostModel) error
+	SearchByHash(prefix, group, posthash string, chnl chan PostModel, limit int) error
 
 	// get full thread model
 	GetThreadModel(prefix, root_msgid string) (ThreadModel, error)
@@ -336,6 +345,9 @@ type Database interface {
 
 	// find headers in group with lo/hi watermark and list of patterns
 	FindHeaders(group, headername string, lo, hi int64) (ArticleHeaders, error)
+
+	// count ukko pages
+	GetUkkoPageCount(perpage int) (int64, error)
 }
 
 func NewDatabase(db_type, schema, host, port, user, password string) Database {

contrib/backends/srndv2/src/srnd/expiration.go

@@ -46,9 +46,8 @@ type expire struct {
 }
 
 func (self expire) ExpirePost(messageID string) {
-	self.handleEvent(deleteEvent(self.store.GetFilename(messageID)))
-	// get article headers
 	headers := self.store.GetHeaders(messageID)
+	// get article headers
 	if headers != nil {
 		group := headers.Get("Newsgroups", "")
 		// is this a root post ?
@@ -57,6 +56,7 @@ func (self expire) ExpirePost(messageID string) {
 			// ya, expire the entire thread
 			self.ExpireThread(group, messageID)
 		} else {
+			self.handleEvent(deleteEvent(self.store.GetFilename(messageID)))
 			self.expireCache(group, messageID, ref)
 		}
 	}
@@ -70,14 +70,26 @@ func (self expire) ExpireGroup(newsgroup string, keep int) {
 }
 
 func (self expire) ExpireThread(group, rootMsgid string) {
-	replies, err := self.database.GetMessageIDByHeader("References", rootMsgid)
+	files, err := self.database.GetThreadAttachments(rootMsgid)
 	if err == nil {
-		for _, reply := range replies {
-			self.handleEvent(deleteEvent(self.store.GetFilename(reply)))
+		for _, file := range files {
+			img := self.store.AttachmentFilepath(file)
+			os.Remove(img)
+			thm := self.store.ThumbnailFilepath(file)
+			os.Remove(thm)
 		}
+	} else {
+		log.Println("expirethread::GetThreadAttachments:", err)
 	}
+
+	replies := self.database.GetThreadReplies(rootMsgid, 0, 0)
+
+	for _, msgid := range replies {
+		self.store.Remove(msgid)
+	}
+
+	self.store.Remove(rootMsgid)
 	self.database.DeleteThread(rootMsgid)
-	self.database.DeleteArticle(rootMsgid)
 	self.expireCache(group, rootMsgid, rootMsgid)
 }
 
@@ -137,11 +149,14 @@ func (self expire) handleEvent(ev deleteEvent) {
 			os.Remove(thm)
 		}
 	}
-	err := self.database.BanArticle(ev.MessageID(), "expired")
-	if err != nil {
-		log.Println("failed to ban for expiration", err)
+	banned := self.database.ArticleBanned(ev.MessageID())
+	if !banned {
+		err := self.database.BanArticle(ev.MessageID(), "expired")
+		if err != nil {
+			log.Println("failed to ban for expiration", err)
+		}
 	}
-	err = self.database.DeleteArticle(ev.MessageID())
+	err := self.database.DeleteArticle(ev.MessageID())
 	if err != nil {
 		log.Println("failed to delete article", err)
 	}

contrib/backends/srndv2/src/srnd/file_cache.go

@@ -145,7 +145,7 @@ func (self *FileCache) pollRegen() {
 			// regen ukko
 		case _ = <-self.ukkoTicker.C:
 			self.regenUkko()
-			self.RegenFrontPage()
+			self.RegenFrontPage(0)
 		case _ = <-self.regenThreadTicker.C:
 			self.regenThreadLock.Lock()
 			for _, entry := range self.regenThreadMap {
@@ -157,8 +157,9 @@ func (self *FileCache) pollRegen() {
 		case _ = <-self.regenBoardTicker.C:
 			self.regenBoardLock.Lock()
 			for _, v := range self.regenBoardMap {
-				self.regenerateBoardPage(v.group, v.page, false)
-				self.regenerateBoardPage(v.group, v.page, true)
+				pages := self.database.GetGroupPageCount(v.group)
+				self.regenerateBoardPage(v.group, int(pages), v.page, false)
+				self.regenerateBoardPage(v.group, int(pages), v.page, true)
 			}
 			self.regenBoardMap = make(map[string]groupRegenRequest)
 			self.regenBoardLock.Unlock()
@@ -173,12 +174,15 @@ func (self *FileCache) pollRegen() {
 	}
 }
 
+func (self *FileCache) InvertPagination() {
+}
+
 // regen every page of the board
 func (self *FileCache) RegenerateBoard(group string) {
 	pages, _ := self.database.GetPagesPerBoard(group)
 	for page := 0; page < pages; page++ {
-		self.regenerateBoardPage(group, page, false)
-		self.regenerateBoardPage(group, page, true)
+		self.regenerateBoardPage(group, int(pages), page, false)
+		self.regenerateBoardPage(group, int(pages), page, true)
 	}
 }
 
@@ -193,14 +197,14 @@ func (self *FileCache) regenerateThread(root ArticleEntry, json bool) {
 			log.Println("did not write", fname, err)
 			return
 		}
-		template.genThread(self.attachments, self.requireCaptcha, root, self.prefix, self.name, wr, self.database, json, nil)
+		template.genThread(self.attachments, self.requireCaptcha, root, self.prefix, self.name, wr, self.database, json, nil, false)
 	} else {
 		log.Println("don't have root post", msgid, "not regenerating thread")
 	}
 }
 
 // regenerate just a page on a board
-func (self *FileCache) regenerateBoardPage(board string, page int, json bool) {
+func (self *FileCache) regenerateBoardPage(board string, pages, page int, json bool) {
 	fname := self.getFilenameForBoardPage(board, page, json)
 	wr, err := os.Create(fname)
 	defer wr.Close()
@@ -208,7 +212,7 @@ func (self *FileCache) regenerateBoardPage(board string, page int, json bool) {
 		log.Println("error generating board page", page, "for", board, err)
 		return
 	}
-	template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, page, wr, self.database, json, nil)
+	template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, pages, page, wr, self.database, json, nil, false, false)
 }
 
 // regenerate the catalog for a board
@@ -220,11 +224,11 @@ func (self *FileCache) regenerateCatalog(board string) {
 		log.Println("error generating catalog for", board, err)
 		return
 	}
-	template.genCatalog(self.prefix, self.name, board, wr, self.database, nil)
+	template.genCatalog(self.prefix, self.name, board, wr, self.database, nil, false)
 }
 
 // regenerate the front page
-func (self *FileCache) RegenFrontPage() {
+func (self *FileCache) RegenFrontPage(_ int) {
 	indexwr, err1 := os.Create(filepath.Join(self.webroot_dir, "index.html"))
 	defer indexwr.Close()
 	if err1 != nil {
@@ -260,7 +264,7 @@ func (self *FileCache) regenUkko() {
 		log.Println("error generating ukko markup", err)
 		return
 	}
-	template.genUkko(self.prefix, self.name, wr, self.database, false, nil)
+	template.genUkko(self.prefix, self.name, wr, self.database, false, nil, false, false)
 
 	// json
 	fname = filepath.Join(self.webroot_dir, "ukko.json")
@@ -270,7 +274,7 @@ func (self *FileCache) regenUkko() {
 		log.Println("error generating ukko json", err)
 		return
 	}
-	template.genUkko(self.prefix, self.name, wr, self.database, true, nil)
+	template.genUkko(self.prefix, self.name, wr, self.database, true, nil, false, false)
 	i := 0
 	for i < 10 {
 		fname := fmt.Sprintf("ukko-%d.html", i)
@@ -281,14 +285,14 @@ func (self *FileCache) regenUkko() {
 			return
 		}
 		defer f.Close()
-		template.genUkkoPaginated(self.prefix, self.name, f, self.database, i, false, nil)
+		template.genUkkoPaginated(self.prefix, self.name, f, self.database, 10, i, false, nil, false, false)
 		j, err := os.Create(jname)
 		if err != nil {
 			log.Printf("failed to create json ukko", i, err)
 			return
 		}
 		defer j.Close()
-		template.genUkkoPaginated(self.prefix, self.name, j, self.database, i, true, nil)
+		template.genUkkoPaginated(self.prefix, self.name, j, self.database, 10, i, true, nil, false, false)
 	}
 }
 

contrib/backends/srndv2/src/srnd/frontend.go

@@ -41,4 +41,7 @@ type Frontend interface {
 	RegenOnModEvent(newsgroup, msgid, root string, page int)
 
 	GetCacheHandler() CacheHandler
+
+	// set archive mode
+	ArchiveMode()
 }

contrib/backends/srndv2/src/srnd/frontend_http.go

@@ -125,7 +125,7 @@ func (lc *liveChan) SendBanned() {
 	msg, _ := json.Marshal(map[string]string{
 		"Type": "ban",
 		// TODO: real ban message
-		"Reason": "your an faget, your IP was: " + lc.IP,
+		"Reason": "your an fagt, your IP was: " + lc.IP,
 	})
 	if lc.datachnl != nil {
 		lc.datachnl <- msg
@@ -209,6 +209,9 @@ type httpFrontend struct {
 
 	// this is a very important thing by the way
 	requireCaptcha bool
+
+	// are we in archive mode?
+	archive bool
 }
 
 // do we allow this newsgroup?
@@ -225,7 +228,8 @@ func (self *httpFrontend) RegenerateBoard(board string) {
 }
 
 func (self *httpFrontend) RegenFrontPage() {
-	self.cache.RegenFrontPage()
+	pages, _ := self.daemon.database.GetUkkoPageCount(10)
+	self.cache.RegenFrontPage(int(pages))
 }
 
 func (self httpFrontend) regenAll() {
@@ -240,6 +244,10 @@ func (self httpFrontend) deleteBoardMarkup(group string) {
 	self.cache.DeleteBoardMarkup(group)
 }
 
+func (self *httpFrontend) ArchiveMode() {
+	self.archive = true
+}
+
 // load post model and inform live ui
 func (self *httpFrontend) informLiveUI(msgid, ref, group string) {
 	// root post
@@ -360,7 +368,7 @@ func (self *httpFrontend) poll_liveui() {
 func (self *httpFrontend) poll() {
 
 	// regenerate front page
-	self.cache.RegenFrontPage()
+	self.RegenFrontPage()
 
 	// trigger regen
 	if self.regen_on_start {
@@ -384,21 +392,32 @@ func (self *httpFrontend) HandleNewPost(nntp frontendPost) {
 	if len(ref) > 0 {
 		msgid = ref
 	}
+
 	entry := ArticleEntry{msgid, group}
 	// regnerate thread
 	self.Regen(entry)
-	// regenerate all board pages
-	self.RegenerateBoard(group)
+	// regenerate all board pages if not archiving
+	if !self.archive {
+		self.RegenerateBoard(group)
+	}
 	// regen front page
 	self.RegenFrontPage()
+
 }
 
 // create a new captcha, return as json object
 func (self *httpFrontend) new_captcha_json(wr http.ResponseWriter, r *http.Request) {
+	s, err := self.store.Get(r, self.name)
+	if err != nil {
+		http.Error(wr, err.Error(), 500)
+		return
+	}
 	captcha_id := captcha.New()
 	resp := make(map[string]string)
 	// the captcha id
 	resp["id"] = captcha_id
+	s.Values["captcha_id"] = captcha_id
+	s.Save(r, wr)
 	// url of the image
 	resp["url"] = fmt.Sprintf("%scaptcha/%s.png", self.prefix, captcha_id)
 	wr.Header().Set("Content-Type", "text/json; encoding=UTF-8")
@@ -418,6 +437,7 @@ func (self *httpFrontend) handle_postform(wr http.ResponseWriter, r *http.Reques
 
 	// the post we will turn into an nntp article
 	pr := new(postRequest)
+	pr.ExtraHeaders = make(map[string]string)
 
 	if sendJson {
 		wr.Header().Add("Content-Type", "text/json; encoding=UTF-8")
@@ -501,6 +521,11 @@ func (self *httpFrontend) handle_postform(wr http.ResponseWriter, r *http.Reques
 				captcha_solution = part_buff.String()
 			} else if partname == "dubs" {
 				pr.Dubs = part_buff.String() == "on"
+			} else if partname == "uri" {
+				str := part_buff.String()
+				if len(str) > 0 {
+					pr.ExtraHeaders["X-References-Uri"] = safeHeader(str)
+				}
 			}
 
 			// we done
@@ -528,15 +553,26 @@ func (self *httpFrontend) handle_postform(wr http.ResponseWriter, r *http.Reques
 		}
 	}
 
-	sess, _ := self.store.Get(r, self.name)
+	sess, err := self.store.Get(r, self.name)
+	if err != nil {
+		errmsg := fmt.Sprintf("session store error: %s", err.Error())
+		if sendJson {
+			json.NewEncoder(wr).Encode(map[string]interface{}{"error": errmsg})
+		} else {
+			io.WriteString(wr, errmsg)
+		}
+		return
+	}
 	if checkCaptcha && len(captcha_id) == 0 {
 		cid, ok := sess.Values["captcha_id"]
 		if ok {
 			captcha_id = cid.(string)
+		} else {
+			log.Println("no captcha id in session?")
 		}
 		sess.Values["captcha_id"] = ""
 	}
-
+	log.Println("captcha", captcha_id, "try '", captcha_solution, "'")
 	if checkCaptcha && !captcha.VerifyString(captcha_id, captcha_solution) {
 		// captcha is not valid
 		captcha_retry = true
@@ -709,8 +745,7 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 		}
 	}
 
-	// always lower case newsgroups
-	board := strings.ToLower(pr.Group)
+	board := pr.Group
 
 	// post fail message
 	banned, err = self.daemon.database.NewsgroupBanned(board)
@@ -733,7 +768,7 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 	}
 
 	ref := pr.Reference
-	if len(ref) > 0 {
+	if ref != "" {
 		if ValidMessageID(ref) {
 			if self.daemon.database.HasArticleLocal(ref) {
 				nntp.headers.Set("References", ref)
@@ -774,10 +809,10 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 		return
 	}
 
-	subject := pr.Subject
+	subject := strings.TrimSpace(pr.Subject)
 
 	// set subject
-	if len(subject) == 0 {
+	if subject == "" {
 		subject = "None"
 	} else if len(subject) > 256 {
 		// subject too big
@@ -785,28 +820,20 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 		return
 	}
 
-	nntp.headers.Set("Subject", subject)
-	if isSage(subject) {
+	nntp.headers.Set("Subject", safeHeader(subject))
+	if isSage(subject) && ref != "" {
 		nntp.headers.Set("X-Sage", "1")
 	}
 
-	name := pr.Name
-
+	name := strings.TrimSpace(pr.Name)
 	var tripcode_privkey []byte
-
-	// set name
-	if len(name) == 0 {
+	// tripcode
+	if idx := strings.IndexByte(name, '#'); idx >= 0 {
+		tripcode_privkey = parseTripcodeSecret(name[idx+1:])
+		name = strings.TrimSpace(name[:idx])
+	}
+	if name == "" {
 		name = "Anonymous"
-	} else {
-		idx := strings.Index(name, "#")
-		// tripcode
-		if idx >= 0 {
-			tripcode_privkey = parseTripcodeSecret(name[idx+1:])
-			name = strings.Trim(name[:idx], "\t ")
-			if name == "" {
-				name = "Anonymous"
-			}
-		}
 	}
 	if len(name) > 128 {
 		// name too long
@@ -819,7 +846,7 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 		msgid = genMessageID(pr.Frontend)
 	}
 
-	nntp.headers.Set("From", nntpSanitize(fmt.Sprintf("%s <poster@%s>", name, pr.Frontend)))
+	nntp.headers.Set("From", formatAddress(safeHeader(name), "poster@"+pr.Frontend))
 	nntp.headers.Set("Message-ID", msgid)
 
 	// set message
@@ -832,7 +859,21 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 	}
 
 	if len(cites) > 0 {
-		nntp.headers.Set("Reply-To", strings.Join(cites, " "))
+		if ref == "" && len(cites) == 1 {
+			/*
+				this is workaround for:
+
+				{RFC 5322}
+				If the parent message does not contain
+				a "References:" field but does have an "In-Reply-To:" field
+				containing a single message identifier, then the "References:" field
+				will contain the contents of the parent's "In-Reply-To:" field
+				followed by the contents of the parent's "Message-ID:" field (if
+				any).
+			*/
+			cites = append(cites, "<0>")
+		}
+		nntp.headers.Set("In-Reply-To", strings.Join(cites, " "))
 	}
 
 	// set date
@@ -949,6 +990,7 @@ func (self *httpFrontend) serve_captcha(wr http.ResponseWriter, r *http.Request)
 	if err == nil {
 		captcha_id := captcha.New()
 		s.Values["captcha_id"] = captcha_id
+		log.Println("captcha_id", captcha_id)
 		s.Save(r, wr)
 		redirect_url := fmt.Sprintf("%scaptcha/%s.png", self.prefix, captcha_id)
 		// redirect to the image
@@ -1028,27 +1070,30 @@ func (self httpFrontend) handle_authed_api(wr http.ResponseWriter, r *http.Reque
 func (self *httpFrontend) handle_api_find(wr http.ResponseWriter, r *http.Request) {
 	q := r.URL.Query()
 	h := q.Get("hash")
+	msgid := q.Get("id")
 	if len(h) > 0 {
-		msgid := q.Get("id")
-		if len(h) > 0 {
-			e, err := self.daemon.database.GetMessageIDByHash(h)
-			if err == nil {
-				msgid = e.MessageID()
-			}
+		e, err := self.daemon.database.GetMessageIDByHash(h)
+		if err == nil {
+			msgid = e.MessageID()
 		}
-		if len(msgid) > 0 {
-			// found it (probaly)
-			model := self.daemon.database.GetPostModel(self.prefix, msgid)
-			if model == nil {
-				// no model
+	}
+
+	if !ValidMessageID(msgid) {
+		msgid = ""
+	}
+
+	if len(msgid) > 0 {
+		self.daemon.store.GetMessage(msgid, func(nntp NNTPMessage) {
+			if nntp == nil {
 				wr.WriteHeader(404)
-			} else {
-				// we found it
-				wr.Header().Add("Content-Type", "text/json; encoding=UTF-8")
-				json.NewEncoder(wr).Encode([]PostModel{model})
+				return
 			}
-			return
-		}
+			model := PostModelFromMessage(self.prefix, nntp)
+			// we found it
+			wr.Header().Add("Content-Type", "text/json; encoding=UTF-8")
+			json.NewEncoder(wr).Encode([]PostModel{model})
+		})
+		return
 	}
 	s := q.Get("text")
 	g := q.Get("group")
@@ -1075,13 +1120,14 @@ func (self *httpFrontend) handle_api_find(wr http.ResponseWriter, r *http.Reques
 		}
 		donechnl <- 0
 	}(wr)
+	limit := 50
 	if len(h) > 0 {
-		self.daemon.database.SearchByHash(self.prefix, g, h, chnl)
+		go self.daemon.database.SearchByHash(self.prefix, g, h, chnl, limit)
 	} else {
-		self.daemon.database.SearchQuery(self.prefix, g, s, chnl)
+		go self.daemon.database.SearchQuery(self.prefix, g, s, chnl, limit)
 	}
+	chnl <- nil
 	<-donechnl
-	close(donechnl)
 	io.WriteString(wr, " null ]")
 	return
 }
@@ -1395,6 +1441,7 @@ func (self *httpFrontend) Mainloop() {
 	m.Path("/mod/feeds").HandlerFunc(self.modui.ServeModPage).Methods("GET")
 	m.Path("/mod/keygen").HandlerFunc(self.modui.HandleKeyGen).Methods("GET")
 	m.Path("/mod/login").HandlerFunc(self.modui.HandleLogin).Methods("POST")
+	m.Path("/mod/spam").HandlerFunc(self.modui.HandlePostSpam).Methods("POST")
 	m.Path("/mod/del/{article_hash}").HandlerFunc(self.modui.HandleDeletePost).Methods("GET")
 	m.Path("/mod/ban/{address}").HandlerFunc(self.modui.HandleBanAddress).Methods("GET")
 	m.Path("/mod/unban/{address}").HandlerFunc(self.modui.HandleUnbanAddress).Methods("GET")
@@ -1439,6 +1486,10 @@ func (self *httpFrontend) Mainloop() {
 	// run daemon's mod engine with our frontend
 	// go RunModEngine(self.daemon.mod, self.cache.RegenOnModEvent)
 
+	if self.archive {
+		self.cache.InvertPagination()
+	}
+
 	// start cache
 	self.cache.Start()
 
@@ -1512,15 +1563,15 @@ func NewHTTPFrontend(daemon *NNTPDaemon, cache CacheInterface, config map[string
 	front.store = sessions.NewCookieStore([]byte(front.secret))
 	front.store.Options = &sessions.Options{
 		// TODO: detect http:// etc in prefix
-		Path:   front.prefix,
+		Path:   "/",
 		MaxAge: 600,
 	}
 
 	// liveui related members
 	front.liveui_chnl = make(chan PostModel, 128)
-	front.liveui_register = make(chan *liveChan)
-	front.liveui_deregister = make(chan *liveChan)
-	front.liveui_chans = make(map[string]*liveChan)
+	front.liveui_register = make(chan *liveChan, 128)
+	front.liveui_deregister = make(chan *liveChan, 128)
+	front.liveui_chans = make(map[string]*liveChan, 128)
 	front.end_liveui = make(chan bool)
 	return front
 }

contrib/backends/srndv2/src/srnd/frontend_multi.go

@@ -15,6 +15,12 @@ func (self multiFrontend) GetCacheHandler() CacheHandler {
 	return nil
 }
 
+func (self multiFrontend) ArchiveMode() {
+	for _, f := range self.frontends {
+		f.ArchiveMode()
+	}
+}
+
 func (self multiFrontend) AllowNewsgroup(newsgroup string) bool {
 	return true
 }

contrib/backends/srndv2/src/srnd/i18n.go

@@ -1,6 +1,7 @@
 package srnd
 
 import (
+	"errors"
 	"github.com/majestrate/configparser"
 	"golang.org/x/text/language"
 	"io/ioutil"
@@ -22,6 +23,7 @@ type I18N struct {
 }
 
 var I18nProvider *I18N = nil
+var ErrNoLang = errors.New("no such language")
 
 //Read all .ini files in dir, where the filenames are BCP 47 tags
 //Use the language matcher to get the best match for the locale preference
@@ -41,6 +43,7 @@ func NewI18n(locale, dir string) (*I18N, error) {
 		return nil, err
 	}
 
+	found := false
 	serverLangs := make([]language.Tag, 1)
 	serverLangs[0] = language.AmericanEnglish // en-US fallback
 	for _, file := range files {
@@ -49,9 +52,14 @@ func NewI18n(locale, dir string) (*I18N, error) {
 			tag, err := language.Parse(name)
 			if err == nil {
 				serverLangs = append(serverLangs, tag)
+				found = true
 			}
 		}
 	}
+	if !found {
+		return nil, ErrNoLang
+	}
+
 	matcher := language.NewMatcher(serverLangs)
 	tag, _, _ := matcher.Match(pref)
 

contrib/backends/srndv2/src/srnd/markup.go

@@ -47,7 +47,7 @@ func backlink(word, prefix string) (markup string) {
 			if len(parts) > 1 {
 				longhash = parts[1]
 			}
-			return `<a class='backlink' backlinkhash="` + longhash + `" href="` + url + `">&gt;&gt;` + link + "</a>"
+			return `<a class='backlink' data-backlinkhash="` + longhash + `" href="` + url + `">&gt;&gt;` + link + "</a>"
 		} else {
 			return escapeline(word)
 		}

contrib/backends/srndv2/src/srnd/message.go

@@ -15,9 +15,9 @@ import (
 	"log"
 	"mime"
 	"mime/multipart"
+	"net/mail"
 	"net/textproto"
 	"strings"
-	"time"
 )
 
 type ArticleHeaders map[string][]string
@@ -89,6 +89,7 @@ type NNTPMessage interface {
 	OP() bool
 	// all attachments
 	Attachments() []NNTPAttachment
+	FrontendPubkey() string
 	// all headers
 	Headers() ArticleHeaders
 	MIMEHeader() textproto.MIMEHeader
@@ -112,6 +113,10 @@ type NNTPMessage interface {
 	BodyReader() io.Reader
 }
 
+func (self *nntpArticle) FrontendPubkey() string {
+	return self.headers.Get("X-Frontend-Pubkey", "")
+}
+
 type nntpArticle struct {
 	// mime header
 	headers ArticleHeaders
@@ -130,7 +135,7 @@ func (self *nntpArticle) Reset() {
 	self.boundary = ""
 	self.message = ""
 	if self.attachments != nil {
-		for idx, _ := range self.attachments {
+		for idx := range self.attachments {
 			self.attachments[idx].Reset()
 			self.attachments[idx] = nil
 		}
@@ -151,7 +156,7 @@ func newPlaintextArticle(message, email, subject, name, instance, message_id, ne
 	nntp := &nntpArticle{
 		headers: make(ArticleHeaders),
 	}
-	nntp.headers.Set("From", fmt.Sprintf("%s <%s>", name, email))
+	nntp.headers.Set("From", formatAddress(name, email))
 	nntp.headers.Set("Subject", subject)
 	if isSage(subject) {
 		nntp.headers.Set("X-Sage", "1")
@@ -177,14 +182,14 @@ func signArticle(nntp NNTPMessage, seed []byte) (signed *nntpArticle, err error)
 		if k == "X-PubKey-Ed25519" || k == "X-Signature-Ed25519-SHA512" || k == "X-Signature-Ed25519-BLAKE2B" {
 			// don't set signature or pubkey header
 		} else if k == "Content-Type" {
-			signed.headers.Set(k, "message/rfc822; charset=UTF-8")
+			signed.headers.Set(k, "message/rfc822")
 		} else {
 			v := h[k][0]
 			signed.headers.Set(k, v)
 		}
 	}
 	sha := sha512.New()
-	blake := blake2b.New256()
+	blake := blake2b.New512()
 	signed.signedPart = &nntpAttachment{}
 	// write body to sign buffer
 	mw := io.MultiWriter(sha, blake, signed.signedPart)
@@ -257,7 +262,7 @@ func (self *nntpArticle) Pubkey() string {
 }
 
 func (self *nntpArticle) MessageID() (msgid string) {
-	for _, h := range []string{"Message-ID", "Messageid", "MessageID", "Message-Id"} {
+	for _, h := range []string{"Message-ID", "Message-Id"} {
 		mid := self.headers.Get(h, "")
 		if mid != "" {
 			msgid = string(mid)
@@ -290,12 +295,32 @@ func (self *nntpArticle) Newsgroup() string {
 }
 
 func (self *nntpArticle) Name() string {
-	from := self.headers.Get("From", "anonymous <a@no.n>")
-	idx := strings.Index(from, "<")
-	if idx > 1 {
-		return from[:idx]
+	const defname = "Anonymous"
+
+	from := strings.TrimSpace(self.headers.Get("From", ""))
+	if from == "" {
+		return defname
 	}
-	return "[Invalid From header]"
+
+	a, e := mail.ParseAddress(from)
+	var name string
+	if e != nil {
+		// try older method - some nodes generate non-compliant stuff
+		if i := strings.IndexByte(from, '<'); i > 1 {
+			name = from[:i]
+		} else {
+			return "[Invalid From header]"
+		}
+	} else {
+		name = a.Name
+	}
+
+	name = safeHeader(name)
+	if name == "" {
+		return defname
+	}
+
+	return name
 }
 
 func (self *nntpArticle) Addr() (addr string) {
@@ -322,13 +347,15 @@ func (self *nntpArticle) Addr() (addr string) {
 }
 
 func (self *nntpArticle) Email() string {
-	from := self.headers.Get("From", "anonymous <a@no.n>")
-	idx := strings.Index(from, "<")
-	if idx > 2 {
-		return from[:idx-2]
+	from := strings.TrimSpace(self.headers.Get("From", ""))
+	if from == "" {
+		return ""
 	}
-	return "[Invalid From header]"
-
+	a, e := mail.ParseAddress(from)
+	if e != nil {
+		return fmt.Sprintf("[Invalid From header: %v]", e)
+	}
+	return a.Address
 }
 
 func (self *nntpArticle) Subject() string {
@@ -337,7 +364,7 @@ func (self *nntpArticle) Subject() string {
 
 func (self *nntpArticle) Posted() int64 {
 	posted := self.headers.Get("Date", "")
-	t, err := time.Parse(time.RFC1123Z, posted)
+	t, err := mail.ParseDate(posted)
 	if err == nil {
 		return t.Unix()
 	}
@@ -458,7 +485,7 @@ func (self *nntpArticle) WriteBody(wr io.Writer, limit int64) (err error) {
 // verify a signed message's body
 // innerHandler must close reader when done
 // returns error if one happens while verifying article
-func verifyMessageSHA512(pk, sig string, body io.Reader, innerHandler func(map[string][]string, io.Reader)) (err error) {
+func verifyMessageSHA512(pk, sig string, body io.Reader, innerHandler func(ArticleHeaders, io.Reader)) (err error) {
 	log.Println("unwrapping signed message from", pk)
 	pk_bytes := unhex(pk)
 	sig_bytes := unhex(sig)
@@ -470,7 +497,7 @@ func verifyMessageSHA512(pk, sig string, body io.Reader, innerHandler func(map[s
 		r := bufio.NewReader(hdr_reader)
 		msg, err := readMIMEHeader(r)
 		if err == nil {
-			innerHandler(msg.Header, msg.Body)
+			innerHandler(ArticleHeaders(msg.Header), msg.Body)
 		}
 		hdr_reader.Close()
 	}(pr)
@@ -493,11 +520,11 @@ func verifyMessageSHA512(pk, sig string, body io.Reader, innerHandler func(map[s
 	return
 }
 
-func verifyMessageBLAKE2B(pk, sig string, body io.Reader, innerHandler func(map[string][]string, io.Reader)) (err error) {
+func verifyMessageBLAKE2B(pk, sig string, body io.Reader, innerHandler func(ArticleHeaders, io.Reader)) (err error) {
 	log.Println("unwrapping signed message from", pk)
 	pk_bytes := unhex(pk)
 	sig_bytes := unhex(sig)
-	h := blake2b.New256()
+	h := blake2b.New512()
 	pr, pw := io.Pipe()
 	// read header
 	// handle inner body
@@ -505,7 +532,7 @@ func verifyMessageBLAKE2B(pk, sig string, body io.Reader, innerHandler func(map[
 		r := bufio.NewReader(hdr_reader)
 		msg, err := readMIMEHeader(r)
 		if err == nil {
-			innerHandler(msg.Header, msg.Body)
+			innerHandler(ArticleHeaders(msg.Header), msg.Body)
 		}
 		hdr_reader.Close()
 	}(pr)

contrib/backends/srndv2/src/srnd/mod.go

@@ -44,6 +44,9 @@ type ModUI interface {
 	HandleKeyGen(wr http.ResponseWriter, r *http.Request)
 	// handle admin command
 	HandleAdminCommand(wr http.ResponseWriter, r *http.Request)
+	// handle mark a post as spam
+	HandlePostSpam(wr http.ResponseWriter, r *http.Request)
+
 	// get outbound message channel
 	MessageChan() chan NNTPMessage
 }
@@ -52,11 +55,14 @@ type ModAction string
 
 const ModInetBan = ModAction("overchan-inet-ban")
 const ModDelete = ModAction("delete")
+const ModRemove = ModAction("remove")
 const ModRemoveAttachment = ModAction("overchan-del-attachment")
 const ModStick = ModAction("overchan-stick")
 const ModLock = ModAction("overchan-lock")
 const ModHide = ModAction("overchan-hide")
 const ModSage = ModAction("overchan-sage")
+const ModSpam = ModAction("spam")
+const ModHam = ModAction("ham")
 const ModDeleteAlt = ModAction("delete")
 
 type ModEvent interface {
@@ -81,11 +87,17 @@ func (self simpleModEvent) String() string {
 }
 
 func (self simpleModEvent) Action() ModAction {
-	switch strings.Split(string(self), " ")[0] {
+	switch strings.ToLower(strings.Split(string(self), " ")[0]) {
+	case "remove":
+		return ModRemove
 	case "delete":
 		return ModDelete
 	case "overchan-inet-ban":
 		return ModInetBan
+	case "spam":
+		return ModSpam
+	case "ham":
+		return ModHam
 	}
 	return ""
 }
@@ -122,6 +134,11 @@ func overchanInetBan(encAddr, key string, expire int64) ModEvent {
 	return simpleModEvent(fmt.Sprintf("overchan-inet-ban %s:%s:%d", encAddr, key, expire))
 }
 
+// create a mark as spam event
+func modMarkSpam(msgid string) ModEvent {
+	return simpleModEvent(fmt.Sprintf("spam %s", msgid))
+}
+
 // moderation message
 // wraps multiple mod events
 // is turned into an NNTPMessage later
@@ -171,6 +188,8 @@ type ModEngine interface {
 	HandleMessage(msgid string)
 	// delete post of a poster
 	DeletePost(msgid string) error
+	// mark message as spam
+	MarkSpam(msgid string) error
 	// ban a cidr
 	BanAddress(cidr string) error
 	// do we allow this public key to delete this message-id ?
@@ -179,8 +198,6 @@ type ModEngine interface {
 	AllowBan(pubkey string) bool
 	// allow janitor
 	AllowJanitor(pubkey string) bool
-	// load a mod message
-	LoadMessage(msgid string) NNTPMessage
 	// execute 1 mod action line by a mod with pubkey
 	Execute(ev ModEvent, pubkey string)
 	// do a mod event unconditionally
@@ -190,11 +207,22 @@ type ModEngine interface {
 type modEngine struct {
 	database Database
 	store    ArticleStore
+	spam     *SpamFilter
 	regen    RegenFunc
 }
 
-func (self *modEngine) LoadMessage(msgid string) NNTPMessage {
-	return self.store.GetMessage(msgid)
+func (self *modEngine) MarkSpam(msgid string) (err error) {
+	if self.spam == nil {
+		err = self.store.MarkSpam(msgid)
+	} else {
+		var f io.ReadCloser
+		f, err = self.store.OpenMessage(msgid)
+		if err == nil {
+			err = self.spam.MarkSpam(f)
+			f.Close()
+		}
+	}
+	return
 }
 
 func (self *modEngine) BanAddress(cidr string) (err error) {
@@ -311,22 +339,23 @@ func (self *modEngine) AllowDelete(pubkey, msgid string) (allow bool) {
 }
 
 func (mod *modEngine) HandleMessage(msgid string) {
-	nntp := mod.store.GetMessage(msgid)
-	if nntp == nil {
-		log.Println("failed to load", msgid, "in mod engine, missing message")
-		return
-	}
-	// sanity check
-	if nntp.Newsgroup() == "ctl" {
-		pubkey := nntp.Pubkey()
-		for _, line := range strings.Split(nntp.Message(), "\n") {
-			line = strings.Trim(line, "\r\t\n ")
-			if len(line) > 0 {
-				ev := ParseModEvent(line)
-				mod.Execute(ev, pubkey)
+	mod.store.GetMessage(msgid, func(nntp NNTPMessage) {
+		if nntp == nil {
+			log.Println("failed to load", msgid, "in mod engine, missing message")
+			return
+		}
+		// sanity check
+		if nntp.Newsgroup() == "ctl" {
+			pubkey := nntp.Pubkey()
+			for _, line := range strings.Split(nntp.Message(), "\n") {
+				line = strings.Trim(line, "\r\t\n ")
+				if len(line) > 0 {
+					ev := ParseModEvent(line)
+					mod.Execute(ev, pubkey)
+				}
 			}
 		}
-	}
+	})
 }
 
 func (mod *modEngine) Do(ev ModEvent) {
@@ -394,6 +423,10 @@ func (mod *modEngine) Do(ev ModEvent) {
 		} else {
 			log.Printf("invalid overchan-inet-ban: target=%s", target)
 		}
+	} else if action == ModSpam {
+		if ValidMessageID(target) {
+			mod.MarkSpam(target)
+		}
 	} else if action == ModHide {
 		// TODO: implement
 	} else if action == ModLock {
@@ -401,7 +434,20 @@ func (mod *modEngine) Do(ev ModEvent) {
 	} else if action == ModSage {
 		// TODO: implement
 	} else if action == ModStick {
-		// TODO: implement
+		// TODP: implement
+	} else if action == ModRemove {
+		if ValidMessageID(target) {
+			err := mod.database.RemoveArticle(target)
+			if err != nil {
+				log.Println("failed to forget", target, "because:", err)
+			}
+			err = mod.store.Remove(target)
+			if err == nil {
+				log.Println("removed", target)
+			} else {
+				log.Println(action, target, "failed:", err)
+			}
+		}
 	} else if action == ModRemoveAttachment {
 		var delfiles []string
 		atts := mod.database.GetPostAttachments(target)
@@ -434,6 +480,11 @@ func (mod *modEngine) Execute(ev ModEvent, pubkey string) {
 			mod.Do(ev)
 		}
 		return
+	case ModSpam:
+		if mod.AllowJanitor(pubkey) {
+			mod.Do(ev)
+		}
+		return
 	case ModHide:
 	case ModLock:
 	case ModSage:
@@ -442,6 +493,7 @@ func (mod *modEngine) Execute(ev ModEvent, pubkey string) {
 		if mod.AllowJanitor(pubkey) {
 			mod.Do(ev)
 		}
+		return
 	default:
 		// invalid action
 	}

contrib/backends/srndv2/src/srnd/mod_http.go

@@ -35,8 +35,8 @@ type httpModUI struct {
 	cache            CacheHandler
 }
 
-func createHttpModUI(frontend *httpFrontend) httpModUI {
-	return httpModUI{frontend.regenAll, frontend.Regen, frontend.RegenerateBoard, frontend.deleteThreadMarkup, frontend.deleteBoardMarkup, make(chan NNTPMessage), frontend.daemon, frontend.daemon.store, frontend.store, frontend.prefix, frontend.prefix + "mod/", frontend.GetCacheHandler()}
+func createHttpModUI(frontend *httpFrontend) *httpModUI {
+	return &httpModUI{frontend.regenAll, frontend.Regen, frontend.RegenerateBoard, frontend.deleteThreadMarkup, frontend.deleteBoardMarkup, make(chan NNTPMessage), frontend.daemon, frontend.daemon.store, frontend.store, frontend.prefix, frontend.prefix + "mod/", frontend.GetCacheHandler()}
 
 }
 
@@ -44,7 +44,7 @@ func extractGroup(param map[string]interface{}) string {
 	return extractParam(param, "newsgroup")
 }
 
-func (self httpModUI) getAdminFunc(funcname string) AdminFunc {
+func (self *httpModUI) getAdminFunc(funcname string) AdminFunc {
 	if funcname == "template.reload" {
 		return func(param map[string]interface{}) (interface{}, error) {
 			tname, ok := param["template"]
@@ -390,7 +390,7 @@ func (self httpModUI) HandleAdminCommand(wr http.ResponseWriter, r *http.Request
 	}, wr, r)
 }
 
-func (self httpModUI) CheckPubkey(pubkey, scope string) (bool, error) {
+func (self *httpModUI) CheckPubkey(pubkey, scope string) (bool, error) {
 	is_admin, err := self.daemon.database.CheckAdminPubkey(pubkey)
 	if is_admin {
 		// admin can do what they want
@@ -413,7 +413,7 @@ func (self httpModUI) CheckPubkey(pubkey, scope string) (bool, error) {
 	return false, err
 }
 
-func (self httpModUI) CheckKey(privkey, scope string) (bool, error) {
+func (self *httpModUI) CheckKey(privkey, scope string) (bool, error) {
 	privkey_bytes, err := hex.DecodeString(privkey)
 	if err == nil {
 		pk, _ := naclSeedToKeyPair(privkey_bytes)
@@ -424,17 +424,17 @@ func (self httpModUI) CheckKey(privkey, scope string) (bool, error) {
 	return false, err
 }
 
-func (self httpModUI) MessageChan() chan NNTPMessage {
+func (self *httpModUI) MessageChan() chan NNTPMessage {
 	return self.modMessageChan
 }
 
-func (self httpModUI) getSession(r *http.Request) *sessions.Session {
+func (self *httpModUI) getSession(r *http.Request) *sessions.Session {
 	s, _ := self.store.Get(r, "nntpchan-mod")
 	return s
 }
 
 // get the session's private key as bytes or nil if we don't have it
-func (self httpModUI) getSessionPrivkeyBytes(r *http.Request) []byte {
+func (self *httpModUI) getSessionPrivkeyBytes(r *http.Request) []byte {
 	s := self.getSession(r)
 	k, ok := s.Values["privkey"]
 	if ok {
@@ -451,7 +451,7 @@ func (self httpModUI) getSessionPrivkeyBytes(r *http.Request) []byte {
 
 // returns true if the session is okay for a scope
 // otherwise redirect to login page
-func (self httpModUI) checkSession(r *http.Request, scope string) bool {
+func (self *httpModUI) checkSession(r *http.Request, scope string) bool {
 	s := self.getSession(r)
 	k, ok := s.Values["privkey"]
 	if ok {
@@ -464,11 +464,11 @@ func (self httpModUI) checkSession(r *http.Request, scope string) bool {
 	return false
 }
 
-func (self httpModUI) writeTemplate(wr http.ResponseWriter, r *http.Request, name string) {
+func (self *httpModUI) writeTemplate(wr http.ResponseWriter, r *http.Request, name string) {
 	self.writeTemplateParam(wr, r, name, nil)
 }
 
-func (self httpModUI) writeTemplateParam(wr http.ResponseWriter, r *http.Request, name string, param map[string]interface{}) {
+func (self *httpModUI) writeTemplateParam(wr http.ResponseWriter, r *http.Request, name string, param map[string]interface{}) {
 	if param == nil {
 		param = make(map[string]interface{})
 	}
@@ -481,7 +481,7 @@ func (self httpModUI) writeTemplateParam(wr http.ResponseWriter, r *http.Request
 
 // do a function as authenticated
 // pass in the request path to the handler
-func (self httpModUI) asAuthed(scope string, handler func(string), wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) asAuthed(scope string, handler func(string), wr http.ResponseWriter, r *http.Request) {
 	if self.checkSession(r, scope) {
 		handler(r.URL.Path)
 	} else {
@@ -490,7 +490,7 @@ func (self httpModUI) asAuthed(scope string, handler func(string), wr http.Respo
 }
 
 // do stuff to a certain message if with have it and are authed
-func (self httpModUI) asAuthedWithMessage(scope string, handler func(ArticleEntry, *http.Request) map[string]interface{}, wr http.ResponseWriter, req *http.Request) {
+func (self *httpModUI) asAuthedWithMessage(scope string, handler func(ArticleEntry, *http.Request) map[string]interface{}, wr http.ResponseWriter, req *http.Request) {
 	self.asAuthed(scope, func(path string) {
 		// get the long hash
 		if strings.Count(path, "/") > 2 {
@@ -523,13 +523,47 @@ func (self httpModUI) asAuthedWithMessage(scope string, handler func(ArticleEntr
 	}, wr, req)
 }
 
-func (self httpModUI) HandleAddPubkey(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandlePostSpam(wr http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		wr.WriteHeader(405)
+		return
+	}
+	resp := make(map[string]interface{})
+	self.asAuthed("spam", func(path string) {
+		var mm ModMessage
+		var err error
+		keys := strings.Split(r.FormValue("spam"), ",")
+		for _, k := range keys {
+			k =  strings.TrimSpace(k)
+			go self.daemon.MarkSpam(k)
+			mm = append(mm, modMarkSpam(k)) 
+		}
+		privkey_bytes := self.getSessionPrivkeyBytes(r)
+		if privkey_bytes == nil {
+			// this should not happen
+			log.Println("failed to get privkey bytes from session")
+			resp["error"] = "failed to get private key from session. wtf?"
+		} else {
+			// wrap and sign
+			nntp := wrapModMessage(mm)
+			nntp, err = signArticle(nntp, privkey_bytes)
+			if err == nil {
+				// federate
+				self.modMessageChan <- nntp
+			}
+			resp["error"] = err
+		}
+	}, wr, r)
+	json.NewEncoder(wr).Encode(resp)
 }
 
-func (self httpModUI) HandleDelPubkey(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleAddPubkey(wr http.ResponseWriter, r *http.Request) {
 }
 
-func (self httpModUI) HandleUnbanAddress(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleDelPubkey(wr http.ResponseWriter, r *http.Request) {
+}
+
+func (self *httpModUI) HandleUnbanAddress(wr http.ResponseWriter, r *http.Request) {
 	self.asAuthed("ban", func(path string) {
 		// extract the ip address
 		// TODO: ip ranges and prefix detection
@@ -559,7 +593,7 @@ func (self httpModUI) HandleUnbanAddress(wr http.ResponseWriter, r *http.Request
 }
 
 // handle ban logic
-func (self httpModUI) handleBanAddress(msg ArticleEntry, r *http.Request) map[string]interface{} {
+func (self *httpModUI) handleBanAddress(msg ArticleEntry, r *http.Request) map[string]interface{} {
 	// get the article headers
 	resp := make(map[string]interface{})
 	msgid := msg.MessageID()
@@ -623,7 +657,7 @@ func (self httpModUI) handleBanAddress(msg ArticleEntry, r *http.Request) map[st
 	return resp
 }
 
-func (self httpModUI) handleDeletePost(msg ArticleEntry, r *http.Request) map[string]interface{} {
+func (self *httpModUI) handleDeletePost(msg ArticleEntry, r *http.Request) map[string]interface{} {
 	var mm ModMessage
 	resp := make(map[string]interface{})
 	msgid := msg.MessageID()
@@ -672,16 +706,16 @@ func (self httpModUI) handleDeletePost(msg ArticleEntry, r *http.Request) map[st
 }
 
 // ban the address of a poster
-func (self httpModUI) HandleBanAddress(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleBanAddress(wr http.ResponseWriter, r *http.Request) {
 	self.asAuthedWithMessage("ban", self.handleBanAddress, wr, r)
 }
 
 // delete a post
-func (self httpModUI) HandleDeletePost(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleDeletePost(wr http.ResponseWriter, r *http.Request) {
 	self.asAuthedWithMessage("login", self.handleDeletePost, wr, r)
 }
 
-func (self httpModUI) HandleLogin(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleLogin(wr http.ResponseWriter, r *http.Request) {
 	privkey := r.FormValue("privkey")
 	msg := "failed login: "
 	if len(privkey) == 0 {
@@ -702,13 +736,13 @@ func (self httpModUI) HandleLogin(wr http.ResponseWriter, r *http.Request) {
 	self.writeTemplateParam(wr, r, "modlogin_result", map[string]interface{}{"message": msg, csrf.TemplateTag: csrf.TemplateField(r)})
 }
 
-func (self httpModUI) HandleKeyGen(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleKeyGen(wr http.ResponseWriter, r *http.Request) {
 	pk, sk := newNaclSignKeypair()
 	tripcode := makeTripcode(pk)
 	self.writeTemplateParam(wr, r, "keygen", map[string]interface{}{"public": pk, "secret": sk, "tripcode": tripcode})
 }
 
-func (self httpModUI) ServeModPage(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) ServeModPage(wr http.ResponseWriter, r *http.Request) {
 	if self.checkSession(r, "login") {
 		wr.Header().Set("X-CSRF-Token", csrf.Token(r))
 		// we are logged in

contrib/backends/srndv2/src/srnd/model.go

@@ -11,6 +11,9 @@ import (
 // base model type
 type BaseModel interface {
 
+	// set sfw flag
+	MarkSFW(sfw bool)
+
 	// site url prefix
 	Prefix() string
 
@@ -44,8 +47,9 @@ type AttachmentModel interface {
 type PostModel interface {
 	BaseModel
 
+	Brief() string
 	CSSClass() string
-
+	FrontendPubkey() string
 	MessageID() string
 	PostHash() string
 	ShortHash() string
@@ -77,6 +81,9 @@ type PostModel interface {
 	// returns true if this post was truncated
 	IsTruncated() bool
 
+	// return true if this post is a mod message
+	IsCtl() bool
+
 	IsI2P() bool
 	IsTor() bool
 	IsClearnet() bool
@@ -137,6 +144,8 @@ type ThreadModel interface {
 	IsDirty() bool
 	// mark thread as dirty
 	MarkDirty()
+	// is the threa bumplocked?
+	BumpLock() bool
 }
 
 // board interface
@@ -183,6 +192,7 @@ type CatalogItemModel interface {
 	OP() PostModel
 	ReplyCount() string
 	Page() string
+	MarkSFW(sfw bool)
 }
 
 type LinkModel interface {
@@ -217,6 +227,8 @@ type boardPageRow struct {
 	Hour  int64
 	Day   int64
 	All   int64
+	Hi    int64
+	Lo    int64
 }
 
 type boardPageRows []boardPageRow

contrib/backends/srndv2/src/srnd/model_mem.go

@@ -15,6 +15,7 @@ import (
 )
 
 type catalogModel struct {
+	SFW      bool
 	frontend string
 	prefix   string
 	board    string
@@ -46,6 +47,13 @@ func (self *thread) I18N(i *I18N) {
 	}
 }
 
+func (self *boardModel) MarkSFW(sfw bool) {
+	for idx := range self.threads {
+		self.threads[idx].MarkSFW(sfw)
+	}
+	self.SFW = sfw
+}
+
 func (self *boardModel) I18N(i *I18N) {
 	self._i18n = i
 	for idx := range self.threads {
@@ -57,13 +65,24 @@ func (self *attachment) I18N(i *I18N) {
 	self._i18n = i
 }
 
+func (self *catalogModel) MarkSFW(sfw bool) {
+	for idx := range self.threads {
+		self.threads[idx].MarkSFW(sfw)
+	}
+	self.SFW = sfw
+}
+
 func (self *catalogModel) Navbar() string {
 	param := make(map[string]interface{})
 	param["name"] = fmt.Sprintf("Catalog for %s", self.board)
 	param["frontend"] = self.frontend
 	var links []LinkModel
+	var sfw int
+	if self.SFW {
+		sfw = 1
+	}
 	links = append(links, linkModel{
-		link: fmt.Sprintf("%sb/%s/?lang=%s", self.prefix, self.board, self._i18n.Name),
+		link: fmt.Sprintf("%sb/%s/?lang=%s&sfw=%d", self.prefix, self.board, self._i18n.Name, sfw),
 		text: "Board index",
 	})
 	param["prefix"] = self.prefix
@@ -99,6 +118,10 @@ func (self *catalogItemModel) OP() PostModel {
 	return self.op
 }
 
+func (self *catalogItemModel) MarkSFW(sfw bool) {
+	self.op.MarkSFW(sfw)
+}
+
 func (self *catalogItemModel) Page() string {
 	return strconv.Itoa(self.page)
 }
@@ -115,6 +138,7 @@ type boardModel struct {
 	board      string
 	page       int
 	pages      int
+	SFW        bool
 	threads    []ThreadModel
 }
 
@@ -176,6 +200,9 @@ func (self *boardModel) PageList() []LinkModel {
 		if i == 0 {
 			board = fmt.Sprintf("%sb/%s/?lang=%s", self.prefix, self.board, self._i18n.Name)
 		}
+		if self.SFW {
+			board += "&sfw=1"
+		}
 		links = append(links, linkModel{
 			link: board,
 			text: fmt.Sprintf("[ %d ]", i),
@@ -238,34 +265,41 @@ func (self *boardModel) Update(db Database) {
 }
 
 type post struct {
-	_i18n            *I18N
-	truncated        bool
-	prefix           string
-	board            string
-	PostName         string
-	PostSubject      string
-	PostMessage      string
-	message_rendered string
-	Message_id       string
-	MessagePath      string
-	addr             string
-	Newsgroup        string
-	op               bool
-	Posted           int64
-	Parent           string
-	sage             bool
-	Key              string
-	Files            []AttachmentModel
-	HashLong         string
-	HashShort        string
-	URL              string
-	Tripcode         string
-	BodyMarkup       string
-	PostMarkup       string
-	PostPrefix       string
-	index            int
-	Type             string
-	nntp_id          int
+	_i18n             *I18N
+	SFW               bool
+	truncated         bool
+	prefix            string
+	board             string
+	PostName          string
+	PostSubject       string
+	PostMessage       string
+	message_rendered  string
+	Message_id        string
+	MessagePath       string
+	addr              string
+	Newsgroup         string
+	op                bool
+	Posted            int64
+	Parent            string
+	sage              bool
+	Key               string
+	Files             []AttachmentModel
+	HashLong          string
+	HashShort         string
+	URL               string
+	Tripcode          string
+	BodyMarkup        string
+	PostMarkup        string
+	PostPrefix        string
+	index             int
+	Type              string
+	nntp_id           int
+	FrontendPublicKey string
+	ReferencedURI     string
+}
+
+func (p *post) IsCtl() bool {
+	return p.board == "ctl"
 }
 
 func (self *post) NNTPID() int {
@@ -276,6 +310,13 @@ func (self *post) Index() int {
 	return self.index + 1
 }
 
+func (self *post) Brief() string {
+	if len(self.PostMessage) > 140 {
+		return self.PostMessage[:140]
+	}
+	return self.PostMessage
+}
+
 func (self *post) NumImages() int {
 	return len(self.Files)
 }
@@ -321,6 +362,7 @@ type attachment struct {
 	Name        string
 	ThumbWidth  int
 	ThumbHeight int
+	SFW         bool
 }
 
 func (self *attachment) MarshalJSON() (b []byte, err error) {
@@ -340,6 +382,10 @@ func (self *attachment) Hash() string {
 	return strings.Split(self.Path, ".")[0]
 }
 
+func (self *attachment) MarkSFW(sfw bool) {
+	self.SFW = sfw
+}
+
 func (self *attachment) ThumbInfo() ThumbInfo {
 	return ThumbInfo{
 		Width:  self.ThumbWidth,
@@ -352,6 +398,9 @@ func (self *attachment) Prefix() string {
 }
 
 func (self *attachment) Thumbnail() string {
+	if self.SFW {
+		return self.prefix + "static/placeholder.png"
+	}
 	return self.prefix + "thm/" + self.Path + ".jpg"
 }
 
@@ -363,7 +412,7 @@ func (self *attachment) Filename() string {
 	return self.Name
 }
 
-func PostModelFromMessage(parent, prefix string, nntp NNTPMessage) PostModel {
+func PostModelFromMessage(prefix string, nntp NNTPMessage) PostModel {
 	p := new(post)
 	p.PostName = nntp.Name()
 	p.PostSubject = nntp.Subject()
@@ -374,10 +423,12 @@ func PostModelFromMessage(parent, prefix string, nntp NNTPMessage) PostModel {
 	p.Posted = nntp.Posted()
 	p.op = nntp.OP()
 	p.prefix = prefix
-	p.Parent = parent
+	p.Parent = nntp.Reference()
 	p.addr = nntp.Addr()
 	p.sage = nntp.Sage()
 	p.Key = nntp.Pubkey()
+	p.ReferencedURI = nntp.Headers().Get("X-References-Uri", "")
+	p.FrontendPublicKey = nntp.FrontendPubkey()
 	for _, att := range nntp.Attachments() {
 		p.Files = append(p.Files, att.ToModel(prefix))
 	}
@@ -407,10 +458,21 @@ func (self *post) ShortHash() string {
 	return ShortHashMessageID(self.MessageID())
 }
 
+func (self *post) MarkSFW(sfw bool) {
+	self.SFW = sfw
+	for idx := range self.Files {
+		self.Files[idx].MarkSFW(sfw)
+	}
+}
+
 func (self *post) PubkeyHex() string {
 	return self.Key
 }
 
+func (self *post) FrontendPubkey() string {
+	return self.FrontendPublicKey
+}
+
 func (self *post) Pubkey() string {
 	if len(self.Key) > 0 {
 		return fmt.Sprintf("<label title=\"%s\">%s</label>", self.Key, makeTripcode(self.Key))
@@ -487,7 +549,12 @@ func (self *post) PostURL() string {
 	if i18n == nil {
 		i18n = I18nProvider
 	}
-	return fmt.Sprintf("%st/%s/?lang=%s#%s", self.Prefix(), HashMessageID(self.Parent), i18n.Name, self.PostHash())
+	u := fmt.Sprintf("%st/%s/?lang=%s", self.Prefix(), HashMessageID(self.Parent), i18n.Name)
+	if self.SFW {
+		u += "&sfw=1"
+	}
+	u += "#" + self.PostHash()
+	return u
 }
 
 func (self *post) Prefix() string {
@@ -558,8 +625,11 @@ func (self *post) Truncate() PostModel {
 		Parent:      self.Parent,
 		sage:        self.sage,
 		Key:         self.Key,
+		SFW:         self.SFW,
 		// TODO: copy?
-		Files: self.Files,
+		Files:             self.Files,
+		FrontendPublicKey: self.FrontendPublicKey,
+		ReferencedURI:     self.ReferencedURI,
 	}
 }
 
@@ -583,13 +653,21 @@ type thread struct {
 	_i18n               *I18N
 	allowFiles          bool
 	prefix              string
-	links               []LinkModel
+	links               []linkModel
 	Posts               []PostModel
+	SFW                 bool
 	dirty               bool
 	truncatedPostCount  int
 	truncatedImageCount int
 }
 
+func (self *thread) MarkSFW(sfw bool) {
+	for idx := range self.Posts {
+		self.Posts[idx].MarkSFW(sfw)
+	}
+	self.SFW = sfw
+}
+
 func (self *thread) MarshalJSON() (b []byte, err error) {
 	posts := []PostModel{self.OP()}
 	posts = append(posts, self.Replies()...)
@@ -621,6 +699,11 @@ func (self *thread) Navbar() string {
 	param := make(map[string]interface{})
 	param["name"] = fmt.Sprintf("Thread %s", self.Posts[0].ShortHash())
 	param["frontend"] = self.Board()
+
+	for idx := range self.links {
+		self.links[idx].link += "?sfw=1"
+	}
+
 	param["links"] = self.links
 	param["prefix"] = self.prefix
 	return template.renderTemplate("navbar", param, self._i18n)
@@ -635,7 +718,11 @@ func (self *thread) BoardURL() string {
 	if i18n == nil {
 		i18n = I18nProvider
 	}
-	return fmt.Sprintf("%sb/%s/?lang=%s", self.Prefix(), self.Board(), i18n.Name)
+	u := fmt.Sprintf("%sb/%s/?lang=%s", self.Prefix(), self.Board(), i18n.Name)
+	if self.SFW {
+		u += "&sfw=1"
+	}
+	return u
 }
 
 func (self *thread) PostCount() int {
@@ -657,7 +744,7 @@ func createThreadModel(posts ...PostModel) ThreadModel {
 		dirty:  true,
 		prefix: prefix,
 		Posts:  posts,
-		links: []LinkModel{
+		links: []linkModel{
 			linkModel{
 				text: group,
 				link: fmt.Sprintf("%sb/%s/", prefix, group),
@@ -677,6 +764,7 @@ func (self *thread) Replies() []PostModel {
 		for idx, post := range self.Posts[1:] {
 			if post != nil {
 				post.SetIndex(idx + 1)
+				post.MarkSFW(self.SFW)
 				replies = append(replies, post)
 			}
 		}
@@ -708,6 +796,7 @@ func (self *thread) Truncate() ThreadModel {
 		for _, p := range t.Posts {
 			imgs += p.NumAttachments()
 		}
+		t.SFW = self.SFW
 		t.truncatedPostCount = len(self.Posts) - trunc
 		t.truncatedImageCount = self.ImageCount() - imgs
 		return t
@@ -731,9 +820,17 @@ func (self *thread) HasOmittedImages() bool {
 	return self.truncatedImageCount > 0
 }
 
+func (self *thread) BumpLock() bool {
+	if self.Posts == nil {
+		return false
+	}
+	return len(self.Posts) >= BumpLimit
+}
+
 func (self *thread) Update(db Database) {
 	root := self.Posts[0].MessageID()
 	self.Posts = append([]PostModel{self.Posts[0]}, db.GetThreadReplyPostModels(self.prefix, root, 0, 0)...)
+	self.MarkSFW(self.SFW)
 	self.dirty = false
 }
 

contrib/backends/srndv2/src/srnd/nntp.go

@@ -382,18 +382,19 @@ func (self *nntpConnection) handleStreaming(daemon *NNTPDaemon, conn *textproto.
 	return
 }
 
-// check if we want the article given its mime header
+// check if we want the article given its auth status and mime header
 // returns empty string if it's okay otherwise an error message
-func (self *nntpConnection) checkMIMEHeader(daemon *NNTPDaemon, h map[string][]string) (reason string, allow bool, err error) {
+func (self *nntpConnection) checkMIMEHeader(daemon *NNTPDaemon, hdr textproto.MIMEHeader) (reason string, allow bool, err error) {
 	if !self.authenticated {
 		reason = "not authenticated"
 		return
 	}
-	hdr := textproto.MIMEHeader(h)
 	reason, allow, err = self.checkMIMEHeaderNoAuth(daemon, hdr)
 	return
 }
 
+// check if we want the article given its mime header without checking auth status
+// returns empty string if it's okay otherwise an error message
 func (self *nntpConnection) checkMIMEHeaderNoAuth(daemon *NNTPDaemon, hdr textproto.MIMEHeader) (reason string, ban bool, err error) {
 	newsgroup := hdr.Get("Newsgroups")
 	reference := hdr.Get("References")
@@ -421,7 +422,7 @@ func (self *nntpConnection) checkMIMEHeaderNoAuth(daemon *NNTPDaemon, hdr textpr
 	}
 
 	if serverPubkeyIsValid(server_pubkey) {
-		b, _ := daemon.database.PubkeyIsBanned(server_pubkey)
+		b, _ := daemon.database.PubkeyRejected(server_pubkey)
 		if b {
 			reason = "server's pubkey is banned"
 			ban = true
@@ -447,7 +448,7 @@ func (self *nntpConnection) checkMIMEHeaderNoAuth(daemon *NNTPDaemon, hdr textpr
 		reason = "newsgroup banned"
 		ban = true
 		return
-	} else if banned, _ = daemon.database.PubkeyIsBanned(pubkey); banned {
+	} else if banned, _ = daemon.database.PubkeyRejected(pubkey); banned {
 		// check for banned pubkey
 		reason = "poster's pubkey is banned"
 		ban = true
@@ -664,9 +665,9 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 			} else if cmd == "CHECK" {
 				// handle check command
 				msgid := parts[1]
-				if self.mode != "STREAM" {
-					// we can't we are not in streaming mode
-					conn.PrintfLine("431 %s", msgid)
+				if !self.authenticated {
+					// if client cannot TAKETHIS it shouldn't be able to CHECK either
+					conn.PrintfLine("480 You have not authenticated")
 					return
 				}
 				// have we seen this article?
@@ -682,55 +683,73 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 				}
 			} else if cmd == "TAKETHIS" {
 				// handle takethis command
+				r := bufio.NewReader(conn.DotReader())
+				if !self.authenticated {
+					// early reject without parsing incase client not allowed to post
+					// send response first to allow client to stop sending
+					// XXX what happens if our send queue is full and this blocks?
+					// other side will probably fill up sending article to us
+					// async receive processing would help there
+					// but this situation has low likehood to happen
+					// operating system/transport buffering will compensate
+					// so leave it this way
+					conn.PrintfLine("480 You have not authenticated")
+					// discard whole article without looking at insides
+					// it should be dot-terminated either way
+					_, err = io.Copy(ioutil.Discard, r)
+					return
+				}
+				// client is allowed to post
 				var msg *mail.Message
 				var reason string
 				var ban bool
 				// read the article header
-				r := bufio.NewReader(conn.DotReader())
 				msg, err = readMIMEHeader(r)
-				if err == nil {
-					hdr := textproto.MIMEHeader(msg.Header)
-					// check the header
-					reason, ban, err = self.checkMIMEHeader(daemon, hdr)
-					if len(reason) > 0 {
-						// discard, we do not want
-						code = 439
-						log.Println(self.name, "rejected", msgid, reason)
-						_, err = io.Copy(ioutil.Discard, msg.Body)
-						if ban {
-							err = daemon.database.BanArticle(msgid, reason)
-						}
-					} else if err == nil {
-						// check if we don't have the rootpost
-						reference := hdr.Get("References")
-						if reference != "" && ValidMessageID(reference) && !daemon.store.HasArticle(reference) && !daemon.database.IsExpired(reference) {
-							log.Println(self.name, "got reply to", reference, "but we don't have it")
-							go daemon.askForArticle(reference)
-						}
-						err = storeMessage(daemon, hdr, msg.Body)
-						if err == nil {
-							code = 239
-							reason = "gotten"
-						} else {
-							code = 439
-							reason = err.Error()
-						}
-					} else {
-						// error?
-						// discard, we do not want
-						code = 439
-						log.Println(self.name, "rejected", msgid, reason)
-						_, err = io.Copy(ioutil.Discard, msg.Body)
-						if ban {
-							err = daemon.database.BanArticle(msgid, reason)
-						}
-					}
-				} else {
+				if err != nil {
 					log.Println(self.name, "error reading mime header:", err)
-					code = 439
-					reason = "error reading mime header"
+					conn.PrintfLine("439 %s error reading mime header", msgid)
+					// if reading header error'd, msg.Body wont be set
+					_, err = io.Copy(ioutil.Discard, r)
+					return
+				}
+				hdr := textproto.MIMEHeader(msg.Header)
+				// check the header
+				reason, ban, err = self.checkMIMEHeaderNoAuth(daemon, hdr)
+				if len(reason) > 0 {
+					// discard, we do not want
+					log.Println(self.name, "rejected", msgid, reason)
+					conn.PrintfLine("439 %s %s", msgid, reason)
+					_, err = io.Copy(ioutil.Discard, msg.Body)
+					if ban {
+						err = daemon.database.BanArticle(msgid, reason)
+					}
+				} else if err == nil {
+					// looks good to accept
+					// check if we don't have the rootpost
+					reference := hdr.Get("References")
+					if reference != "" && ValidMessageID(reference) && !daemon.store.HasArticle(reference) && !daemon.database.IsExpired(reference) {
+						log.Println(self.name, "got reply to", reference, "but we don't have it")
+						go daemon.askForArticle(reference)
+					}
+					err = storeMessage(daemon, hdr, msg.Body)
+					if err == nil {
+						code = 239
+						reason = "gotten"
+					} else {
+						code = 439
+						reason = err.Error()
+					}
+					conn.PrintfLine("%d %s %s", code, msgid, reason)
+				} else {
+					// error?
+					// discard, we do not want
+					log.Println(self.name, "rejected", msgid, "unexpected error", err)
+					conn.PrintfLine("439 %s unexpected error", msgid)
+					_, err = io.Copy(ioutil.Discard, msg.Body)
+					if ban {
+						err = daemon.database.BanArticle(msgid, reason)
+					}
 				}
-				conn.PrintfLine("%d %s %s", code, msgid, reason)
 			} else if cmd == "ARTICLE" {
 				if !ValidMessageID(msgid) {
 					if len(self.group) > 0 {
@@ -740,26 +759,34 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 						}
 					}
 				}
-				if ValidMessageID(msgid) && daemon.store.HasArticle(msgid) {
-					// we have it yeh
-					f, err := daemon.store.OpenMessage(msgid)
-					if err == nil {
-						conn.PrintfLine("220 %s", msgid)
-						dw := conn.DotWriter()
-						_, err = io.Copy(dw, f)
-						dw.Close()
-						f.Close()
+				if ValidMessageID(msgid) {
+					if daemon.database.ArticleBanned(msgid) {
+						// article banned
+						conn.PrintfLine("439 %s article banned from server", msgid)
+					} else if daemon.store.HasArticle(msgid) {
+						// we have it yeh
+						f, err := daemon.store.OpenMessage(msgid)
+						if err == nil {
+							conn.PrintfLine("220 %s", msgid)
+							dw := conn.DotWriter()
+							_, err = io.Copy(dw, f)
+							dw.Close()
+							f.Close()
+						} else {
+							// wtf?!
+							conn.PrintfLine("503 idkwtf happened: %s", err.Error())
+						}
 					} else {
-						// wtf?!
-						conn.PrintfLine("503 idkwtf happened: %s", err.Error())
+						// we dont got it (by msgid)
+						conn.PrintfLine("430 %s", msgid)
 					}
 				} else {
-					// we dont got it
-					conn.PrintfLine("430 %s", msgid)
+					// we dont got it (by num)
+					conn.PrintfLine("423 %s", msgid)
 				}
 			} else if cmd == "IHAVE" {
 				if !self.authenticated {
-					conn.PrintfLine("483 You have not authenticated")
+					conn.PrintfLine("480 You have not authenticated")
 				} else {
 					// handle IHAVE command
 					msgid := parts[1]
@@ -887,7 +914,25 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 						for _, model := range models {
 							if model != nil {
 								if err == nil {
-									io.WriteString(dw, fmt.Sprintf("%.6d\t%s\t\"%s\" <%s@%s>\t%s\t%s\t%s\r\n", model.NNTPID(), model.Subject(), model.Name(), model.Name(), model.Frontend(), model.Date(), model.MessageID(), model.Reference()))
+									/*
+										The first 8 fields MUST be the following, in order:
+										     "0" or article number (see below)
+										     Subject header content
+										     From header content
+										     Date header content
+										     Message-ID header content
+										     References header content
+										     :bytes metadata item
+										     :lines metadata item
+									*/
+									fmt.Fprintf(dw,
+										"%.6d\t%s\t\"%s\" <%s@%s>\t%s\t%s\t%s\r\n",
+										model.NNTPID(),
+										safeHeader(model.Subject()),
+										safeHeader(model.Name()), safeHeader(model.Name()), safeHeader(model.Frontend()),
+										safeHeader(model.Date()),
+										safeHeader(model.MessageID()),
+										safeHeader(model.Reference()))
 								}
 							}
 						}
@@ -1151,7 +1196,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 					dw := conn.DotWriter()
 					for _, entry := range list {
 						if ValidNewsgroup(entry[0]) {
-							io.WriteString(dw, fmt.Sprintf("%s %s %s y\r\n", entry[0], entry[1], entry[2]))
+							io.WriteString(dw, fmt.Sprintf("%s %s %s y\r\n", entry[0], entry[2], entry[1]))
 						}
 					}
 					dw.Close()
@@ -1195,10 +1240,14 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 										log.Println(self.name, "got reply to", reference, "but we don't have it")
 										go daemon.askForArticle(reference)
 									} else {
-										h := daemon.store.GetMIMEHeader(reference)
-										if strings.Trim(h.Get("References"), " ") == "" {
-											hdr.Set("References", getMessageID(h))
+										// get rootiest post
+										ref := reference
+										var h textproto.MIMEHeader	
+										for ref != "" {
+											h = daemon.store.GetMIMEHeader(ref)
+											ref = strings.Trim(h.Get("References"), " ")
 										}
+										hdr.Set("References", getMessageID(h))
 									}
 								} else if reference != "" {
 									// bad message id
@@ -1246,6 +1295,48 @@ func (self *nntpConnection) startStreaming(daemon *NNTPDaemon, reader bool, conn
 	}
 }
 
+// interprets result string from response code 211 of GROUP message
+// for parts it fails, returns zeros
+func interpretGroupResult(line string) (es, lo, hi uint64, group string) {
+	// this code is braindead but I was lazy to search stdlib
+	startes := 0
+	for startes < len(line) && (line[startes] == ' ' || line[startes] == '\t') {
+		startes++
+	}
+	endes := startes
+	for endes < len(line) && line[endes] != ' ' && line[endes] != '\t' {
+		endes++
+	}
+	startlo := endes
+	for startlo < len(line) && (line[startlo] == ' ' || line[startlo] == '\t') {
+		startlo++
+	}
+	endlo := startlo
+	for endlo < len(line) && line[endlo] != ' ' && line[endlo] != '\t' {
+		endlo++
+	}
+	starthi := endlo
+	for starthi < len(line) && (line[starthi] == ' ' || line[starthi] == '\t') {
+		starthi++
+	}
+	endhi := starthi
+	for endhi < len(line) && line[endhi] != ' ' && line[endhi] != '\t' {
+		endhi++
+	}
+	startgroup := endhi
+	for startgroup < len(line) && (line[startgroup] == ' ' || line[startgroup] == '\t') {
+		startgroup++
+	}
+	// will return 0 if failed to parse. which is OK for us
+	es, _ = strconv.ParseUint(line[startes:endes], 10, 64)
+	lo, _ = strconv.ParseUint(line[startlo:endlo], 10, 64)
+	hi, _ = strconv.ParseUint(line[starthi:endhi], 10, 64)
+	group = line[startgroup:]
+	return
+}
+
+const maxXOVERRange = 800
+
 // scrape all posts in a newsgroup
 // download ones we do not have
 func (self *nntpConnection) scrapeGroup(daemon *NNTPDaemon, conn *textproto.Conn, group string) (err error) {
@@ -1255,14 +1346,46 @@ func (self *nntpConnection) scrapeGroup(daemon *NNTPDaemon, conn *textproto.Conn
 	err = conn.PrintfLine("GROUP %s", group)
 	if err == nil {
 		// read reply to GROUP command
-		code := 0
-		code, _, err = conn.ReadCodeLine(211)
+		var code int
+		var ret string
+		code, ret, err = conn.ReadCodeLine(211)
 		// check code
 		if code == 211 {
 			// success
-			// send XOVER command, dummy parameter for now
-			err = conn.PrintfLine("XOVER 0")
-			if err == nil {
+			es, lo, hi, _ := interpretGroupResult(ret)
+			for {
+				if lo > hi {
+					// server indicated empty group
+					// or
+					// we finished pulling stuff
+					break
+				}
+				if hi-lo+1 <= maxXOVERRange {
+					// not too much for us to pull
+					if es == 0 && lo == 0 && hi == 0 {
+						// empty group
+						break
+					}
+					if lo != hi {
+						// usual normal case
+						err = conn.PrintfLine("XOVER %d-%d", lo, hi)
+					} else if lo == 0 {
+						// probably something went wrong. try pulling more
+						err = conn.PrintfLine("XOVER 0-")
+					} else {
+						// normal case with one article
+						err = conn.PrintfLine("XOVER %d", lo)
+					}
+					lo = hi + 1
+				} else {
+					// too much to pull in one shot
+					err = conn.PrintfLine("XOVER %d-%d", lo, lo+maxXOVERRange-1)
+					lo += maxXOVERRange
+				}
+				if err != nil {
+					// something went very wrong
+					return
+				}
 				// no error sending command, read first line
 				code, _, err = conn.ReadCodeLine(224)
 				if code == 224 {
@@ -1279,6 +1402,11 @@ func (self *nntpConnection) scrapeGroup(daemon *NNTPDaemon, conn *textproto.Conn
 							msgid := parts[4]
 							// msgid -> reference
 							articles[msgid] = parts[5]
+							// incase server returned more articles than we requested
+							if num, nerr := strconv.ParseUint(parts[0], 10, 64); nerr == nil && num >= lo {
+								// fix lo so that we wont request them again
+								lo = num + 1
+							}
 						} else {
 							// probably not valid line
 							// ignore
@@ -1302,6 +1430,7 @@ func (self *nntpConnection) scrapeGroup(daemon *NNTPDaemon, conn *textproto.Conn
 									if err != nil {
 										// something bad happened
 										log.Println(self.name, "failed to obtain root post", refid, err)
+										// it fails only when REALLY bad stuff happens
 										return
 									}
 								}
@@ -1319,6 +1448,7 @@ func (self *nntpConnection) scrapeGroup(daemon *NNTPDaemon, conn *textproto.Conn
 									if err != nil {
 										// something bad happened
 										log.Println(self.name, "failed to obtain article", msgid, err)
+										// it fails only when REALLY bad stuff happens
 										return
 									}
 								}
@@ -1328,6 +1458,9 @@ func (self *nntpConnection) scrapeGroup(daemon *NNTPDaemon, conn *textproto.Conn
 						// something bad went down when reading multiline
 						log.Println(self.name, "failed to read multiline for", group, "XOVER command")
 					}
+				} else if code >= 420 || code <= 429 {
+					// group doesn't have articles in one way or another. not really error
+					err = nil
 				}
 			}
 		} else if err == nil {
@@ -1379,12 +1512,16 @@ func (self *nntpConnection) scrapeServer(daemon *NNTPDaemon, conn *textproto.Con
 				sc := bufio.NewScanner(dr)
 				for sc.Scan() {
 					line := sc.Text()
-					idx := strings.Index(line, " ")
+					idx := strings.IndexAny(line, " \t")
 					if idx > 0 {
+						log.Println(self.name, "got newsgroup", line[:idx])
 						groups = append(groups, line[:idx])
+					} else if idx < 0 {
+						log.Println(self.name, "got newsgroup", line)
+						groups = append(groups, line)
 					} else {
-						// invalid line? wtf.
-						log.Println(self.name, "invalid line in newsgroups multiline response:", line)
+						// can't have it starting with WS
+						log.Printf("%s invalid line in newsgroups multiline response [%s]\n", self.name, line)
 					}
 				}
 				err = sc.Err()
@@ -1402,8 +1539,8 @@ func (self *nntpConnection) scrapeServer(daemon *NNTPDaemon, conn *textproto.Con
 								// scrape the group
 								err = self.scrapeGroup(daemon, conn, group)
 								if err != nil {
-									log.Println(self.name, "did not scrape", group, err)
-									break
+									log.Println(self.name, "failure scraping group", group, "error:", err)
+									// do not break here, continue with other groups
 								}
 							}
 						} else {
@@ -1649,7 +1786,6 @@ func (self *nntpConnection) runConnection(daemon *NNTPDaemon, inbound, stream, r
 								conn.PrintfLine("203 Stream it brah")
 								self.mode = "STREAM"
 								log.Println(self.name, "streaming enabled")
-								go self.startStreaming(daemon, reader, conn)
 							}
 						}
 					}

contrib/backends/srndv2/src/srnd/null_cache.go

@@ -15,15 +15,20 @@ type NullCache struct {
 	handler *nullHandler
 }
 
+func (self *NullCache) InvertPagination() {
+	self.handler.invertPagination = true
+}
+
 type nullHandler struct {
-	database       Database
-	attachments    bool
-	requireCaptcha bool
-	name           string
-	prefix         string
-	translations   string
-	i18n           map[string]*I18N
-	access         sync.Mutex
+	database         Database
+	attachments      bool
+	requireCaptcha   bool
+	name             string
+	prefix           string
+	translations     string
+	i18n             map[string]*I18N
+	access           sync.Mutex
+	invertPagination bool
 }
 
 func (self *nullHandler) ForEachI18N(v func(string)) {
@@ -46,6 +51,7 @@ func (self *nullHandler) GetI18N(r *http.Request) *I18N {
 		i, err = NewI18n(lang, self.translations)
 		if err != nil {
 			log.Println(err)
+			return nil
 		}
 		if i != nil {
 			self.i18n[lang] = i
@@ -57,8 +63,12 @@ func (self *nullHandler) GetI18N(r *http.Request) *I18N {
 
 func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
+	sfw := strings.Count(r.URL.RawQuery, "sfw=1") > 0
 	i18n := self.GetI18N(r)
-
+	if i18n == nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
 	path := r.URL.Path
 	_, file := filepath.Split(path)
 
@@ -75,7 +85,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				goto notfound
 			}
 
-			template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n)
+			template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n, sfw)
 			return
 		} else {
 			goto notfound
@@ -83,15 +93,25 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	if strings.Trim(path, "/") == "overboard" {
 		// generate ukko aka overboard
-		template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n)
+		template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination, sfw)
 		return
 	}
 
+	// board list page
+	if strings.ToLower(path) == "/b/" {
+		template.genBoardList(self.prefix, self.name, w, self.database, i18n)
+		return
+	}
+	
 	if strings.HasPrefix(path, "/b/") {
 		// board handler
 		parts := strings.Split(path[3:], "/")
-		page := 0
 		group := parts[0]
+		page := 0
+		pages := self.database.GetGroupPageCount(group)
+		if self.invertPagination {
+			page = int(pages) - 1
+		}
 		if len(parts) > 1 && parts[1] != "" && parts[1] != "json" {
 			var err error
 			page, err = strconv.Atoi(parts[1])
@@ -111,12 +131,11 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if banned {
 			goto notfound
 		}
-
-		pages := self.database.GetGroupPageCount(group)
 		if page >= int(pages) {
 			goto notfound
 		}
-		template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, page, w, self.database, isjson, i18n)
+
+		template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination, sfw)
 		return
 	}
 
@@ -130,7 +149,13 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				goto notfound
 			}
 		}
-		template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n)
+		pages, _ := self.database.GetUkkoPageCount(10)
+		if path == "/o/" {
+			if self.invertPagination {
+				page = int(pages)
+			}
+		}
+		template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination, sfw)
 		return
 	}
 
@@ -159,17 +184,18 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if strings.HasPrefix(file, "ukko.html") {
-		template.genUkko(self.prefix, self.name, w, self.database, false, i18n)
+		template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination, sfw)
 		return
 	}
 	if strings.HasPrefix(file, "ukko.json") {
-		template.genUkko(self.prefix, self.name, w, self.database, true, i18n)
+		template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination, sfw)
 		return
 	}
 
 	if strings.HasPrefix(file, "ukko-") {
 		page := getUkkoPage(file)
-		template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n)
+		pages, _ := self.database.GetUkkoPageCount(10)
+		template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination, sfw)
 		return
 	}
 	if strings.HasPrefix(file, "thread-") {
@@ -187,7 +213,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			goto notfound
 		}
 
-		template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n)
+		template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n, sfw)
 		return
 	}
 	if strings.HasPrefix(file, "catalog-") {
@@ -199,7 +225,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if !hasgroup {
 			goto notfound
 		}
-		template.genCatalog(self.prefix, self.name, group, w, self.database, i18n)
+		template.genCatalog(self.prefix, self.name, group, w, self.database, i18n, sfw)
 		return
 	} else {
 		group, page := getGroupAndPage(file)
@@ -214,7 +240,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if page >= int(pages) {
 			goto notfound
 		}
-		template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, page, w, self.database, isjson, i18n)
+		template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination, sfw)
 		return
 	}
 
@@ -233,7 +259,7 @@ func (self *NullCache) DeleteThreadMarkup(root_post_id string) {
 func (self *NullCache) RegenAll() {
 }
 
-func (self *NullCache) RegenFrontPage() {
+func (self *NullCache) RegenFrontPage(pagestart int) {
 }
 
 func (self *NullCache) SetRequireCaptcha(required bool) {

contrib/backends/srndv2/src/srnd/postgres.go

@@ -97,7 +97,9 @@ const DeleteArticle_2 = "DeleteArticle_2"
 const DeleteArticle_3 = "DeleteArticle_3"
 const DeleteArticle_4 = "DeleteArticle_4"
 const DeleteArticle_5 = "DeleteArticle_5"
+const DeleteArticleV8 = "DeleteArticleV8"
 const DeleteThread = "DeleteThread"
+const DeleteThreadV8 = "DeleteThreadV8"
 const GetThreadReplyPostModels_1 = "GetThreadReplyPostModels_1"
 const GetThreadReplyPostModels_2 = "GetThreadReplyPostModels_2"
 const GetThreadReplies_1 = "GetThreadReplies_1"
@@ -109,7 +111,9 @@ const HasNewsgroup = "HasNewsgroup"
 const HasArticle = "HasArticle"
 const HasArticleLocal = "HasArticleLocal"
 const GetPostAttachments = "GetPostAttachments"
+const GetThreadAttachments = "GetThreadAttachments"
 const GetPostAttachmentModels = "GetPostAttachmentModels"
+const RegisterArticle_GetLastBump = "RegisterArticle_GetLastBump"
 const RegisterArticle_1 = "RegisterArticle_1"
 const RegisterArticle_2 = "RegisterArticle_2"
 const RegisterArticle_3 = "RegisterArticle_3"
@@ -147,16 +151,20 @@ const GetNNTPPostsInGroup = "GetNNTPPostsInGroup"
 const GetCitesByPostHashLike = "GetCitesByPostHashLike"
 const GetYearlyPostHistory = "GetYearlyPostHistory"
 const GetNewsgroupList = "GetNewsgroupList"
+const CountUkko = "CountUkko"
+const GetNewsgroupStats = "GetNewsgroupStats"
+const RemoveArticle = "RemoveArticle"
 
 func (self *PostgresDatabase) prepareStatements() {
 	self.stmt = map[string]string{
+		GetNewsgroupStats:               "SELECT COUNT(message_id), newsgroup FROM articleposts WHERE time_posted > (EXTRACT(epoch FROM NOW()) - (24*3600)) GROUP BY newsgroup",
 		NewsgroupBanned:                 "SELECT 1 FROM BannedGroups WHERE newsgroup = $1",
 		ArticleBanned:                   "SELECT 1 FROM BannedArticles WHERE message_id = $1",
 		GetAllNewsgroups:                "SELECT name FROM Newsgroups WHERE name NOT IN ( SELECT newsgroup FROM BannedGroups )",
-		GetPostsInGroup:                 "SELECT newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr FROM ArticlePosts WHERE newsgroup = $1 ORDER BY time_posted",
-		GetPostModel:                    "SELECT newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr FROM ArticlePosts WHERE message_id = $1 LIMIT 1",
+		GetPostsInGroup:                 "SELECT newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr, frontendpubkey FROM ArticlePosts WHERE newsgroup = $1 ORDER BY time_posted",
+		GetPostModel:                    "SELECT newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr, frontendpubkey FROM ArticlePosts WHERE message_id = $1 LIMIT 1",
 		GetArticlePubkey:                "SELECT pubkey FROM ArticleKeys WHERE message_id = $1",
-		GetThreadModel:                  "SELECT ArticlePosts.newsgroup, ArticlePosts.message_id, ArticlePosts.name, ArticlePosts.subject, ArticlePosts.time_posted, ArticlePosts.message, ArticlePosts.addr FROM ArticlePosts WHERE ArticlePosts.message_id = $1 OR ArticlePosts.ref_id = $1 ORDER BY ArticlePosts.time_posted",
+		GetThreadModel:                  "SELECT ArticlePosts.newsgroup, ArticlePosts.message_id, ArticlePosts.name, ArticlePosts.subject, ArticlePosts.time_posted, ArticlePosts.message, ArticlePosts.addr, ArticlePosts.frontendpubkey FROM ArticlePosts WHERE ArticlePosts.message_id = $1 OR ArticlePosts.ref_id = $1 ORDER BY ArticlePosts.time_posted",
 		GetThreadModelPubkeys:           "SELECT pubkey, message_id from ArticleKeys WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 OR message_id = $1 )",
 		GetThreadModelAttachments:       "SELECT filename, filepath, message_id from ArticleAttachments WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 OR message_id = $1 )",
 		DeleteArticle_1:                 "DELETE FROM NNTPHeaders WHERE header_article_message_id = $1",
@@ -165,8 +173,10 @@ func (self *PostgresDatabase) prepareStatements() {
 		DeleteArticle_4:                 "DELETE FROM ArticleKeys WHERE message_id = $1",
 		DeleteArticle_5:                 "DELETE FROM ArticleAttachments WHERE message_id = $1",
 		DeleteThread:                    "DELETE FROM ArticleThreads WHERE root_message_id = $1",
-		GetThreadReplyPostModels_1:      "SELECT newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr FROM ArticlePosts WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 ORDER BY time_posted DESC LIMIT $2 ) ORDER BY time_posted ASC",
-		GetThreadReplyPostModels_2:      "SELECT newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr FROM ArticlePosts WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 ) ORDER BY time_posted ASC",
+		DeleteArticleV8:                 "DELETE FROM ArticlePosts WHERE message_id = $1",
+		DeleteThreadV8:                  "DELETE FROM ArticlePosts WHERE ref_id = $1 OR message_id = $1",
+		GetThreadReplyPostModels_1:      "SELECT newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr, frontendpubkey FROM ArticlePosts WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 ORDER BY time_posted DESC LIMIT $2 ) ORDER BY time_posted ASC",
+		GetThreadReplyPostModels_2:      "SELECT newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr, frontendpubkey FROM ArticlePosts WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 ) ORDER BY time_posted ASC",
 		GetThreadReplies_1:              "SELECT message_id FROM ArticlePosts WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 ORDER BY time_posted DESC LIMIT $2 ) ORDER BY time_posted ASC",
 		GetThreadReplies_2:              "SELECT message_id FROM ArticlePosts WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 ) ORDER BY time_posted ASC",
 		GetGroupThreads:                 "SELECT message_id FROM ArticlePosts WHERE newsgroup = $1 AND ref_id = '' ",
@@ -176,10 +186,12 @@ func (self *PostgresDatabase) prepareStatements() {
 		HasArticle:                      "SELECT 1 FROM Articles WHERE message_id = $1",
 		HasArticleLocal:                 "SELECT 1 FROM ArticlePosts WHERE message_id = $1",
 		GetPostAttachments:              "SELECT filepath FROM ArticleAttachments WHERE message_id = $1",
+		GetThreadAttachments:            "SELECT filepath FROM ArticleAttachments WHERE message_id IN ( SELECT message_id FROM ArticlePosts WHERE ref_id = $1 OR message_id = $1)",
 		GetPostAttachmentModels:         "SELECT filepath, filename FROM ArticleAttachments WHERE message_id = $1",
+		RegisterArticle_GetLastBump:     "SELECT last_bump FROM ArticleThreads WHERE root_message_id = $1",
 		RegisterArticle_1:               "INSERT INTO Articles (message_id, message_id_hash, message_newsgroup, time_obtained, message_ref_id) VALUES($1, $2, $3, $4, $5)",
 		RegisterArticle_2:               "UPDATE Newsgroups SET last_post = $1 WHERE name = $2",
-		RegisterArticle_3:               "INSERT INTO ArticlePosts(newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)",
+		RegisterArticle_3:               "INSERT INTO ArticlePosts(newsgroup, message_id, ref_id, name, subject, path, time_posted, message, addr, frontendpubkey) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
 		RegisterArticle_4:               "INSERT INTO ArticleThreads(root_message_id, last_bump, last_post, newsgroup) VALUES($1, $2, $2, $3)",
 		RegisterArticle_5:               "SELECT COUNT(*) FROM ArticlePosts WHERE ref_id = $1",
 		RegisterArticle_6:               "UPDATE ArticleThreads SET last_bump = $2 WHERE root_message_id = $1",
@@ -192,7 +204,7 @@ func (self *PostgresDatabase) prepareStatements() {
 		GetMessageIDByHash:              "SELECT message_id, message_newsgroup FROM Articles WHERE message_id_hash = $1 LIMIT 1",
 		CheckEncIPBanned:                "SELECT 1 FROM EncIPBans WHERE encaddr = $1",
 		GetFirstAndLastForGroup:         "WITH x(min_no, max_no) AS ( SELECT MIN(message_no) AS min_no, MAX(message_no) AS max_no FROM ArticleNumbers WHERE newsgroup = $1) SELECT CASE WHEN min_no IS NULL THEN 0 ELSE min_no END AS min_no FROM x UNION SELECT CASE WHEN max_no IS NULL THEN 1 ELSE max_no END AS max_no FROM x",
-		GetNewsgroupList:                "SELECT newsgroup, min(message_no), max(message_no) FROM ArticleNumbers GROUP BY newsgroup ORDER BY newsgroup",
+		GetNewsgroupList:                "SELECT newsgroup, min(message_no), max(message_no) FROM ArticleNumbers WHERE newsgroup NOT IN ( SELECT newsgroup FROM bannedgroups ) GROUP BY newsgroup ORDER BY newsgroup",
 		GetMessageIDForNNTPID:           "SELECT message_id FROM ArticleNumbers WHERE newsgroup = $1 AND message_no = $2 LIMIT 1",
 		GetNNTPIDForMessageID:           "SELECT message_no FROM ArticleNumbers WHERE newsgroup = $1 AND message_id = $2 LIMIT 1",
 		IsExpired:                       "WITH x(msgid) AS ( SELECT message_id FROM Articles WHERE message_id = $1 INTERSECT ( SELECT message_id FROM ArticlePosts WHERE message_id = $1 ) ) SELECT COUNT(*) FROM x",
@@ -207,13 +219,15 @@ func (self *PostgresDatabase) prepareStatements() {
 		GetMessageIDByCIDR:              "SELECT message_id FROM ArticlePosts WHERE addr IN ( SELECT encaddr FROM EncryptedAddrs WHERE addr_cidr <<= cidr($1) )",
 		GetMessageIDByEncryptedIP:       "SELECT message_id FROM ArticlePosts WHERE addr = $1",
 		GetPostsBefore:                  "SELECT message_id FROM ArticlePosts WHERE time_posted < $1",
-		SearchQuery_1:                   "SELECT newsgroup, message_id, ref_id FROM ArticlePosts WHERE message LIKE $1 ORDER BY time_posted DESC",
-		SearchQuery_2:                   "SELECT newsgroup, message_id, ref_id FROM ArticlePosts WHERE newsgroup = $1 AND message LIKE $2 ORDER BY time_posted DESC",
-		SearchByHash_1:                  "SELECT message_newsgroup, message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1 ORDER BY time_obtained DESC",
-		SearchByHash_2:                  "SELECT message_newsgroup, message_id, message_ref_id FROM Articles WHERE message_newsgroup = $2 AND message_id_hash LIKE $1 ORDER BY time_obtained DESC",
+		SearchQuery_1:                   "SELECT newsgroup, message_id, ref_id FROM ArticlePosts WHERE message LIKE $1 ORDER BY time_posted DESC LIMIT $2",
+		SearchQuery_2:                   "SELECT newsgroup, message_id, ref_id FROM ArticlePosts WHERE newsgroup = $1 AND message LIKE $2 ORDER BY time_posted DESC LIMIT $3",
+		SearchByHash_1:                  "SELECT message_newsgroup, message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1 ORDER BY time_obtained DESC LIMIT $2",
+		SearchByHash_2:                  "SELECT message_newsgroup, message_id, message_ref_id FROM Articles WHERE message_newsgroup = $2 AND message_id_hash LIKE $1 ORDER BY time_obtained DESC LIMIT $3",
 		GetNNTPPostsInGroup:             "SELECT message_no, ArticlePosts.message_id, subject, time_posted, ref_id, name, path FROM ArticleNumbers INNER JOIN ArticlePosts ON ArticleNumbers.message_id = ArticlePosts.message_id WHERE ArticlePosts.newsgroup = $1 ORDER BY message_no",
 		GetCitesByPostHashLike:          "SELECT message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1",
 		GetYearlyPostHistory:            "WITH times(endtime, begintime) AS ( SELECT CAST(EXTRACT(epoch from i) AS BIGINT) AS endtime, CAST(EXTRACT(epoch from i - interval '1 month') AS BIGINT) AS begintime FROM generate_series(now() - interval '10 year', now(), '1 month'::interval) i ) SELECT begintime, endtime, ( SELECT count(*) FROM ArticlePosts WHERE time_posted > begintime AND time_posted < endtime) FROM times",
+		CountUkko:                       "SELECT COUNT(message_id) FROM ArticlePosts WHERE newsgroup != 'ctl' AND ref_id = '' OR ref_id = message_id",
+		RemoveArticle:                   "DELETE FROM Articles WHERE message_id = $1",
 	}
 
 }
@@ -247,6 +261,10 @@ func (self *PostgresDatabase) CreateTables() {
 			// upgrade to version 7
 			self.upgrade6to7()
 		} else if version == 7 {
+			self.upgrade7to8()
+		} else if version == 8 {
+			self.upgrade8to9()
+		} else if version == 9 {
 			// we are up to date
 			log.Println("we are up to date at version", version)
 			break
@@ -589,6 +607,38 @@ func (self *PostgresDatabase) upgrade6to7() {
 	self.setDBVersion(7)
 }
 
+func (self *PostgresDatabase) upgrade7to8() {
+	log.Println("migrating 7 -> 8")
+	cmds := []string{
+		"ALTER TABLE ArticleNumbers DROP CONSTRAINT IF EXISTS articlenumbers_message_id_fkey",
+		"ALTER TABLE ArticleNumbers ADD CONSTRAINT msgid_depends FOREIGN KEY (message_id) REFERENCES ArticlePosts(message_id) ON DELETE CASCADE",
+		"ALTER TABLE NNTPHeaders DROP CONSTRAINT IF EXISTS nntpheaders_header_article_message_id_fkey",
+		"ALTER TABLE NNTPHeaders ADD CONSTRAINT msgid_depends FOREIGN KEY (header_article_message_id) REFERENCES ArticlePosts(message_id) ON DELETE CASCADE",
+	}
+	for _, cmd := range cmds {
+		log.Println("exec", cmd)
+		_, err := self.conn.Exec(cmd)
+		if err != nil {
+			log.Fatalf("%s: %s", cmd, err.Error())
+		}
+	}
+	self.setDBVersion(8)
+}
+
+func (self *PostgresDatabase) upgrade8to9() {
+	cmds := []string{
+		"ALTER TABLE ArticlePosts ADD COLUMN frontendpubkey TEXT",
+		"CREATE TABLE IF NOT EXISTS nntpchan_pubkeys(status VARCHAR(16) NOT NULL, pubkey VARCHAR(64) PRIMARY KEY)",
+	}
+	for _, cmd := range cmds {
+		_, err := self.conn.Exec(cmd)
+		if err != nil {
+			log.Fatalf("%s: %s", cmd, err.Error())
+		}
+	}
+	self.setDBVersion(9)
+}
+
 // create all tables for database version 0
 func (self *PostgresDatabase) createTablesV0() {
 	tables := make(map[string]string)
@@ -1090,7 +1140,7 @@ func (self *PostgresDatabase) GetPostsInGroup(newsgroup string) (models []PostMo
 	if err == nil {
 		for rows.Next() {
 			model := new(post)
-			rows.Scan(&model.board, &model.Message_id, &model.Parent, &model.PostName, &model.PostSubject, &model.MessagePath, &model.Posted, &model.PostMessage, &model.addr)
+			rows.Scan(&model.board, &model.Message_id, &model.Parent, &model.PostName, &model.PostSubject, &model.MessagePath, &model.Posted, &model.PostMessage, &model.addr, &model.FrontendPublicKey)
 			models = append(models, model)
 		}
 		rows.Close()
@@ -1100,7 +1150,7 @@ func (self *PostgresDatabase) GetPostsInGroup(newsgroup string) (models []PostMo
 
 func (self *PostgresDatabase) GetPostModel(prefix, messageID string) PostModel {
 	model := new(post)
-	err := self.conn.QueryRow(self.stmt[GetPostModel], messageID).Scan(&model.board, &model.Message_id, &model.Parent, &model.PostName, &model.PostSubject, &model.MessagePath, &model.Posted, &model.PostMessage, &model.addr)
+	err := self.conn.QueryRow(self.stmt[GetPostModel], messageID).Scan(&model.board, &model.Message_id, &model.Parent, &model.PostName, &model.PostSubject, &model.MessagePath, &model.Posted, &model.PostMessage, &model.addr, &model.FrontendPublicKey)
 	if err == nil {
 		model.op = len(model.Parent) == 0
 		if len(model.Parent) == 0 {
@@ -1146,7 +1196,7 @@ func (self *PostgresDatabase) GetThreadModel(prefix, msgid string) (th ThreadMod
 	for err == nil && rows.Next() {
 		p := new(post)
 		p.Parent = msgid
-		err = rows.Scan(&p.board, &p.Message_id, &p.PostName, &p.PostSubject, &p.Posted, &p.PostMessage, &p.addr)
+		err = rows.Scan(&p.board, &p.Message_id, &p.PostName, &p.PostSubject, &p.Posted, &p.PostMessage, &p.addr, &p.FrontendPublicKey)
 		pmap[p.Message_id] = p
 		posts = append(posts, p)
 	}
@@ -1182,16 +1232,27 @@ func (self *PostgresDatabase) GetThreadModel(prefix, msgid string) (th ThreadMod
 }
 
 func (self *PostgresDatabase) DeleteThread(msgid string) (err error) {
-	_, err = self.conn.Exec(self.stmt[DeleteThread], msgid)
+	_, err = self.conn.Exec(self.stmt[DeleteThreadV8], msgid)
 	return
 }
 
 func (self *PostgresDatabase) DeleteArticle(msgid string) (err error) {
-	for _, q := range []string{DeleteArticle_1, DeleteArticle_2, DeleteArticle_3, DeleteArticle_4, DeleteArticle_5} {
-		_, err = self.conn.Exec(self.stmt[q], msgid)
-		if err != nil {
-			break
+	/*
+		for _, q := range []string{DeleteArticle_1, DeleteArticle_2, DeleteArticle_3, DeleteArticle_4, DeleteArticle_5} {
+			_, err = self.conn.Exec(self.stmt[q], msgid)
+			if err != nil {
+				break
+			}
 		}
+	*/
+	_, err = self.conn.Exec(self.stmt[DeleteArticleV8], msgid)
+	return
+
+}
+func (self *PostgresDatabase) RemoveArticle(msgid string) (err error) {
+	_, err = self.conn.Exec(self.stmt[DeleteArticleV8], msgid)
+	if err == nil {
+		_, err = self.conn.Exec(self.stmt[RemoveArticle], msgid)
 	}
 	return
 }
@@ -1214,7 +1275,7 @@ func (self *PostgresDatabase) GetThreadReplyPostModels(prefix, rootpost string,
 			}
 			model := new(post)
 			model.prefix = prefix
-			rows.Scan(&model.board, &model.Message_id, &model.Parent, &model.PostName, &model.PostSubject, &model.MessagePath, &model.Posted, &model.PostMessage, &model.addr)
+			rows.Scan(&model.board, &model.Message_id, &model.Parent, &model.PostName, &model.PostSubject, &model.MessagePath, &model.Posted, &model.PostMessage, &model.addr, &model.FrontendPublicKey)
 			model.op = len(model.Parent) == 0
 			if len(model.Parent) == 0 {
 				model.Parent = model.Message_id
@@ -1367,6 +1428,22 @@ func (self *PostgresDatabase) RegisterNewsgroup(group string) {
 	}
 }
 
+func (self *PostgresDatabase) GetThreadAttachments(rootmsgid string) (atts []string, err error) {
+	var rows *sql.Rows
+	rows, err = self.conn.Query(self.stmt[GetThreadAttachments], rootmsgid)
+	if err == nil {
+		for rows.Next() {
+			var msgid string
+			rows.Scan(&msgid)
+			atts = append(atts, msgid)
+		}
+		rows.Close()
+	} else if err == sql.ErrNoRows {
+		err = nil
+	}
+	return
+}
+
 func (self *PostgresDatabase) GetPostAttachments(messageID string) (atts []string) {
 	rows, err := self.conn.Query(self.stmt[GetPostAttachments], messageID)
 	if err == nil {
@@ -1427,7 +1504,7 @@ func (self *PostgresDatabase) RegisterArticle(message NNTPMessage) (err error) {
 		return
 	}
 	// insert article post
-	_, err = self.conn.Exec(self.stmt[RegisterArticle_3], group, msgid, message.Reference(), message.Name(), message.Subject(), message.Path(), message.Posted(), message.Message(), message.Addr())
+	_, err = self.conn.Exec(self.stmt[RegisterArticle_3], group, msgid, message.Reference(), message.Name(), message.Subject(), message.Path(), message.Posted(), message.Message(), message.Addr(), message.FrontendPubkey())
 	if err != nil {
 		log.Println("cannot insert article post", err)
 		return
@@ -1444,13 +1521,23 @@ func (self *PostgresDatabase) RegisterArticle(message NNTPMessage) (err error) {
 		}
 	} else {
 		ref := message.Reference()
+		postedAt := message.Posted()
+		var other int64
+		err = self.conn.QueryRow(self.stmt[RegisterArticle_GetLastBump], ref).Scan(&other)
+		if err == nil && other > postedAt {
+			postedAt = other
+		}
+		now := timeNow()
+		if postedAt > now {
+			postedAt = now
+		}
 		if !message.Sage() {
 			// TODO: this could be 1 query possibly?
 			var posts int64
 			err = self.conn.QueryRow(self.stmt[RegisterArticle_5], ref).Scan(&posts)
 			if err == nil && posts <= BumpLimit {
 				// bump it nigguh
-				_, err = self.conn.Exec(self.stmt[RegisterArticle_6], ref, message.Posted())
+				_, err = self.conn.Exec(self.stmt[RegisterArticle_6], ref, postedAt)
 			}
 			if err != nil {
 				log.Println("failed to bump thread", ref, err)
@@ -1458,7 +1545,7 @@ func (self *PostgresDatabase) RegisterArticle(message NNTPMessage) (err error) {
 			}
 		}
 		// update last posted
-		_, err = self.conn.Exec(self.stmt[RegisterArticle_7], ref, message.Posted())
+		_, err = self.conn.Exec(self.stmt[RegisterArticle_7], ref, postedAt)
 		if err != nil {
 			log.Println("failed to update post time for", ref, err)
 			return
@@ -1830,7 +1917,6 @@ func (self *PostgresDatabase) GetMessageIDByEncryptedIP(encaddr string) (msgids
 		if err == nil {
 			msgids = append(msgids, msgid)
 		}
-
 	}
 	if rows != nil {
 		rows.Close()
@@ -1838,15 +1924,32 @@ func (self *PostgresDatabase) GetMessageIDByEncryptedIP(encaddr string) (msgids
 	return
 }
 
-func (self *PostgresDatabase) BanPubkey(pubkey string) (err error) {
-	// TODO: implement
-	err = errors.New("ban pubkey not implemented")
+func (self *PostgresDatabase) WhitelistPubkey(pubkey string) (err error) {
+	_, err = self.conn.Exec("INSERT INTO nntpchan_pubkeys VALUES ('whitelist', $1)", pubkey)
 	return
 }
 
-func (self *PostgresDatabase) PubkeyIsBanned(pubkey string) (bool, error) {
-	// TODO: implement
-	return false, nil
+func (self *PostgresDatabase) DeletePubkey(pubkey string) (err error) {
+	_, err = self.conn.Exec("DELETE FROM nntpchan_pubkeys WHERE pubkey = $1", pubkey)
+	return
+}
+
+func (self *PostgresDatabase) BlacklistPubkey(pubkey string) (err error) {
+	_, err = self.conn.Exec("INSERT INTO nntpchan_pubkeys VALUES ('blacklist', $1)", pubkey)
+	return
+}
+
+// return true if we should drop this message with this frontend pubkey
+func (self *PostgresDatabase) PubkeyRejected(pubkey string) (bool, error) {
+	var num int64
+	var drop bool
+	var err error
+	err = self.conn.QueryRow("SELECT COUNT(pubkey) FROM nntpchan_pubkeys WHERE pubkey = $1 AND status = 'whitelist'", pubkey).Scan(&num)
+	if err == nil && num == 0 {
+		err = self.conn.QueryRow("SELECT COUNT(pubkey) FROM nntpchan_pubkeys WHERE pubkey = $1 and status = 'blacklist'", pubkey).Scan(&num)
+		drop = num > 0
+	}
+	return drop, err
 }
 
 func (self *PostgresDatabase) GetPostsBefore(t time.Time) (msgids []string, err error) {
@@ -1867,14 +1970,14 @@ func (self *PostgresDatabase) GetPostingStats(gran, begin, end int64) (st Postin
 	return
 }
 
-func (self *PostgresDatabase) SearchQuery(prefix, group string, text string, chnl chan PostModel) (err error) {
+func (self *PostgresDatabase) SearchQuery(prefix, group string, text string, chnl chan PostModel, limit int) (err error) {
 	if text != "" && strings.Count(text, "%") == 0 {
 		text = "%" + text + "%"
 		var rows *sql.Rows
 		if group == "" {
-			rows, err = self.conn.Query(self.stmt[SearchQuery_1], text)
+			rows, err = self.conn.Query(self.stmt[SearchQuery_1], text, limit)
 		} else {
-			rows, err = self.conn.Query(self.stmt[SearchQuery_2], group, text)
+			rows, err = self.conn.Query(self.stmt[SearchQuery_2], group, text, limit)
 		}
 		if err == nil {
 			for rows.Next() {
@@ -1888,15 +1991,15 @@ func (self *PostgresDatabase) SearchQuery(prefix, group string, text string, chn
 	close(chnl)
 	return
 }
-func (self *PostgresDatabase) SearchByHash(prefix, group, text string, chnl chan PostModel) (err error) {
+func (self *PostgresDatabase) SearchByHash(prefix, group, text string, chnl chan PostModel, limit int) (err error) {
 	if text != "" && strings.Count(text, "%") == 0 {
 		text = "%" + text + "%"
 		var rows *sql.Rows
 		if group == "" {
-			rows, err = self.conn.Query(self.stmt[SearchByHash_1], text)
+			rows, err = self.conn.Query(self.stmt[SearchByHash_1], text, limit)
 		} else {
 
-			rows, err = self.conn.Query(self.stmt[SearchByHash_2], text, group)
+			rows, err = self.conn.Query(self.stmt[SearchByHash_2], text, group, limit)
 		}
 		if err == nil {
 			for rows.Next() {
@@ -1954,6 +2057,26 @@ func (self *PostgresDatabase) FindCitesInText(text string) (msgids []string, err
 	return
 }
 
+func (self *PostgresDatabase) GetUkkoPageCount(perpage int) (count int64, err error) {
+	err = self.conn.QueryRow(self.stmt[CountUkko]).Scan(&count)
+	count /= int64(perpage)
+	return
+}
+
+func (self *PostgresDatabase) GetNewsgroupStats() (stats []NewsgroupStats, err error) {
+	var rows *sql.Rows
+	rows, err = self.conn.Query(self.stmt[GetNewsgroupStats])
+	if err == nil {
+		for rows.Next() {
+			var s NewsgroupStats
+			rows.Scan(&s.PPD, &s.Name)
+			stats = append(stats, s)
+		}
+		rows.Close()
+	}
+	return
+}
+
 func (self *PostgresDatabase) FindHeaders(group, headername string, lo, hi int64) (hdr ArticleHeaders, err error) {
 	hdr = make(ArticleHeaders)
 	q := "SELECT header_value FROM nntpheaders WHERE header_name = $1 AND header_article_message_id IN ( SELECT message_id FROM articleposts WHERE newsgroup = $2 )"

contrib/backends/srndv2/src/srnd/spam.go

@@ -34,20 +34,46 @@ type SpamResult struct {
 	IsSpam bool
 }
 
+// feed spam subsystem a spam post
+func (sp *SpamFilter) MarkSpam(msg io.Reader) (err error) {
+	var buf [65636]byte
+
+	var u *user.User
+	u, err = user.Current()
+	if err != nil {
+		return
+	}
+	var conn *net.TCPConn
+	conn, err = sp.openConn()
+	if err != nil {
+		return
+	}
+	defer conn.Close()
+	fmt.Fprintf(conn, "TELL SPAMC/1.5\r\nUser: %s\r\nMessage-class: spam\r\nSet: local\r\n\r\n", u.Username)
+	io.CopyBuffer(conn, msg, buf[:])
+	conn.CloseWrite()
+	r := bufio.NewReader(conn)
+	io.Copy(Discard, r)
+	return
+}
+
+func (sp *SpamFilter) openConn() (*net.TCPConn, error) {
+	addr, err := net.ResolveTCPAddr("tcp", sp.addr)
+	if err != nil {
+		return nil, err
+	}
+	return net.DialTCP("tcp", nil, addr)
+}
+
 func (sp *SpamFilter) Rewrite(msg io.Reader, out io.WriteCloser, group string) (result SpamResult) {
 	var buff [65636]byte
 	if !sp.Enabled(group) {
 		result.Err = ErrSpamFilterNotEnabled
 		return
 	}
-	var addr *net.TCPAddr
-	var c *net.TCPConn
 	var u *user.User
-	addr, result.Err = net.ResolveTCPAddr("tcp", sp.addr)
-	if result.Err != nil {
-		return
-	}
-	c, result.Err = net.DialTCP("tcp", nil, addr)
+	var c *net.TCPConn
+	c, result.Err = sp.openConn()
 	if result.Err != nil {
 		return
 	}

contrib/backends/srndv2/src/srnd/store.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
@@ -25,8 +26,17 @@ import (
 
 var ErrOversizedMessage = errors.New("oversized message")
 
-// ~ 10 MB unbased64'd
-const DefaultMaxMessageSize = 1024 * 1024 * 10
+// (cathugger)
+// my test showed that 8MiB of attachments split in 5 parts
+// plus some text produce something close to typhical big message
+// resulted in 11483923 bytes.
+// that's consistent with rough size calculation mentioned in
+// <https://en.wikipedia.org/wiki/Base64#MIME>
+// ((origlen * 1.37) + 814)
+// which resulted in 11493206 bytes for 8MiB of data.
+// previous default of 10MiB (10485760) was too low in practice.
+// use 11MiB (11534336) to leave some space for longer than usual texts.
+const DefaultMaxMessageSize = 11 * 1024 * 1024
 
 // HARD max message size
 const MaxMessageSize = 1024 * 1024 * 1024
@@ -74,7 +84,7 @@ type ArticleStore interface {
 	// register signed message
 	RegisterSigned(msgid, pk string) error
 
-	GetMessage(msgid string) NNTPMessage
+	GetMessage(msgid string, visit func(NNTPMessage))
 
 	// get size of message on disk
 	GetMessageSize(msgid string) (int64, error)
@@ -115,6 +125,7 @@ type articleStore struct {
 	identify_path string
 	placeholder   string
 	spamdir       string
+	hamdir        string
 	compression   bool
 	compWriter    *gzip.Writer
 	spamd         *SpamFilter
@@ -136,7 +147,9 @@ func createArticleStore(config map[string]string, thumbConfig *ThumbnailConfig,
 		compression:   config["compression"] == "1",
 		spamd:         spamd,
 		spamdir:       filepath.Join(config["store_dir"], "spam"),
-		thumbnails:    thumbConfig,
+		hamdir:        filepath.Join(config["store_dir"], "ham"),
+
+		thumbnails: thumbConfig,
 	}
 	store.Init()
 	return store
@@ -543,8 +556,8 @@ func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter
 			if e != nil {
 				log.Println("failed to read entire message", e)
 			}
-			pw_in.Close()
-			pr_in.Close()
+			pw_in.CloseWithError(e)
+			pr_in.CloseWithError(e)
 		}()
 		r := bufio.NewReader(pr_out)
 		m, e := readMIMEHeader(r)
@@ -575,33 +588,26 @@ func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter
 			return
 		}
 		writeMIMEHeader(wr, m.Header)
-		read_message_body(m.Body, m.Header, self, wr, false, process)
+		err = read_message_body(m.Body, m.Header, self, wr, false, process)
 	}
 	return
 }
 
-func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
+func (self *articleStore) GetMessage(msgid string, visit func(NNTPMessage)) {
 	r, err := self.OpenMessage(msgid)
 	if err == nil {
 		defer r.Close()
 		br := bufio.NewReader(r)
 		msg, err := readMIMEHeader(br)
 		if err == nil {
-			chnl := make(chan NNTPMessage)
 			hdr := textproto.MIMEHeader(msg.Header)
 			err = read_message_body(msg.Body, hdr, nil, nil, true, func(n NNTPMessage) {
-				c := chnl
 				if n != nil {
 					// inject pubkey for mod
 					n.Headers().Set("X-PubKey-Ed25519", hdr.Get("X-PubKey-Ed25519"))
-					c <- n
 				}
+				visit(n)
 			})
-			if err == nil {
-				nntp = <-chnl
-			} else {
-				log.Println("GetMessage() failed to load", msgid, err)
-			}
 		}
 	}
 	return
@@ -628,7 +634,7 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
 		body = io.TeeReader(body, wr)
 	}
 	boundary, ok := params["boundary"]
-	if ok || content_type == "multipart/mixed" {
+	if strings.HasPrefix(media_type, "multipart/") && ok {
 		partReader := multipart.NewReader(body, boundary)
 		for {
 			part, err := partReader.NextPart()
@@ -638,7 +644,11 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
 			} else if err == nil {
 				hdr := part.Header
 				// get content type of part
-				part_type := hdr.Get("Content-Type")
+				part_type := strings.TrimSpace(hdr.Get("Content-Type"))
+				if part_type == "" {
+					// default if unspecified
+					part_type = "text/plain"
+				}
 				// parse content type
 				media_type, _, err = mime.ParseMediaType(part_type)
 				if err == nil {
@@ -692,7 +702,14 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
 		}
 		// process inner body
 		// verify message
-		f := func(h map[string][]string, innerBody io.Reader) {
+		f := func(h ArticleHeaders, innerBody io.Reader) {
+			// override some of headers of inner message
+			msgid := nntp.MessageID()
+			if msgid != "" {
+				h.Set("Message-Id", msgid)
+			}
+			h.Set("Path", nntp.headers.Get("Path", ""))
+			h.Set("X-Pubkey-Ed25519", pk)
 			// handle inner message
 			e := read_message_body(innerBody, h, store, nil, true, callback)
 			if e != nil {
@@ -781,7 +798,11 @@ func (self *articleStore) AcceptTempArticle(msgid string) (err error) {
 			} else {
 				err = os.Rename(temp, store)
 			}
+		} else {
+			err = fmt.Errorf("no such inbound article %s", temp)
 		}
+	} else {
+		err = fmt.Errorf("invalid message id %s", msgid)
 	}
 	return
 }

contrib/backends/srndv2/src/srnd/templates_impl.go

@@ -222,57 +222,80 @@ func (self *templateEngine) obtainBoard(prefix, frontend, group string, db Datab
 	return
 }
 
-func (self *templateEngine) genCatalog(prefix, frontend, group string, wr io.Writer, db Database, i18n *I18N) {
+func (self *templateEngine) genCatalog(prefix, frontend, group string, wr io.Writer, db Database, i18n *I18N, sfw bool) {
 	board := self.obtainBoard(prefix, frontend, group, db)
 	catalog := new(catalogModel)
 	catalog.prefix = prefix
 	catalog.frontend = frontend
 	catalog.board = group
 	catalog.I18N(i18n)
+	catalog.MarkSFW(sfw)
 	for page, bm := range board {
 		for _, th := range bm.Threads() {
 			th.Update(db)
 			catalog.threads = append(catalog.threads, &catalogItemModel{op: th.OP(), page: page, replycount: len(th.Replies())})
 		}
 	}
-	self.writeTemplate("catalog", map[string]interface{}{"board": catalog}, wr, i18n)
+	self.writeTemplate("catalog", map[string]interface{}{"board": catalog, "sfw": sfw}, wr, i18n)
 }
 
 // generate a board page
-func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, page int, wr io.Writer, db Database, json bool, i18n *I18N) {
+func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, pages, page int, wr io.Writer, db Database, json bool, i18n *I18N, invertPagination, sfw bool) {
 	// get the board page model
 	perpage, _ := db.GetThreadsPerPage(newsgroup)
-	boardPage := db.GetGroupForPage(prefix, frontend, newsgroup, page, int(perpage))
+	var boardPage BoardModel
+	if invertPagination {
+		boardPage = db.GetGroupForPage(prefix, frontend, newsgroup, int(pages-1)-page, int(perpage))
+	} else {
+		boardPage = db.GetGroupForPage(prefix, frontend, newsgroup, page, int(perpage))
+	}
 	boardPage.Update(db)
 	boardPage.I18N(i18n)
+	boardPage.MarkSFW(sfw)
 	// render it
 	if json {
 		self.renderJSON(wr, boardPage)
 	} else {
 		form := renderPostForm(prefix, newsgroup, "", allowFiles, requireCaptcha, i18n)
-		self.writeTemplate("board", map[string]interface{}{"board": boardPage, "page": page, "form": form}, wr, i18n)
+		self.writeTemplate("board", map[string]interface{}{"board": boardPage, "page": page, "form": form, "sfw": sfw}, wr, i18n)
 	}
 }
 
-func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N) {
-	self.genUkkoPaginated(prefix, frontend, wr, database, 0, json, i18n)
+func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N, invertPagination, sfw bool) {
+	var page int64
+	pages, err := database.GetUkkoPageCount(10)
+	if invertPagination {
+		page = pages
+	}
+	if err == nil {
+		self.genUkkoPaginated(prefix, frontend, wr, database, int(pages), int(page), json, i18n, invertPagination, sfw)
+	} else {
+		log.Println("genUkko()", err.Error())
+	}
 }
 
-func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, page int, json bool, i18n *I18N) {
+func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, pages, page int, json bool, i18n *I18N, invertPagination, sfw bool) {
 	var threads []ThreadModel
-	for _, article := range database.GetLastBumpedThreadsPaginated("", 10, page*10) {
+	var articles []ArticleEntry
+	if invertPagination {
+		articles = database.GetLastBumpedThreadsPaginated("", 10, (pages-page)*10)
+	} else {
+		articles = database.GetLastBumpedThreadsPaginated("", 10, page*10)
+	}
+	for _, article := range articles {
 		root := article[0]
 		thread, err := database.GetThreadModel(prefix, root)
 		if err == nil {
 			thread.I18N(i18n)
+			thread.MarkSFW(sfw)
 			threads = append(threads, thread)
 		}
 	}
-	obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page}
+	obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page, "sfw": sfw}
 	if page > 0 {
 		obj["prev"] = map[string]interface{}{"no": page - 1}
 	}
-	if page < 10 {
+	if page < pages {
 		obj["next"] = map[string]interface{}{"no": page + 1}
 	}
 	if json {
@@ -290,7 +313,7 @@ func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writ
 	}
 }
 
-func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root ArticleEntry, prefix, frontend string, wr io.Writer, db Database, json bool, i18n *I18N) {
+func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root ArticleEntry, prefix, frontend string, wr io.Writer, db Database, json bool, i18n *I18N, sfw bool) {
 	newsgroup := root.Newsgroup()
 	msgid := root.MessageID()
 
@@ -302,12 +325,13 @@ func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root Arti
 	*/
 	t, err := db.GetThreadModel(prefix, msgid)
 	if err == nil {
+		t.MarkSFW(sfw)
 		if json {
 			self.renderJSON(wr, t)
 		} else {
 			t.I18N(i18n)
 			form := renderPostForm(prefix, newsgroup, msgid, allowFiles, requireCaptcha, i18n)
-			self.writeTemplate("thread", map[string]interface{}{"thread": t, "board": map[string]interface{}{"Name": newsgroup, "Frontend": frontend, "AllowFiles": allowFiles}, "form": form, "prefix": prefix}, wr, i18n)
+			self.writeTemplate("thread", map[string]interface{}{"sfw": sfw, "thread": t, "board": map[string]interface{}{"Name": newsgroup, "Frontend": frontend, "AllowFiles": allowFiles}, "form": form, "prefix": prefix}, wr, i18n)
 		}
 	} else {
 		log.Println("templates: error getting thread for ", msgid, err.Error())
@@ -451,31 +475,29 @@ func (self *templateEngine) genGraphs(prefix string, wr io.Writer, db Database,
 
 func (self *templateEngine) genBoardList(prefix, name string, wr io.Writer, db Database, i18n *I18N) {
 	// the graph for the front page
-	var frontpage_graph boardPageRows
+	var graph boardPageRows
 
-	// for each group
-	groups := db.GetAllNewsgroups()
-	for _, group := range groups {
-		// posts this hour
-		hour := db.CountPostsInGroup(group, 3600)
-		// posts today
-		day := db.CountPostsInGroup(group, 86400)
-		// posts total
-		all := db.CountPostsInGroup(group, 0)
-		frontpage_graph = append(frontpage_graph, boardPageRow{
-			All:   all,
-			Day:   day,
-			Hour:  hour,
-			Board: group,
+	stats, err := db.GetNewsgroupStats()
+	if err != nil {
+		log.Println("error getting board list", err)
+		io.WriteString(wr, err.Error())
+		return
+	}
+
+	for idx := range stats {
+		graph = append(graph, boardPageRow{
+			Board: stats[idx].Name,
+			Day: stats[idx].PPD,
 		})
 	}
+	
 	param := map[string]interface{}{
 		"prefix":   prefix,
 		"frontend": name,
 	}
-	sort.Sort(frontpage_graph)
-	param["graph"] = frontpage_graph
-	_, err := io.WriteString(wr, self.renderTemplate("boardlist", param, i18n))
+	sort.Sort(graph)
+	param["graph"] = graph
+	_, err = io.WriteString(wr, self.renderTemplate("boardlist", param, i18n))
 	if err != nil {
 		log.Println("error writing board list page", err)
 	}

contrib/backends/srndv2/src/srnd/tls.go

@@ -42,6 +42,7 @@ func HandleStartTLS(conn net.Conn, config *tls.Config) (econn *textproto.Conn, s
 				econn = textproto.NewConn(tconn)
 				return
 			} else {
+				log.Println("tls handshake error: ", err.Error())
 				certs := state.PeerCertificates
 				if len(certs) == 0 {
 					log.Println("starttls failed, no peer certs provided")
@@ -53,6 +54,7 @@ func HandleStartTLS(conn net.Conn, config *tls.Config) (econn *textproto.Conn, s
 					}
 				}
 				tconn.Close()
+				conn.Close()
 			}
 		}
 	}

contrib/backends/srndv2/src/srnd/util.go

@@ -15,6 +15,7 @@ import (
 	"golang.org/x/crypto/ed25519"
 	"io"
 	"log"
+	"mime"
 	"net"
 	"net/http"
 	"net/mail"
@@ -26,6 +27,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 )
 
 func DelFile(fname string) {
@@ -60,10 +62,38 @@ func EnsureDir(dirname string) {
 	}
 }
 
-var exp_valid_message_id = regexp.MustCompilePOSIX(`^<[a-zA-Z0-9$.]{2,128}@[a-zA-Z0-9\-.]{2,63}>$`)
+// printableASCII tells whether string is made of US-ASCII printable characters
+// except of specified one.
+func printableASCII(s string, e byte) bool {
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		// NOTE: doesn't include space, which is neither printable nor control
+		if c <= 32 || c >= 127 || c == e {
+			return false
+		}
+	}
+	return true
+}
 
 func ValidMessageID(id string) bool {
-	return exp_valid_message_id.MatchString(id)
+	/*
+	   {RFC 3977}
+	   o  A message-id MUST begin with "<", end with ">", and MUST NOT
+	      contain the latter except at the end.
+	   o  A message-id MUST be between 3 and 250 octets in length.
+	   o  A message-id MUST NOT contain octets other than printable US-ASCII
+	      characters.
+
+	   additionally, we check path characters, they may be dangerous
+	*/
+	return len(id) >= 3 && len(id) <= 250 &&
+		id[0] == '<' && id[len(id)-1] == '>' &&
+		printableASCII(id[1:len(id)-1], '>') &&
+		strings.IndexAny(id[1:len(id)-1], "/\\") < 0
+}
+
+func ReservedMessageID(id string) bool {
+	return id == "<0>" || id == "<keepalive@dummy.tld>"
 }
 
 // message id hash
@@ -137,6 +167,133 @@ func nntpSanitize(data string) (ret string) {
 	return ret
 }
 
+var safeHeaderReplacer = strings.NewReplacer(
+	"\t", " ",
+	"\n", string(unicode.ReplacementChar),
+	"\r", string(unicode.ReplacementChar),
+	"\000", string(unicode.ReplacementChar))
+
+// safeHeader replaces dangerous stuff from header,
+// also replaces space with tab for XOVER/OVER output
+func safeHeader(s string) string {
+	return strings.TrimSpace(safeHeaderReplacer.Replace(s))
+}
+
+func isVchar(r rune) bool {
+	// RFC 5234 B.1: VCHAR =  %x21-7E ; visible (printing) characters
+	// RFC 6532 3.2: VCHAR =/ UTF8-non-ascii
+	return (r >= 0x21 && r <= 0x7E) || r >= 0x80
+}
+
+func isAtext(r rune) bool {
+	// RFC 5322: Printable US-ASCII characters not including specials.  Used for atoms.
+	switch r {
+	case '(', ')', '<', '>', '[', ']', ':', ';', '@', '\\', ',', '.', '"':
+		return false
+	}
+	return isVchar(r)
+}
+
+func isWSP(r rune) bool { return r == ' ' || r == '\t' }
+
+func isQtext(r rune) bool {
+	if r == '\\' || r == '"' {
+		return false
+	}
+	return isVchar(r)
+}
+
+func writeQuoted(b *strings.Builder, s string) {
+	b.WriteByte('"')
+	for _, r := range s {
+		if isQtext(r) || isWSP(r) {
+			b.WriteRune(r)
+		} else {
+			b.WriteByte('\\')
+			b.WriteRune(r)
+		}
+	}
+	b.WriteByte('"')
+}
+
+func formatAddress(name, email string) string {
+	// somewhat based on stdlib' mail.Address.String()
+
+	b := &strings.Builder{}
+
+	if name != "" {
+		needsEncoding := false
+		needsQuoting := false
+		for i, r := range name {
+			if r >= 0x80 || (!isWSP(r) && !isVchar(r)) {
+				needsEncoding = true
+				break
+			}
+			if isAtext(r) {
+				continue
+			}
+			if r == ' ' && i > 0 && name[i-1] != ' ' && i < len(name)-1 {
+				// allow spaces but only surrounded by non-spaces
+				// otherwise they will be removed by receiver
+				continue
+			}
+			needsQuoting = true
+		}
+
+		if needsEncoding {
+			// Text in an encoded-word in a display-name must not contain certain
+			// characters like quotes or parentheses (see RFC 2047 section 5.3).
+			// When this is the case encode the name using base64 encoding.
+			if strings.ContainsAny(name, "\"#$%&'(),.:;<>@[]^`{|}~") {
+				b.WriteString(mime.BEncoding.Encode("utf-8", name))
+			} else {
+				b.WriteString(mime.QEncoding.Encode("utf-8", name))
+			}
+		} else if needsQuoting {
+			writeQuoted(b, name)
+		} else {
+			b.WriteString(name)
+		}
+
+		b.WriteByte(' ')
+	}
+
+	at := strings.LastIndex(email, "@")
+	var local, domain string
+	if at >= 0 {
+		local, domain = email[:at], email[at+1:]
+	} else {
+		local = email
+	}
+
+	quoteLocal := false
+	for i, r := range local {
+		if isAtext(r) {
+			// if atom then okay
+			continue
+		}
+		if r == '.' && r > 0 && local[i-1] != '.' && i < len(local)-1 {
+			// dots are okay but only if surrounded by non-dots
+			continue
+		}
+		quoteLocal = true
+		break
+	}
+
+	b.WriteByte('<')
+	if !quoteLocal {
+		b.WriteString(local)
+	} else {
+		writeQuoted(b, local)
+	}
+	b.WriteByte('@')
+	b.WriteString(domain)
+	b.WriteByte('>')
+
+	return b.String()
+}
+
+
 type int64Sorter []int64
 
 func (self int64Sorter) Len() int {
@@ -320,28 +477,47 @@ func newNaclSignKeypair() (string, string) {
 	return hex.EncodeToString(pk), hex.EncodeToString(sk)
 }
 
+func makeTripcodeLen(pubkey string, length int) string {
+	var b strings.Builder
+
+	data, err := hex.DecodeString(pubkey)
+	if err != nil {
+		return "[invalid]"
+	}
+
+	if length <= 0 || length > len(data) {
+		length = len(data)
+	}
+
+	// originally srnd (and srndv2) used 9600==0x2580
+	// however, range shifted by 0x10 looks better to me (cathugger)
+	// (instead of `▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏` it'll use `⚀⚁⚂⚃⚄⚅⚆⚇⚈⚉⚊⚋⚌⚍⚎⚏`)
+	// and display equaly good both in torbrowser+DejaVuSans and phone
+	// since jeff ack'd it (he doesn't care probably), I'll just use it
+	const rstart = 0x2590
+	// 0x2500 can display with TBB font whitelist, but looks too cryptic.
+	// startin from 0x2600 needs more than DejaVuSans so I'll avoid it
+
+	// logic (same as in srnd):
+	// it first writes length/2 chars of begining
+	// and then length/2 chars of ending
+	// if length==len(data), that essentially means just using whole
+	i := 0
+	for ; i < length/2; i++ {
+		b.WriteRune(rstart + rune(data[i]))
+		b.WriteRune(0xFE0E) // text style variant
+	}
+	for ; i < length; i++ {
+		b.WriteRune(rstart + rune(data[len(data)-length+i]))
+		b.WriteRune(0xFE0E) // text style variant
+	}
+
+	return b.String()
+}
+
 // make a utf-8 tripcode
 func makeTripcode(pk string) string {
-	data, err := hex.DecodeString(pk)
-	if err == nil {
-		tripcode := ""
-		//  here is the python code this is based off of
-		//  i do something slightly different but this is the base
-		//
-		//  for x in range(0, length / 2):
-		//    pub_short += '&#%i;' % (9600 + int(full_pubkey_hex[x*2:x*2+2], 16))
-		//  length -= length / 2
-		//  for x in range(0, length):
-		//    pub_short += '&#%i;' % (9600 + int(full_pubkey_hex[-(length*2):][x*2:x*2+2], 16))
-		//
-		for _, c := range data {
-			ch := 9600
-			ch += int(c)
-			tripcode += fmt.Sprintf("&#%04d;", ch)
-		}
-		return tripcode
-	}
-	return "[invalid]"
+	return makeTripcodeLen(pk, 0)
 }
 
 // generate a new message id with base name
@@ -482,7 +658,7 @@ func IPNet2MinMax(inet *net.IPNet) (min, max net.IP) {
 	maskb := []byte(inet.Mask)
 	maxb := make([]byte, len(netb))
 
-	for i, _ := range maxb {
+	for i := range maxb {
 		maxb[i] = netb[i] | (^maskb[i])
 	}
 	min = net.IP(netb)
@@ -745,7 +921,7 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
 		log.Println("dropping message with invalid mime header, no message-id")
 		_, err = io.Copy(Discard, body)
 		return
-	} else if ValidMessageID(msgid) {
+	} else if ValidMessageID(msgid) && !ReservedMessageID(msgid) {
 		f = daemon.store.CreateFile(msgid)
 	} else {
 		// invalid message-id
@@ -761,9 +937,9 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
 	}
 
 	// ask for replies
-	replyTos := strings.Split(hdr.Get("Reply-To"), " ")
+	replyTos := strings.Split(hdr.Get("In-Reply-To"), " ")
 	for _, reply := range replyTos {
-		if ValidMessageID(reply) {
+		if ValidMessageID(reply) && !ReservedMessageID(reply) {
 			if !daemon.store.HasArticle(reply) {
 				go daemon.askForArticle(reply)
 			}
@@ -777,8 +953,8 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
 	go func() {
 		var buff [65536]byte
 		writeMIMEHeader(pw, hdr)
-		io.CopyBuffer(pw, body, buff[:])
-		pw.Close()
+		_, e := io.CopyBuffer(pw, body, buff[:])
+		pw.CloseWithError(e)
 	}()
 	err = daemon.store.ProcessMessage(f, pr, daemon.CheckText, hdr.Get("Newsgroups"))
 	pr.Close()
@@ -789,14 +965,16 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
 	} else {
 		log.Println("error processing message body", err)
 	}
-	if err != nil {
-		// clean up
-		if ValidMessageID(msgid) {
-			fname := daemon.store.GetFilenameTemp(msgid)
-			log.Println("clean up", fname)
-			DelFile(fname)
-		}
-		log.Println("error processing message", err)
+
+	// clean up
+	if ValidMessageID(msgid) {
+		fname := daemon.store.GetFilenameTemp(msgid)
+		DelFile(fname)
 	}
 	return
 }
+
+func hasAtLeastNWords(str string, n int) bool {
+	parts := strings.Split(str, " ")
+	return len(parts) > n
+}

contrib/backends/srndv2/src/srnd/varnish_cache.go

@@ -13,7 +13,14 @@ type VarnishCache struct {
 	prefix           string
 	handler          *nullHandler
 	client           *http.Client
+	transport        *http.Transport
+	workers          int
 	threadsRegenChan chan ArticleEntry
+	invalidateChan   chan *url.URL
+}
+
+func (self *VarnishCache) InvertPagination() {
+	self.handler.invertPagination = true
 }
 
 func (self *VarnishCache) invalidate(r string) {
@@ -29,18 +36,26 @@ func (self *VarnishCache) invalidate(r string) {
 			q.Add("lang", lang)
 			u.RawQuery = q.Encode()
 		}
-		resp, err := self.client.Do(&http.Request{
-			Method: "PURGE",
-			URL:    u,
-		})
-		if err == nil {
-			resp.Body.Close()
-		} else {
-			log.Println("varnish cache error", err)
-		}
+		self.invalidateChan <- u
 	}
 }
 
+func (self *VarnishCache) doRequest(u *url.URL) {
+	if u == nil {
+		return
+	}
+	resp, err := self.client.Do(&http.Request{
+		Method: "PURGE",
+		URL:    u,
+	})
+	if err == nil {
+		resp.Body.Close()
+	} else {
+		log.Println("varnish cache error", err)
+	}
+	self.transport.CloseIdleConnections()
+}
+
 func (self *VarnishCache) DeleteBoardMarkup(group string) {
 	n, _ := self.handler.database.GetPagesPerBoard(group)
 	for n > 0 {
@@ -69,20 +84,25 @@ func (self *VarnishCache) RegenAll() {
 	}
 }
 
-func (self *VarnishCache) RegenFrontPage() {
+func (self *VarnishCache) RegenFrontPage(pagestart int) {
 	self.invalidate(fmt.Sprintf("%s%s", self.varnish_url, self.prefix))
 	// TODO: this is also lazy af
 	self.invalidate(fmt.Sprintf("%s%shistory.html", self.varnish_url, self.prefix))
-	self.invalidateUkko(10)
+	if self.handler.invertPagination {
+		self.invalidateUkko(50, pagestart-50)
+	} else {
+		self.invalidateUkko(pagestart, 0)
+	}
 }
 
-func (self *VarnishCache) invalidateUkko(pages int) {
+func (self *VarnishCache) invalidateUkko(pages, start int) {
 	self.invalidate(fmt.Sprintf("%s%sukko.html", self.varnish_url, self.prefix))
 	self.invalidate(fmt.Sprintf("%s%so/", self.varnish_url, self.prefix))
 	self.invalidate(fmt.Sprintf("%s%sukko.json", self.varnish_url, self.prefix))
 	self.invalidate(fmt.Sprintf("%s%so/json", self.varnish_url, self.prefix))
-	n := 0
-	for n < pages {
+	n := start
+	end := start + pages
+	for n < end {
 		self.invalidate(fmt.Sprintf("%s%so/%d/json", self.varnish_url, self.prefix, n))
 		self.invalidate(fmt.Sprintf("%s%so/%d/", self.varnish_url, self.prefix, n))
 		n++
@@ -119,6 +139,20 @@ func (self *VarnishCache) poll() {
 
 func (self *VarnishCache) Start() {
 	go self.poll()
+	workers := self.workers
+	if workers <= 0 {
+		workers = 1
+	}
+	for workers > 0 {
+		go self.doWorker()
+		workers--
+	}
+}
+
+func (self *VarnishCache) doWorker() {
+	for {
+		self.doRequest(<-self.invalidateChan)
+	}
 }
 
 func (self *VarnishCache) Regen(msg ArticleEntry) {
@@ -137,25 +171,32 @@ func (self *VarnishCache) SetRequireCaptcha(required bool) {
 	self.handler.requireCaptcha = required
 }
 
-func NewVarnishCache(varnish_url, bind_addr, prefix, webroot, name, translations string, attachments bool, db Database, store ArticleStore) CacheInterface {
+func NewVarnishCache(varnish_url, bind_addr, prefix, webroot, name, translations string, workers int, attachments bool, db Database, store ArticleStore) CacheInterface {
 	cache := new(VarnishCache)
+	cache.invalidateChan = make(chan *url.URL)
 	cache.threadsRegenChan = make(chan ArticleEntry)
+	cache.workers = workers
 	local_addr, err := net.ResolveTCPAddr("tcp", bind_addr)
 	if err != nil {
 		log.Fatalf("failed to resolve %s for varnish cache: %s", bind_addr, err)
 	}
-	cache.client = &http.Client{
-		Transport: &http.Transport{
-			Dial: func(network, addr string) (c net.Conn, err error) {
-				var remote_addr *net.TCPAddr
-				remote_addr, err = net.ResolveTCPAddr(network, addr)
-				if err == nil {
-					c, err = net.DialTCP(network, local_addr, remote_addr)
-				}
-				return
-			},
+	cache.transport = &http.Transport{
+		Dial: func(network, addr string) (c net.Conn, err error) {
+			var remote_addr *net.TCPAddr
+			remote_addr, err = net.ResolveTCPAddr(network, addr)
+			if err == nil {
+				c, err = net.DialTCP(network, local_addr, remote_addr)
+			}
+			return
 		},
+		DisableKeepAlives:   true,
+		MaxIdleConnsPerHost: workers,
+		MaxIdleConns:        workers,
 	}
+	cache.client = &http.Client{
+		Transport: cache.transport,
+	}
+
 	cache.prefix = "/"
 	cache.handler = &nullHandler{
 		prefix:         prefix,

contrib/backends/srndv2/src/srnd/vendor/github.com/lib/pq/.travis.sh

@@ -70,4 +70,17 @@ postgresql_uninstall() {
 	sudo rm -rf /var/lib/postgresql
 }
 
+megacheck_install() {
+	# Lock megacheck version at $MEGACHECK_VERSION to prevent spontaneous
+	# new error messages in old code.
+	go get -d honnef.co/go/tools/...
+	git -C $GOPATH/src/honnef.co/go/tools/ checkout $MEGACHECK_VERSION
+	go install honnef.co/go/tools/cmd/megacheck
+	megacheck --version
+}
+
+golint_install() {
+	go get golang.org/x/lint/golint
+}
+
 $1

contrib/backends/srndv2/src/srnd/vendor/github.com/lib/pq/.travis.yml

@@ -1,10 +1,9 @@
 language: go
 
 go:
-  - 1.5.x
-  - 1.6.x
-  - 1.7.x
-  - 1.8.x
+  - 1.9.x
+  - 1.10.x
+  - 1.11.x
   - master
 
 sudo: true
@@ -15,7 +14,9 @@ env:
     - PQGOSSLTESTS=1
     - PQSSLCERTTEST_PATH=$PWD/certs
     - PGHOST=127.0.0.1
+    - MEGACHECK_VERSION=2017.2.2
   matrix:
+    - PGVERSION=10
     - PGVERSION=9.6
     - PGVERSION=9.5
     - PGVERSION=9.4
@@ -30,6 +31,8 @@ before_install:
   - ./.travis.sh postgresql_install
   - ./.travis.sh postgresql_configure
   - ./.travis.sh client_configure
+  - ./.travis.sh megacheck_install
+  - ./.travis.sh golint_install
   - go get golang.org/x/tools/cmd/goimports
 
 before_script:
@@ -41,5 +44,7 @@ script:
   - >
     goimports -d -e $(find -name '*.go') | awk '{ print } END { exit NR == 0 ? 0 : 1 }'
   - go vet ./...
+  - megacheck -go 1.9 ./...
+  - golint ./...
   - PQTEST_BINARY_PARAMETERS=no  go test -race -v ./...
   - PQTEST_BINARY_PARAMETERS=yes go test -race -v ./...

contrib/backends/srndv2/src/srnd/vendor/github.com/lib/pq/README.md

@@ -1,5 +1,6 @@
 # pq - A pure Go postgres driver for Go's database/sql package
 
+[![GoDoc](https://godoc.org/github.com/lib/pq?status.svg)](https://godoc.org/github.com/lib/pq)
 [![Build Status](https://travis-ci.org/lib/pq.svg?branch=master)](https://travis-ci.org/lib/pq)
 
 ## Install
@@ -9,22 +10,11 @@
 ## Docs
 
 For detailed documentation and basic usage examples, please see the package
-documentation at <http://godoc.org/github.com/lib/pq>.
+documentation at <https://godoc.org/github.com/lib/pq>.
 
 ## Tests
 
-`go test` is used for testing.  A running PostgreSQL server is
-required, with the ability to log in.  The default database to connect
-to test with is "pqgotest," but it can be overridden using environment
-variables.
-
-Example:
-
-	PGHOST=/run/postgresql go test github.com/lib/pq
-
-Optionally, a benchmark suite can be run as part of the tests:
-
-	PGHOST=/run/postgresql go test -bench .
+`go test` is used for testing.  See [TESTS.md](TESTS.md) for more details.
 
 ## Features
 

contrib/backends/srndv2/src/srnd/vendor/github.com/lib/pq/TESTS.md

@@ -0,0 +1,33 @@
+# Tests
+
+## Running Tests
+
+`go test` is used for testing. A running PostgreSQL
+server is required, with the ability to log in. The
+database to connect to test with is "pqgotest," on
+"localhost" but these can be overridden using [environment
+variables](https://www.postgresql.org/docs/9.3/static/libpq-envars.html).
+
+Example:
+
+	PGHOST=/run/postgresql go test
+
+## Benchmarks
+
+A benchmark suite can be run as part of the tests:
+
+	go test -bench .
+
+## Example setup (Docker)
+
+Run a postgres container:
+
+```
+docker run --expose 5432:5432 postgres
+```
+
+Run tests:
+
+```
+PGHOST=localhost PGPORT=5432 PGUSER=postgres PGSSLMODE=disable PGDATABASE=postgres go test
+```

contrib/backends/srndv2/src/srnd/vendor/github.com/lib/pq/array.go

@@ -13,7 +13,7 @@ import (
 
 var typeByteSlice = reflect.TypeOf([]byte{})
 var typeDriverValuer = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
-var typeSqlScanner = reflect.TypeOf((*sql.Scanner)(nil)).Elem()
+var typeSQLScanner = reflect.TypeOf((*sql.Scanner)(nil)).Elem()
 
 // Array returns the optimal driver.Valuer and sql.Scanner for an array or
 // slice of any dimension.
@@ -278,7 +278,7 @@ func (GenericArray) evaluateDestination(rt reflect.Type) (reflect.Type, func([]b
 	// TODO calculate the assign function for other types
 	// TODO repeat this section on the element type of arrays or slices (multidimensional)
 	{
-		if reflect.PtrTo(rt).Implements(typeSqlScanner) {
+		if reflect.PtrTo(rt).Implements(typeSQLScanner) {
 			// dest is always addressable because it is an element of a slice.
 			assign = func(src []byte, dest reflect.Value) (err error) {
 				ss := dest.Addr().Interface().(sql.Scanner)
@@ -587,7 +587,7 @@ func appendArrayElement(b []byte, rv reflect.Value) ([]byte, string, error) {
 		}
 	}
 
-	var del string = ","
+	var del = ","
 	var err error
 	var iv interface{} = rv.Interface()
 

contrib/backends/srndv2/src/srnd/vendor/github.com/lib/pq/array_test.go

@@ -89,9 +89,7 @@ func TestParseArrayError(t *testing.T) {
 }
 
 func TestArrayScanner(t *testing.T) {
-	var s sql.Scanner
-
-	s = Array(&[]bool{})
+	var s sql.Scanner = Array(&[]bool{})
 	if _, ok := s.(*BoolArray); !ok {
 		t.Errorf("Expected *BoolArray, got %T", s)
 	}
@@ -126,9 +124,7 @@ func TestArrayScanner(t *testing.T) {
 }
 
 func TestArrayValuer(t *testing.T) {
-	var v driver.Valuer
-
-	v = Array([]bool{})
+	var v driver.Valuer = Array([]bool{})
 	if _, ok := v.(*BoolArray); !ok {
 		t.Errorf("Expected *BoolArray, got %T", v)
 	}
@@ -1193,9 +1189,7 @@ func TestGenericArrayValue(t *testing.T) {
 }
 
 func TestGenericArrayValueErrors(t *testing.T) {
-	var v []interface{}
-
-	v = []interface{}{func() {}}
+	v := []interface{}{func() {}}
 	if _, err := (GenericArray{v}).Value(); err == nil {
 		t.Errorf("Expected error for %q, got nil", v)
 	}