Merge pull request #1694 from oxen-io/dev

0.9.5

CMakeLists.txt

@@ -16,7 +16,7 @@ if(CCACHE_PROGRAM)
 endif()
 
 project(lokinet
-    VERSION 0.9.4
+    VERSION 0.9.5
     DESCRIPTION "lokinet - IP packet onion router"
     LANGUAGES C CXX)
 
@@ -141,7 +141,7 @@ if(LIBUV_FOUND AND NOT BUILD_STATIC_DEPS)
   target_link_libraries(libuv INTERFACE PkgConfig::LIBUV)
 else()
   if(NOT BUILD_STATIC_DEPS)
-    message(FATAL_ERROR "Could not find libu >= 1.28.0; install it on your system or use -DBUILD_STATIC_DEPS=ON")
+    message(FATAL_ERROR "Could not find libuv >= 1.28.0; install it on your system or use -DBUILD_STATIC_DEPS=ON")
   endif()
 endif()
 
@@ -302,6 +302,19 @@ endif()
 add_subdirectory(external)
 include_directories(SYSTEM external/sqlite_orm/include)
 
+option(USE_JEMALLOC "Link to jemalloc for memory allocations, if found" ON)
+if(USE_JEMALLOC AND NOT STATIC_LINK)
+  pkg_check_modules(JEMALLOC jemalloc IMPORTED_TARGET)
+  if(JEMALLOC_FOUND)
+    target_link_libraries(base_libs INTERFACE PkgConfig::JEMALLOC)
+  else()
+    message(STATUS "jemalloc not found, not linking to jemalloc")
+  endif()
+else()
+  message(STATUS "jemalloc support disabled")
+endif()
+
+
 if(ANDROID)
   target_link_libraries(base_libs INTERFACE log)
   target_compile_definitions(base_libs INTERFACE ANDROID)

contrib/ci/docker/nodejs.dockerfile

@@ -0,0 +1,3 @@
+FROM node:14.16.1
+RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
+RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ wine'

contrib/ci/docker/rebuild-docker-images.sh

@@ -8,6 +8,6 @@ test "x$registry" != "x" || exit 1
 for file in ${@:2} ; do
     name="$(echo $file | cut -d'.' -f1)"
     echo "rebuild $name"
-    docker build -f $file -t $registry/lokinet-ci-$name --pull --no-cache --quiet .
+    docker build -f $file -t $registry/lokinet-ci-$name .
     docker push $registry/lokinet-ci-$name
 done

contrib/py/admin/lokinetmon

@@ -6,6 +6,7 @@ import sys
 import time
 import platform
 import os
+import re
 from argparse import ArgumentParser as AP
 
 is_windows = lambda : platform.system().lower() == 'windows'
@@ -40,12 +41,13 @@ except ImportError:
         print("for other linuxs do:")
         print("\tpip3 install --user geoip")
         print("for other linuxs you are responsible for obtaining your owen geoip databases, glhf")
+        time.sleep(1)
     else:
         print("install it with:")
         print("\tpip3 install --user geoip")
-    print("")
-    print("press enter to continue without geoip")
-    sys.stdin.read(1)
+        print()
+        print("press enter to continue without geoip")
+        sys.stdin.read(1)
 else:
     try:
         geoip_env_var = 'GEOIP_DB_FILE'
@@ -86,6 +88,7 @@ def ip_to_flag(ip):
 class Monitor:
 
     _sample_size = 12
+    filter = lambda x : True
 
     def __init__(self, url, introsetMode=False):
         self.txrate = 0
@@ -144,7 +147,7 @@ class Monitor:
             self.win.addstr(" {} ->".format(hopstr))
 
         self.win.addstr(" [{} ms latency]".format(path["intro"]["latency"]))
-        self.win.addstr(" [{} until expire]".format(self.time_to(path["expiresAt"])))
+        self.win.addstr(" [expires: {}]".format(self.time_to(path["expiresAt"])))
         if path["expiresSoon"]:
             self.win.addstr("(expiring)")
         elif path["expired"]:
@@ -153,13 +156,17 @@ class Monitor:
 
     @staticmethod
     def time_to(timestamp):
-        """ return time until timestamp in seconds formatted"""
+        """ return time until timestamp formatted"""
         if timestamp:
+            unit = 'seconds'
             val = (timestamp - now()) / 1000.0
+            if abs(val) > 60.0:
+                val /= 60.0
+                unit = 'minutes'
             if val < 0:
-                return "{} seconds ago".format(0-val)
+                return "{:.2f} {} ago".format(0-val, unit)
             else:
-                return "{} seconds".format(val)
+                return "in {:.2f} {}".format(val, unit)
         else:
             return 'never'
 
@@ -201,15 +208,18 @@ class Monitor:
         paths = status["paths"]
         self.win.addstr("paths: {}".format(len(paths)))
         for path in paths:
-            y_pos = self._render_path(y_pos, path, "inbound")
+            if self.filter('localhost.loki'):
+                y_pos = self._render_path(y_pos, path, "localhost.loki")
         for session in (status["remoteSessions"] or []):
             for path in session["paths"]:
-                y_pos = self._render_path(
-                    y_pos, path, "[active] {}".format(session["currentConvoTag"])
-                )
+                if self.filter(session["remoteIdentity"]):
+                    y_pos = self._render_path(
+                        y_pos, path, "[active] {}".format(session["currentConvoTag"])
+                    )
         for session in (status["snodeSessions"] or []):
             for path in session["paths"]:
-                y_pos = self._render_path(y_pos, path, "[snode]")
+                if self.filter(session["endpoint"]):
+                    y_pos = self._render_path(y_pos, path, "[snode]")
         return y_pos
 
     def display_links(self, y_pos, data):
@@ -407,18 +417,20 @@ class Monitor:
         """
         y_pos += 1
         self.win.move(y_pos, 1)
-        self.win.addstr("localhost.loki")
-        y_pos = self._display_our_introset(y_pos, service)
-        y_pos += 1
+        if self.filter("localhost.loki"):
+            self.win.addstr("localhost.loki")
+            y_pos = self._display_our_introset(y_pos, service)
+            y_pos += 1
         remotes = service['remoteSessions'] or []
         for session in remotes:
-            y_pos = self._display_session_introset(y_pos, session)
+            if self.filter(session['remoteIdentity']):
+                y_pos = self._display_session_introset(y_pos, session)
 
     def _display_intro(self, y_pos, intro, label, paths):
         y_pos += 1
         self.win.move(y_pos, 1)
         path = 'path' in intro and intro['path'][:4] or '????'
-        self.win.addstr('{}: ({}|{}) [expires in: {}] [{} paths]'.format(label, intro['router'][:8], path, self.time_to(intro['expiresAt']), self.count_endpoints_in_path(paths, intro['router'])))
+        self.win.addstr('{}: ({}|{}) [expires: {}] [{} paths]'.format(label, intro['router'][:8], path, self.time_to(intro['expiresAt']), self.count_endpoints_in_path(paths, intro['router'])))
         return y_pos
 
     @staticmethod
@@ -457,10 +469,13 @@ class Monitor:
         #print(context.keys())
         y_pos += 1
         self.win.move(y_pos, 1)
-        readyState = context['readyToSend'] and '✔️' or '❌'
+        readyState = context['readyToSend'] and '️✅' or '❌'
         self.win.addstr('{} ({}) [{}]'.format(context['remoteIdentity'], context['currentConvoTag'], readyState))
         y_pos += 1
         self.win.move(y_pos, 1)
+        self.win.addstr('created: {}'.format(self.time_to(context['sessionCreatedAt'])))
+        y_pos += 1
+        self.win.move(y_pos, 1)
         self.win.addstr('last good send: {}'.format(self.time_to(context['lastGoodSend'])))
         y_pos += 1
         self.win.move(y_pos, 1)
@@ -544,13 +559,19 @@ def main():
 
     ap.add_argument("--introset", action='store_const', const=True, default=False, help="run in introset inspection mode")
     ap.add_argument("--url", default='tcp://127.0.0.1:1190', type=str, help='url to lokinet rpc')
+    ap.add_argument('--filter', default='.+', type=str, help="regex to filter entries")
+    ap.add_argument('--invert-filter', const=True, default=False, action='store_const', help='invert regex filter matching')
 
     args = ap.parse_args()
 
     mon = Monitor(
         args.url,
         args.introset
     )
+    mon.filter = lambda x : re.match(args.filter, x) is not None
+    if args.invert_filter:
+        old_filter = mon.filter
+        mon.filter = lambda x : not old_filter(x)
     mon.run()
 
 if __name__ == "__main__":

docs/CMakeLists.txt

@@ -4,7 +4,7 @@ if (NOT DOXYGEN)
     return()
 endif()
 find_program(SPHINX_BUILD sphinx-build)
-if (NOT DOXYGEN)
+if (NOT SPHINX_BUILD)
     message(STATUS "Documentation generation disabled (sphinx-build not found)")
     return()
   endif()
@@ -22,7 +22,7 @@ add_custom_command(
 
 add_custom_command(
     OUTPUT html/index.html
-    COMMAND ${SPHINX_BUILD}
+    COMMAND ${SPHINX_BUILD} -j auto
         -Dbreathe_projects.lokinet=${CMAKE_CURRENT_BINARY_DIR}/doxyxml
         -Dversion=${lokinet_VERSION} -Drelease=${lokinet_VERSION}
         -Aversion=${lokinet_VERSION} -Aversions=${lokinet_VERSION_MAJOR},${lokinet_VERSION_MINOR},${lokinet_VERSION_PATCH}

llarp/exit/session.cpp

@@ -46,6 +46,7 @@ namespace llarp
       obj["lastExitUse"] = to_json(m_LastUse);
       auto pub = m_ExitIdentity.toPublic();
       obj["exitIdentity"] = pub.ToString();
+      obj["endpoint"] = m_ExitRouter.ToString();
       return obj;
     }
 

llarp/iwp/linklayer.cpp

@@ -24,7 +24,7 @@ namespace llarp::iwp
           keyManager, getrc, h, sign, before, est, reneg, timeout, closed, pumpDone, worker)
       , m_Wakeup{ev->make_waker([this]() { HandleWakeupPlaintext(); })}
       , m_PlaintextRecv{1024}
-      , permitInbound{allowInbound}
+      , m_Inbound{allowInbound}
 
   {}
 
@@ -34,6 +34,15 @@ namespace llarp::iwp
     return "iwp";
   }
 
+  std::string
+  LinkLayer::PrintableName() const
+  {
+    if (m_Inbound)
+      return "inbound iwp link";
+    else
+      return "outbound iwp link";
+  }
+
   uint16_t
   LinkLayer::Rank() const
   {
@@ -48,10 +57,10 @@ namespace llarp::iwp
     bool isNewSession = false;
     if (itr == m_AuthedAddrs.end())
     {
-      Lock_t lock(m_PendingMutex);
+      Lock_t lock{m_PendingMutex};
       if (m_Pending.count(from) == 0)
       {
-        if (not permitInbound)
+        if (not m_Inbound)
           return;
         isNewSession = true;
         m_Pending.insert({from, std::make_shared<Session>(this, from)});
@@ -60,14 +69,13 @@ namespace llarp::iwp
     }
     else
     {
-      Lock_t lock(m_AuthedLinksMutex);
-      auto range = m_AuthedLinks.equal_range(itr->second);
-      session = range.first->second;
+      if (auto s_itr = m_AuthedLinks.find(itr->second); s_itr != m_AuthedLinks.end())
+        session = s_itr->second;
     }
     if (session)
     {
       bool success = session->Recv_LL(std::move(pkt));
-      if (!success and isNewSession)
+      if (not success and isNewSession)
       {
         LogWarn("Brand new session failed; removing from pending sessions list");
         m_Pending.erase(m_Pending.find(from));
@@ -78,7 +86,7 @@ namespace llarp::iwp
   bool
   LinkLayer::MapAddr(const RouterID& r, ILinkSession* s)
   {
-    if (!ILinkLayer::MapAddr(r, s))
+    if (not ILinkLayer::MapAddr(r, s))
       return false;
     m_AuthedAddrs.emplace(s->GetRemoteEndpoint(), r);
     return true;
@@ -93,6 +101,8 @@ namespace llarp::iwp
   std::shared_ptr<ILinkSession>
   LinkLayer::NewOutboundSession(const RouterContact& rc, const AddressInfo& ai)
   {
+    if (m_Inbound)
+      throw std::logic_error{"inbound link cannot make outbound sessions"};
     return std::make_shared<Session>(this, rc, ai);
   }
 

llarp/iwp/linklayer.hpp

@@ -56,14 +56,17 @@ namespace llarp::iwp
     void
     AddWakeup(std::weak_ptr<Session> peer);
 
+    std::string
+    PrintableName() const;
+
    private:
     void
     HandleWakeupPlaintext();
 
     const std::shared_ptr<EventLoopWakeup> m_Wakeup;
     std::unordered_map<SockAddr, std::weak_ptr<Session>> m_PlaintextRecv;
     std::unordered_map<SockAddr, RouterID> m_AuthedAddrs;
-    const bool permitInbound;
+    const bool m_Inbound;
   };
 
   using LinkLayer_ptr = std::shared_ptr<LinkLayer>;