First Commit

controller/ConnectionPool.hpp

@@ -0,0 +1,175 @@
+/*
+ * Copyright (c)2021 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_CONNECTION_POOL_H_
+#define ZT_CONNECTION_POOL_H_
+
+
+#ifndef _DEBUG
+	#define _DEBUG(x)
+#endif
+
+#include "../node/Metrics.hpp"
+
+#include <deque>
+#include <set>
+#include <memory>
+#include <mutex>
+#include <exception>
+#include <string>
+
+namespace ZeroTier {
+
+struct ConnectionUnavailable : std::exception { 
+    char const* what() const throw() {
+        return "Unable to allocate connection";
+    }; 
+};
+
+
+class Connection {
+public:
+    virtual ~Connection() {};
+};
+
+class ConnectionFactory {
+public:
+    virtual ~ConnectionFactory() {};
+    virtual std::shared_ptr<Connection> create()=0;
+};
+
+struct ConnectionPoolStats {
+    size_t pool_size;
+    size_t borrowed_size;
+};
+
+template<class T>
+class ConnectionPool {
+public:
+    ConnectionPool(size_t max_pool_size, size_t min_pool_size, std::shared_ptr<ConnectionFactory> factory)
+        : m_maxPoolSize(max_pool_size)
+        , m_minPoolSize(min_pool_size)
+        , m_factory(factory)
+    {
+        Metrics::max_pool_size += max_pool_size;
+        Metrics::min_pool_size += min_pool_size;
+        while(m_pool.size() < m_minPoolSize){
+            m_pool.push_back(m_factory->create());
+            Metrics::pool_avail++;
+        }
+    };
+
+    ConnectionPoolStats get_stats() {
+        std::unique_lock<std::mutex> lock(m_poolMutex);
+
+        ConnectionPoolStats stats;
+        stats.pool_size = m_pool.size();
+        stats.borrowed_size = m_borrowed.size();			
+
+        return stats;
+    };
+
+    ~ConnectionPool() {
+    };
+
+    /**
+     * Borrow
+     *
+     * Borrow a connection for temporary use
+     *
+     * When done, either (a) call unborrow() to return it, or (b) (if it's bad) just let it go out of scope.  This will cause it to automatically be replaced.
+     * @retval a shared_ptr to the connection object
+     */
+    std::shared_ptr<T> borrow() {
+        std::unique_lock<std::mutex> l(m_poolMutex);
+        
+        while((m_pool.size() + m_borrowed.size()) < m_minPoolSize) {
+            std::shared_ptr<Connection> conn = m_factory->create();
+            m_pool.push_back(conn);
+            Metrics::pool_avail++;
+        }
+
+        if(m_pool.size()==0){
+            
+            if ((m_pool.size() + m_borrowed.size()) < m_maxPoolSize) {
+                try {
+                    std::shared_ptr<Connection> conn = m_factory->create();
+                    m_borrowed.insert(conn);
+                    Metrics::pool_in_use++;
+                    return std::static_pointer_cast<T>(conn);
+                } catch (std::exception &e) {
+                    Metrics::pool_errors++;
+                    throw ConnectionUnavailable();
+                }
+            } else {
+                for(auto it = m_borrowed.begin(); it != m_borrowed.end(); ++it){
+                    if((*it).unique()) {
+                        // This connection has been abandoned! Destroy it and create a new connection
+                        try {
+                            // If we are able to create a new connection, return it
+                            _DEBUG("Creating new connection to replace discarded connection");
+                            std::shared_ptr<Connection> conn = m_factory->create();
+                            m_borrowed.erase(it);
+                            m_borrowed.insert(conn);
+                            return std::static_pointer_cast<T>(conn);
+                        } catch(std::exception& e) {
+                            // Error creating a replacement connection
+                            Metrics::pool_errors++;
+                            throw ConnectionUnavailable();
+                        }
+                    }
+                }
+                // Nothing available
+                Metrics::pool_errors++;
+                throw ConnectionUnavailable();
+            }
+        }
+
+        // Take one off the front
+        std::shared_ptr<Connection> conn = m_pool.front();
+        m_pool.pop_front();
+        Metrics::pool_avail--;
+        // Add it to the borrowed list
+        m_borrowed.insert(conn);
+        Metrics::pool_in_use++;
+        return std::static_pointer_cast<T>(conn);
+    };
+
+    /**
+     * Unborrow a connection
+     *
+     * Only call this if you are returning a working connection.  If the connection was bad, just let it go out of scope (so the connection manager can replace it).
+     * @param the connection
+     */
+    void unborrow(std::shared_ptr<T> conn) {
+        // Lock
+        std::unique_lock<std::mutex> lock(m_poolMutex);
+        m_borrowed.erase(conn);
+        Metrics::pool_in_use--;
+        if ((m_pool.size() + m_borrowed.size()) < m_maxPoolSize) {
+            Metrics::pool_avail++;
+            m_pool.push_back(conn);
+        }
+    };
+protected:
+    size_t m_maxPoolSize;
+    size_t m_minPoolSize;
+    std::shared_ptr<ConnectionFactory> m_factory;
+    std::deque<std::shared_ptr<Connection> > m_pool;
+    std::set<std::shared_ptr<Connection> > m_borrowed;
+    std::mutex m_poolMutex;
+};
+
+}
+
+#endif

controller/DB.cpp

@@ -0,0 +1,422 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "DB.hpp"
+#include "EmbeddedNetworkController.hpp"
+#include "../node/Metrics.hpp"
+
+#include <chrono>
+#include <algorithm>
+#include <stdexcept>
+
+using json = nlohmann::json;
+
+namespace ZeroTier {
+
+void DB::initNetwork(nlohmann::json &network)
+{
+	if (!network.count("private")) network["private"] = true;
+	if (!network.count("creationTime")) network["creationTime"] = OSUtils::now();
+	if (!network.count("name")) network["name"] = "";
+	if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32;
+	if (!network.count("enableBroadcast")) network["enableBroadcast"] = true;
+	if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}};
+	if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}};
+	if (!network.count("authTokens")) network["authTokens"] = {{}};
+	if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array();
+	if (!network.count("tags")) network["tags"] = nlohmann::json::array();
+	if (!network.count("routes")) network["routes"] = nlohmann::json::array();
+	if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array();
+	if (!network.count("mtu")) network["mtu"] = ZT_DEFAULT_MTU;
+	if (!network.count("remoteTraceTarget")) network["remoteTraceTarget"] = nlohmann::json();
+	if (!network.count("removeTraceLevel")) network["remoteTraceLevel"] = 0;
+	if (!network.count("rulesSource")) network["rulesSource"] = "";
+	if (!network.count("rules")) {
+		// If unspecified, rules are set to allow anything and behave like a flat L2 segment
+		network["rules"] = {{
+			{ "not",false },
+			{ "or", false },
+			{ "type","ACTION_ACCEPT" }
+		}};
+	}
+	if (!network.count("dns")) network["dns"] = nlohmann::json::array();
+	if (!network.count("ssoEnabled")) network["ssoEnabled"] = false;
+	if (!network.count("clientId")) network["clientId"] = "";
+	if (!network.count("authorizationEndpoint")) network["authorizationEndpoint"] = "";
+
+	network["objtype"] = "network";
+}
+
+void DB::initMember(nlohmann::json &member)
+{
+	if (!member.count("authorized")) member["authorized"] = false;
+	if (!member.count("ssoExempt")) member["ssoExempt"] = false;
+	if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array();
+	if (!member.count("activeBridge")) member["activeBridge"] = false;
+	if (!member.count("tags")) member["tags"] = nlohmann::json::array();
+	if (!member.count("capabilities")) member["capabilities"] = nlohmann::json::array();
+	if (!member.count("creationTime")) member["creationTime"] = OSUtils::now();
+	if (!member.count("noAutoAssignIps")) member["noAutoAssignIps"] = false;
+	if (!member.count("revision")) member["revision"] = 0ULL;
+	if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL;
+	if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL;
+	if (!member.count("lastAuthorizedCredentialType")) member["lastAuthorizedCredentialType"] = nlohmann::json();
+	if (!member.count("lastAuthorizedCredential")) member["lastAuthorizedCredential"] = nlohmann::json();
+	if (!member.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = 0LL;
+	if (!member.count("vMajor")) member["vMajor"] = -1;
+	if (!member.count("vMinor")) member["vMinor"] = -1;
+	if (!member.count("vRev")) member["vRev"] = -1;
+	if (!member.count("vProto")) member["vProto"] = -1;
+	if (!member.count("remoteTraceTarget")) member["remoteTraceTarget"] = nlohmann::json();
+	if (!member.count("removeTraceLevel")) member["remoteTraceLevel"] = 0;
+	member["objtype"] = "member";
+}
+
+void DB::cleanNetwork(nlohmann::json &network)
+{
+	network.erase("clock");
+	network.erase("authorizedMemberCount");
+	network.erase("activeMemberCount");
+	network.erase("totalMemberCount");
+	network.erase("lastModified");
+}
+
+void DB::cleanMember(nlohmann::json &member)
+{
+	member.erase("clock");
+	member.erase("physicalAddr");
+	member.erase("recentLog");
+	member.erase("lastModified");
+	member.erase("lastRequestMetaData");
+	member.erase("authenticationURL"); // computed
+	member.erase("authenticationClientID"); // computed
+}
+
+DB::DB() {}
+DB::~DB() {}
+
+bool DB::get(const uint64_t networkId,nlohmann::json &network)
+{
+	waitForReady();
+	Metrics::db_get_network++;
+	std::shared_ptr<_Network> nw;
+	{
+		std::shared_lock<std::shared_mutex> l(_networks_l);
+		auto nwi = _networks.find(networkId);
+		if (nwi == _networks.end())
+			return false;
+		nw = nwi->second;
+	}
+	{
+		std::shared_lock<std::shared_mutex> l2(nw->lock);
+		network = nw->config;
+	}
+	return true;
+}
+
+bool DB::get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member)
+{
+	waitForReady();
+	Metrics::db_get_network_and_member++;
+	std::shared_ptr<_Network> nw;
+	{
+		std::shared_lock<std::shared_mutex> l(_networks_l);
+		auto nwi = _networks.find(networkId);
+		if (nwi == _networks.end())
+			return false;
+		nw = nwi->second;
+	}
+	{
+		std::shared_lock<std::shared_mutex> l2(nw->lock);
+		network = nw->config;
+		auto m = nw->members.find(memberId);
+		if (m == nw->members.end())
+			return false;
+		member = m->second;
+	}
+	return true;
+}
+
+bool DB::get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,NetworkSummaryInfo &info)
+{
+	waitForReady();
+	Metrics::db_get_network_and_member_and_summary++;
+	std::shared_ptr<_Network> nw;
+	{
+		std::shared_lock<std::shared_mutex> l(_networks_l);
+		auto nwi = _networks.find(networkId);
+		if (nwi == _networks.end())
+			return false;
+		nw = nwi->second;
+	}
+	{
+		std::shared_lock<std::shared_mutex> l2(nw->lock);
+		network = nw->config;
+		_fillSummaryInfo(nw,info);
+		auto m = nw->members.find(memberId);
+		if (m == nw->members.end())
+			return false;
+		member = m->second;
+	}
+	return true;
+}
+
+bool DB::get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohmann::json> &members)
+{
+	waitForReady();
+	Metrics::db_get_member_list++;
+	std::shared_ptr<_Network> nw;
+	{
+		std::shared_lock<std::shared_mutex> l(_networks_l);
+		auto nwi = _networks.find(networkId);
+		if (nwi == _networks.end())
+			return false;
+		nw = nwi->second;
+	}
+	{
+		std::shared_lock<std::shared_mutex> l2(nw->lock);
+		network = nw->config;
+		for(auto m=nw->members.begin();m!=nw->members.end();++m) {
+			members.push_back(m->second);
+		}
+	}
+	return true;
+}
+
+void DB::networks(std::set<uint64_t> &networks)
+{
+	waitForReady();
+	Metrics::db_get_network_list++;
+	std::shared_lock<std::shared_mutex> l(_networks_l);
+	for(auto n=_networks.begin();n!=_networks.end();++n)
+		networks.insert(n->first);
+}
+
+void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners)
+{
+	Metrics::db_member_change++;
+	uint64_t memberId = 0;
+	uint64_t networkId = 0;
+	bool isAuth = false;
+	bool wasAuth = false;
+	std::shared_ptr<_Network> nw;
+
+	if (old.is_object()) {
+		memberId = OSUtils::jsonIntHex(old["id"],0ULL);
+		networkId = OSUtils::jsonIntHex(old["nwid"],0ULL);
+		if ((memberId)&&(networkId)) {
+			{
+				std::unique_lock<std::shared_mutex> l(_networks_l);
+				auto nw2 = _networks.find(networkId);
+				if (nw2 != _networks.end()) {
+					nw = nw2->second;
+				}
+			}
+			if (nw) {
+				std::unique_lock<std::shared_mutex> l(nw->lock);
+				if (OSUtils::jsonBool(old["activeBridge"],false)) {
+					nw->activeBridgeMembers.erase(memberId);
+				}
+				wasAuth = OSUtils::jsonBool(old["authorized"],false);
+				if (wasAuth) {
+					nw->authorizedMembers.erase(memberId);
+				}
+				json &ips = old["ipAssignments"];
+				if (ips.is_array()) {
+					for(unsigned long i=0;i<ips.size();++i) {
+						json &ipj = ips[i];
+						if (ipj.is_string()) {
+							const std::string ips = ipj;
+							InetAddress ipa(ips.c_str());
+							ipa.setPort(0);
+							nw->allocatedIps.erase(ipa);
+						}
+					}
+				}
+			}
+		}
+	}
+
+	if (memberConfig.is_object()) {
+		if (!nw) {
+			memberId = OSUtils::jsonIntHex(memberConfig["id"],0ULL);
+			networkId = OSUtils::jsonIntHex(memberConfig["nwid"],0ULL);
+			if ((!memberId)||(!networkId))
+				return;
+			std::unique_lock<std::shared_mutex> l(_networks_l);
+			std::shared_ptr<_Network> &nw2 = _networks[networkId];
+			if (!nw2)
+				nw2.reset(new _Network);
+			nw = nw2;
+		}
+
+		{
+			std::unique_lock<std::shared_mutex> l(nw->lock);
+
+			nw->members[memberId] = memberConfig;
+
+			if (OSUtils::jsonBool(memberConfig["activeBridge"],false)) {
+				nw->activeBridgeMembers.insert(memberId);
+			}
+			isAuth = OSUtils::jsonBool(memberConfig["authorized"],false);
+			if (isAuth) {
+				Metrics::member_auths++;
+				nw->authorizedMembers.insert(memberId);
+			}
+			json &ips = memberConfig["ipAssignments"];
+			if (ips.is_array()) {
+				for(unsigned long i=0;i<ips.size();++i) {
+					json &ipj = ips[i];
+					if (ipj.is_string()) {
+						const std::string ips = ipj;
+						InetAddress ipa(ips.c_str());
+						ipa.setPort(0);
+						nw->allocatedIps.insert(ipa);
+					}
+				}
+			}
+
+			if (!isAuth) {
+				const int64_t ldt = (int64_t)OSUtils::jsonInt(memberConfig["lastDeauthorizedTime"],0ULL);
+				if (ldt > nw->mostRecentDeauthTime)
+					nw->mostRecentDeauthTime = ldt;
+			}
+		}
+
+		if (notifyListeners) {
+			std::unique_lock<std::shared_mutex> ll(_changeListeners_l);
+			for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) {
+				(*i)->onNetworkMemberUpdate(this,networkId,memberId,memberConfig);
+			}
+		}
+	} else if (memberId) {
+		if (nw) {
+			std::unique_lock<std::shared_mutex> l(nw->lock);
+			nw->members.erase(memberId);
+		}
+		if (networkId) {
+			std::unique_lock<std::shared_mutex> l(_networks_l);
+			auto er = _networkByMember.equal_range(memberId);
+			for(auto i=er.first;i!=er.second;++i) {
+				if (i->second == networkId) {
+					_networkByMember.erase(i);
+					break;
+				}
+			}
+		}
+	}
+
+	if (notifyListeners) {
+		if(networkId != 0 && memberId != 0 && old.is_object() && !memberConfig.is_object()) {
+			// member delete
+			Metrics::member_count--;
+		} else if (networkId != 0 && memberId != 0 && !old.is_object() && memberConfig.is_object()) {
+			// new member
+			Metrics::member_count++;
+		}
+
+		if (!wasAuth && isAuth) {
+			Metrics::member_auths++;
+		} else if (wasAuth && !isAuth) {
+			Metrics::member_deauths++;
+		} else {
+			Metrics::member_changes++;
+		}
+	}
+
+	if ((notifyListeners)&&((wasAuth)&&(!isAuth)&&(networkId)&&(memberId))) {
+		std::unique_lock<std::shared_mutex> ll(_changeListeners_l);
+		for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) {
+			(*i)->onNetworkMemberDeauthorize(this,networkId,memberId);
+		}
+	}
+}
+
+void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners)
+{
+	Metrics::db_network_change++;
+	if (notifyListeners) {
+		if (old.is_object() && old.contains("id") && networkConfig.is_object() && networkConfig.contains("id")) {
+			Metrics::network_changes++;
+		} else if (!old.is_object() && networkConfig.is_object() && networkConfig.contains("id")) {
+			Metrics::network_count++;
+		} else if (old.is_object() && old.contains("id") && !networkConfig.is_object()) {
+			Metrics::network_count--;
+		}
+	}
+
+	if (networkConfig.is_object()) {
+		const std::string ids = networkConfig["id"];
+		const uint64_t networkId = Utils::hexStrToU64(ids.c_str());
+		if (networkId) {
+			std::shared_ptr<_Network> nw;
+			{
+				std::unique_lock<std::shared_mutex> l(_networks_l);
+				std::shared_ptr<_Network> &nw2 = _networks[networkId];
+				if (!nw2)
+					nw2.reset(new _Network);
+				nw = nw2;
+			}
+			{
+				std::unique_lock<std::shared_mutex> l2(nw->lock);
+				nw->config = networkConfig;
+			}
+			if (notifyListeners) {
+				std::unique_lock<std::shared_mutex> ll(_changeListeners_l);
+				for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) {
+					(*i)->onNetworkUpdate(this,networkId,networkConfig);
+				}
+			}
+		}
+	} else if (old.is_object()) {
+		const std::string ids = old["id"];
+		const uint64_t networkId = Utils::hexStrToU64(ids.c_str());
+		if (networkId) {
+			try {
+				// deauth all members on the network
+				nlohmann::json network;
+				std::vector<nlohmann::json> members;
+				this->get(networkId, network, members);
+				for(auto i=members.begin();i!=members.end();++i) {
+					const std::string nodeID = (*i)["id"];
+					const uint64_t memberId = Utils::hexStrToU64(nodeID.c_str());
+					std::unique_lock<std::shared_mutex> ll(_changeListeners_l);
+					for(auto j=_changeListeners.begin();j!=_changeListeners.end();++j) {
+						(*j)->onNetworkMemberDeauthorize(this,networkId,memberId);
+					}
+				}
+			} catch (std::exception &e) {
+				std::cerr << "Error deauthorizing members on network delete: " << e.what() << std::endl;
+			}
+
+			// delete the network
+			std::unique_lock<std::shared_mutex> l(_networks_l);
+			_networks.erase(networkId);
+		}
+	}
+}
+
+void DB::_fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info)
+{
+	for(auto ab=nw->activeBridgeMembers.begin();ab!=nw->activeBridgeMembers.end();++ab)
+		info.activeBridges.push_back(Address(*ab));
+	std::sort(info.activeBridges.begin(),info.activeBridges.end());
+	for(auto ip=nw->allocatedIps.begin();ip!=nw->allocatedIps.end();++ip)
+		info.allocatedIps.push_back(*ip);
+	std::sort(info.allocatedIps.begin(),info.allocatedIps.end());
+	info.authorizedMemberCount = (unsigned long)nw->authorizedMembers.size();
+	info.totalMemberCount = (unsigned long)nw->members.size();
+	info.mostRecentDeauthTime = nw->mostRecentDeauthTime;
+}
+
+} // namespace ZeroTier

controller/DB.hpp

@@ -0,0 +1,197 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_CONTROLLER_DB_HPP
+#define ZT_CONTROLLER_DB_HPP
+
+//#define ZT_CONTROLLER_USE_LIBPQ
+
+#include "../node/Constants.hpp"
+#include "../node/Identity.hpp"
+#include "../node/InetAddress.hpp"
+#include "../osdep/OSUtils.hpp"
+#include "../osdep/BlockingQueue.hpp"
+
+#include <memory>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+#include <atomic>
+#include <shared_mutex>
+#include <set>
+#include <map>
+
+#include <nlohmann/json.hpp>
+
+#include <prometheus/simpleapi.h>
+
+#define ZT_MEMBER_AUTH_TIMEOUT_NOTIFY_BEFORE 25000
+
+namespace ZeroTier
+{
+
+struct AuthInfo
+{
+public:
+	AuthInfo() 
+	: enabled(false)
+	, version(0)
+	, authenticationURL()
+	, authenticationExpiryTime(0)
+	, issuerURL()
+	, centralAuthURL()
+	, ssoNonce()
+	, ssoState()
+	, ssoClientID()
+	, ssoProvider("default")
+	{}
+
+	bool enabled;
+	uint64_t version;
+	std::string authenticationURL;
+	uint64_t authenticationExpiryTime;
+	std::string issuerURL;
+	std::string centralAuthURL;
+	std::string ssoNonce;
+	std::string ssoState;
+	std::string ssoClientID;
+	std::string ssoProvider;
+};
+
+/**
+ * Base class with common infrastructure for all controller DB implementations
+ */
+class DB
+{
+public:
+	class ChangeListener
+	{
+	public:
+		ChangeListener() {}
+		virtual ~ChangeListener() {}
+		virtual void onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network) {}
+		virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member) {}
+		virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId) {}
+	};
+
+	struct NetworkSummaryInfo
+	{
+		NetworkSummaryInfo() : authorizedMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {}
+		std::vector<Address> activeBridges;
+		std::vector<InetAddress> allocatedIps;
+		unsigned long authorizedMemberCount;
+		unsigned long totalMemberCount;
+		int64_t mostRecentDeauthTime;
+	};
+
+	static void initNetwork(nlohmann::json &network);
+	static void initMember(nlohmann::json &member);
+	static void cleanNetwork(nlohmann::json &network);
+	static void cleanMember(nlohmann::json &member);
+
+	DB();
+	virtual ~DB();
+
+	virtual bool waitForReady() = 0;
+	virtual bool isReady() = 0;
+
+	inline bool hasNetwork(const uint64_t networkId) const
+	{
+		std::shared_lock<std::shared_mutex> l(_networks_l);
+		return (_networks.find(networkId) != _networks.end());
+	}
+
+	bool get(const uint64_t networkId,nlohmann::json &network);
+	bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member);
+	bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,NetworkSummaryInfo &info);
+	bool get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohmann::json> &members);
+
+	void networks(std::set<uint64_t> &networks);
+
+	template<typename F>
+	inline void each(F f)
+	{
+		nlohmann::json nullJson;
+		std::unique_lock<std::shared_mutex> lck(_networks_l);
+		for(auto nw=_networks.begin();nw!=_networks.end();++nw) {
+			f(nw->first,nw->second->config,0,nullJson); // first provide network with 0 for member ID
+			for(auto m=nw->second->members.begin();m!=nw->second->members.end();++m) {
+				f(nw->first,nw->second->config,m->first,m->second);
+			}
+		}
+	}
+
+	virtual bool save(nlohmann::json &record,bool notifyListeners) = 0;
+	virtual void eraseNetwork(const uint64_t networkId) = 0;
+	virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0;
+	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0;
+
+	virtual AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL) { return AuthInfo(); }
+
+	inline void addListener(DB::ChangeListener *const listener)
+	{
+		std::unique_lock<std::shared_mutex> l(_changeListeners_l);
+		_changeListeners.push_back(listener);
+	}
+
+protected:
+	static inline bool _compareRecords(const nlohmann::json &a,const nlohmann::json &b)
+	{
+		if (a.is_object() == b.is_object()) {
+			if (a.is_object()) {
+				if (a.size() != b.size())
+					return false;
+				auto amap = a.get<nlohmann::json::object_t>();
+				auto bmap = b.get<nlohmann::json::object_t>();
+				for(auto ai=amap.begin();ai!=amap.end();++ai) {
+					if (ai->first != "revision") { // ignore revision, compare only non-revision-counter fields
+						auto bi = bmap.find(ai->first);
+						if ((bi == bmap.end())||(bi->second != ai->second))
+							return false;
+					}
+				}
+				return true;
+			}
+			return (a == b);
+		}
+		return false;
+	}
+
+	struct _Network
+	{
+		_Network() : mostRecentDeauthTime(0) {}
+		nlohmann::json config;
+		std::unordered_map<uint64_t,nlohmann::json> members;
+		std::unordered_set<uint64_t> activeBridgeMembers;
+		std::unordered_set<uint64_t> authorizedMembers;
+		std::unordered_set<InetAddress,InetAddress::Hasher> allocatedIps;
+		int64_t mostRecentDeauthTime;
+		std::shared_mutex lock;
+	};
+
+	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners);
+	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners);
+	void _fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info);
+
+	std::vector<DB::ChangeListener *> _changeListeners;
+	std::unordered_map< uint64_t,std::shared_ptr<_Network> > _networks;
+	std::unordered_multimap< uint64_t,uint64_t > _networkByMember;
+	mutable std::shared_mutex _changeListeners_l;
+	mutable std::shared_mutex _networks_l;
+};
+
+} // namespace ZeroTier
+
+#endif

controller/DBMirrorSet.cpp

@@ -0,0 +1,246 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "DBMirrorSet.hpp"
+
+namespace ZeroTier {
+
+DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener)
+	: _listener(listener)
+	, _running(true)
+	, _syncCheckerThread()
+	, _dbs()
+	, _dbs_l()
+{
+	_syncCheckerThread = std::thread([this]() {
+		for(;;) {
+			for(int i=0;i<120;++i) { // 1 minute delay between checks
+				if (!_running)
+					return;
+				std::this_thread::sleep_for(std::chrono::milliseconds(500));
+			}
+
+			std::vector< std::shared_ptr<DB> > dbs;
+			{
+				std::unique_lock<std::shared_mutex> l(_dbs_l);
+				if (_dbs.size() <= 1)
+					continue; // no need to do this if there's only one DB, so skip the iteration
+				dbs = _dbs;
+			}
+
+			for(auto db=dbs.begin();db!=dbs.end();++db) {
+				(*db)->each([&dbs,&db](uint64_t networkId,const nlohmann::json &network,uint64_t memberId,const nlohmann::json &member) {
+					try {
+						if (network.is_object()) {
+							if (memberId == 0) {
+								for(auto db2=dbs.begin();db2!=dbs.end();++db2) {
+									if (db->get() != db2->get()) {
+										nlohmann::json nw2;
+										if ((!(*db2)->get(networkId,nw2))||((nw2.is_object())&&(OSUtils::jsonInt(nw2["revision"],0) < OSUtils::jsonInt(network["revision"],0)))) {
+											nw2 = network;
+											(*db2)->save(nw2,false);
+										}
+									}
+								}
+							} else if (member.is_object()) {
+								for(auto db2=dbs.begin();db2!=dbs.end();++db2) {
+									if (db->get() != db2->get()) {
+										nlohmann::json nw2,m2;
+										if ((!(*db2)->get(networkId,nw2,memberId,m2))||((m2.is_object())&&(OSUtils::jsonInt(m2["revision"],0) < OSUtils::jsonInt(member["revision"],0)))) {
+											m2 = member;
+											(*db2)->save(m2,false);
+										}
+									}
+								}
+							}
+						}
+					} catch ( ... ) {} // skip entries that generate JSON errors
+				});
+			}
+		}
+	});
+}
+
+DBMirrorSet::~DBMirrorSet()
+{
+	_running = false;
+	_syncCheckerThread.join();
+}
+
+bool DBMirrorSet::hasNetwork(const uint64_t networkId) const
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->hasNetwork(networkId))
+			return true;
+	}
+	return false;
+}
+
+bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network)
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->get(networkId,network)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member)
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->get(networkId,network,memberId,member))
+			return true;
+	}
+	return false;
+}
+
+bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,DB::NetworkSummaryInfo &info)
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->get(networkId,network,memberId,member,info))
+			return true;
+	}
+	return false;
+}
+
+bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohmann::json> &members)
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->get(networkId,network,members))
+			return true;
+	}
+	return false;
+}
+
+AuthInfo DBMirrorSet::getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL) 
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) { 
+		AuthInfo info = (*d)->getSSOAuthInfo(member, redirectURL);
+		if (info.enabled) {
+			return info;
+		}
+	}
+	return AuthInfo();
+}
+
+void DBMirrorSet::networks(std::set<uint64_t> &networks)
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		(*d)->networks(networks);
+	}
+}
+
+bool DBMirrorSet::waitForReady()
+{
+	bool r = false;
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		r |= (*d)->waitForReady();
+	}
+	return r;
+}
+
+bool DBMirrorSet::isReady()
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if (!(*d)->isReady())
+			return false;
+	}
+	return true;
+}
+
+bool DBMirrorSet::save(nlohmann::json &record,bool notifyListeners)
+{
+	std::vector< std::shared_ptr<DB> > dbs;
+	{
+		std::unique_lock<std::shared_mutex> l(_dbs_l);
+		dbs = _dbs;
+	}
+	if (notifyListeners) {
+		for(auto d=dbs.begin();d!=dbs.end();++d) {
+			if ((*d)->save(record,true))
+				return true;
+		}
+		return false;
+	} else {
+		bool modified = false;
+		for(auto d=dbs.begin();d!=dbs.end();++d) {
+			modified |= (*d)->save(record,false);
+		}
+		return modified;
+	}
+}
+
+void DBMirrorSet::eraseNetwork(const uint64_t networkId)
+{
+	std::unique_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		(*d)->eraseNetwork(networkId);
+	}
+}
+
+void DBMirrorSet::eraseMember(const uint64_t networkId,const uint64_t memberId)
+{
+	std::unique_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		(*d)->eraseMember(networkId,memberId);
+	}
+}
+
+void DBMirrorSet::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress)
+{
+	std::shared_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		(*d)->nodeIsOnline(networkId,memberId,physicalAddress);
+	}
+}
+
+void DBMirrorSet::onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network)
+{
+	nlohmann::json record(network);
+	std::unique_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if (d->get() != db) {
+			(*d)->save(record,false);
+		}
+	}
+	_listener->onNetworkUpdate(this,networkId,network);
+}
+
+void DBMirrorSet::onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member)
+{
+	nlohmann::json record(member);
+	std::unique_lock<std::shared_mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if (d->get() != db) {
+			(*d)->save(record,false);
+		}
+	}
+	_listener->onNetworkMemberUpdate(this,networkId,memberId,member);
+}
+
+void DBMirrorSet::onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId)
+{
+	_listener->onNetworkMemberDeauthorize(this,networkId,memberId);
+}
+
+} // namespace ZeroTier

controller/DBMirrorSet.hpp

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_DBMIRRORSET_HPP
+#define ZT_DBMIRRORSET_HPP
+
+#include "DB.hpp"
+
+#include <vector>
+#include <memory>
+#include <shared_mutex>
+#include <set>
+#include <thread>
+
+namespace ZeroTier {
+
+class DBMirrorSet : public DB::ChangeListener
+{
+public:
+	DBMirrorSet(DB::ChangeListener *listener);
+	virtual ~DBMirrorSet();
+
+	bool hasNetwork(const uint64_t networkId) const;
+
+	bool get(const uint64_t networkId,nlohmann::json &network);
+	bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member);
+	bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,DB::NetworkSummaryInfo &info);
+	bool get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohmann::json> &members);
+
+	void networks(std::set<uint64_t> &networks);
+
+	bool waitForReady();
+	bool isReady();
+	bool save(nlohmann::json &record,bool notifyListeners);
+	void eraseNetwork(const uint64_t networkId);
+	void eraseMember(const uint64_t networkId,const uint64_t memberId);
+	void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress);
+
+	// These are called by various DB instances when changes occur.
+	virtual void onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network);
+	virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member);
+	virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId);
+
+	AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL);
+
+	inline void addDB(const std::shared_ptr<DB> &db)
+	{
+		db->addListener(this);
+		std::unique_lock<std::shared_mutex> l(_dbs_l);
+		_dbs.push_back(db);
+	}
+
+private:
+	DB::ChangeListener *const _listener;
+	std::atomic_bool _running;
+	std::thread _syncCheckerThread;
+	std::vector< std::shared_ptr< DB > > _dbs;
+	mutable std::shared_mutex _dbs_l;
+};
+
+} // namespace ZeroTier
+
+#endif

controller/EmbeddedNetworkController.hpp

@@ -0,0 +1,183 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_SQLITENETWORKCONTROLLER_HPP
+#define ZT_SQLITENETWORKCONTROLLER_HPP
+
+#include <stdint.h>
+
+#include <string>
+#include <map>
+#include <vector>
+#include <set>
+#include <list>
+#include <thread>
+#include <unordered_map>
+#include <atomic>
+
+#include "../node/Constants.hpp"
+#include "../node/NetworkController.hpp"
+#include "../node/Utils.hpp"
+#include "../node/Address.hpp"
+#include "../node/InetAddress.hpp"
+
+#include "../osdep/OSUtils.hpp"
+#include "../osdep/Thread.hpp"
+#include "../osdep/BlockingQueue.hpp"
+
+#include <nlohmann/json.hpp>
+
+#include <cpp-httplib/httplib.h>
+
+#include "DB.hpp"
+#include "DBMirrorSet.hpp"
+
+namespace ZeroTier {
+
+class Node;
+struct RedisConfig;
+
+class EmbeddedNetworkController : public NetworkController,public DB::ChangeListener
+{
+public:
+	/**
+	 * @param node Parent node
+	 * @param dbPath Database path (file path or database credentials)
+	 */
+	EmbeddedNetworkController(Node *node,const char *ztPath,const char *dbPath, int listenPort, RedisConfig *rc);
+	virtual ~EmbeddedNetworkController();
+
+	virtual void init(const Identity &signingId,Sender *sender);
+
+	void setSSORedirectURL(const std::string &url);
+
+	virtual void request(
+		uint64_t nwid,
+		const InetAddress &fromAddr,
+		uint64_t requestPacketId,
+		const Identity &identity,
+		const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData);
+
+	void configureHTTPControlPlane(
+		httplib::Server &s,
+		httplib::Server &sV6,
+		const std::function<void(const httplib::Request&, httplib::Response&, std::string)>);
+
+	void handleRemoteTrace(const ZT_RemoteTrace &rt);
+
+	virtual void onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network);
+	virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member);
+	virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId);
+
+private:
+	void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData);
+	void _startThreads();
+	void _ssoExpiryThread();
+
+	std::string networkUpdateFromPostData(uint64_t networkID, const std::string &body);
+
+	struct _RQEntry
+	{
+		uint64_t nwid;
+		uint64_t requestPacketId;
+		InetAddress fromAddr;
+		Identity identity;
+		Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> metaData;
+		enum {
+			RQENTRY_TYPE_REQUEST = 0
+		} type;
+	};
+
+	struct _MemberStatusKey
+	{
+		_MemberStatusKey() : networkId(0),nodeId(0) {}
+		_MemberStatusKey(const uint64_t nwid,const uint64_t nid) : networkId(nwid),nodeId(nid) {}
+		uint64_t networkId;
+		uint64_t nodeId;
+		inline bool operator==(const _MemberStatusKey &k) const { return ((k.networkId == networkId)&&(k.nodeId == nodeId)); }
+		inline bool operator<(const _MemberStatusKey &k) const { return (k.networkId < networkId) || ((k.networkId == networkId)&&(k.nodeId < nodeId)); }
+	};
+	struct _MemberStatus
+	{
+		_MemberStatus() : lastRequestTime(0),authenticationExpiryTime(-1),vMajor(-1),vMinor(-1),vRev(-1),vProto(-1) {}
+		int64_t lastRequestTime;
+		int64_t authenticationExpiryTime;
+		int vMajor,vMinor,vRev,vProto;
+		Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> lastRequestMetaData;
+		Identity identity;
+		inline bool online(const int64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); }
+	};
+	struct _MemberStatusHash
+	{
+		inline std::size_t operator()(const _MemberStatusKey &networkIdNodeId) const
+		{
+			return (std::size_t)(networkIdNodeId.networkId + networkIdNodeId.nodeId);
+		}
+	};
+
+	const int64_t _startTime;
+	int _listenPort;
+	Node *const _node;
+	std::string _ztPath;
+	std::string _path;
+	Identity _signingId;
+	std::string _signingIdAddressString;
+	NetworkController::Sender *_sender;
+
+	DBMirrorSet _db;
+	BlockingQueue< _RQEntry * > _queue;
+
+	std::vector<std::thread> _threads;
+	std::mutex _threads_l;
+
+	std::unordered_map< _MemberStatusKey,_MemberStatus,_MemberStatusHash > _memberStatus;
+	std::mutex _memberStatus_l;
+
+	std::set< std::pair<int64_t, _MemberStatusKey> > _expiringSoon;
+	std::mutex _expiringSoon_l;
+
+	RedisConfig *_rc;
+	std::string _ssoRedirectURL;
+
+	bool _ssoExpiryRunning;
+	std::thread _ssoExpiry;
+
+#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
+	prometheus::simpleapi::benchmark_family_t _member_status_lookup;
+	prometheus::simpleapi::counter_family_t   _member_status_lookup_count;
+	prometheus::simpleapi::benchmark_family_t _node_is_online;
+	prometheus::simpleapi::counter_family_t   _node_is_online_count;
+	prometheus::simpleapi::benchmark_family_t _get_and_init_member;
+	prometheus::simpleapi::counter_family_t   _get_and_init_member_count;
+	prometheus::simpleapi::benchmark_family_t _have_identity;
+	prometheus::simpleapi::counter_family_t   _have_identity_count;
+	prometheus::simpleapi::benchmark_family_t _determine_auth;
+	prometheus::simpleapi::counter_family_t   _determine_auth_count;
+	prometheus::simpleapi::benchmark_family_t _sso_check;
+	prometheus::simpleapi::counter_family_t   _sso_check_count;
+	prometheus::simpleapi::benchmark_family_t _auth_check;
+	prometheus::simpleapi::counter_family_t   _auth_check_count;
+	prometheus::simpleapi::benchmark_family_t _json_schlep;
+	prometheus::simpleapi::counter_family_t   _json_schlep_count;
+	prometheus::simpleapi::benchmark_family_t _issue_certificate;
+	prometheus::simpleapi::counter_family_t   _issue_certificate_count;
+	prometheus::simpleapi::benchmark_family_t _save_member;
+	prometheus::simpleapi::counter_family_t   _save_member_count;
+	prometheus::simpleapi::benchmark_family_t _send_netconf;
+	prometheus::simpleapi::counter_family_t   _send_netconf_count;
+#endif
+};
+
+} // namespace ZeroTier
+
+#endif

controller/FileDB.cpp

@@ -0,0 +1,167 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "FileDB.hpp"
+
+#include "../node/Metrics.hpp"
+
+namespace ZeroTier
+{
+
+FileDB::FileDB(const char *path) :
+	DB(),
+	_path(path),
+	_networksPath(_path + ZT_PATH_SEPARATOR_S + "network"),
+	_tracePath(_path + ZT_PATH_SEPARATOR_S + "trace"),
+	_running(true)
+{
+	OSUtils::mkdir(_path.c_str());
+	OSUtils::lockDownFile(_path.c_str(),true);
+	OSUtils::mkdir(_networksPath.c_str());
+	OSUtils::mkdir(_tracePath.c_str());
+
+	std::vector<std::string> networks(OSUtils::listDirectory(_networksPath.c_str(),false));
+	std::string buf;
+	for(auto n=networks.begin();n!=networks.end();++n) {
+		buf.clear();
+		if ((n->length() == 21)&&(OSUtils::readFile((_networksPath + ZT_PATH_SEPARATOR_S + *n).c_str(),buf))) {
+			try {
+				nlohmann::json network(OSUtils::jsonParse(buf));
+				const std::string nwids = network["id"];
+				if (nwids.length() == 16) {
+					nlohmann::json nullJson;
+					_networkChanged(nullJson,network,false);
+					Metrics::network_count++;
+					std::string membersPath(_networksPath + ZT_PATH_SEPARATOR_S + nwids + ZT_PATH_SEPARATOR_S "member");
+					std::vector<std::string> members(OSUtils::listDirectory(membersPath.c_str(),false));
+					for(auto m=members.begin();m!=members.end();++m) {
+						buf.clear();
+						if ((m->length() == 15)&&(OSUtils::readFile((membersPath + ZT_PATH_SEPARATOR_S + *m).c_str(),buf))) {
+							try {
+								nlohmann::json member(OSUtils::jsonParse(buf));
+								const std::string addrs = member["id"];
+								if (addrs.length() == 10) {
+									nlohmann::json nullJson2;
+									_memberChanged(nullJson2,member,false);
+									Metrics::member_count++;
+								}
+							} catch ( ... ) {}
+						}
+					}
+				}
+			} catch ( ... ) {}
+		}
+	}
+}
+
+FileDB::~FileDB()
+{
+	try {
+		_online_l.lock();
+		_running = false;
+		_online_l.unlock();
+		_onlineUpdateThread.join();
+	} catch ( ... ) {}
+}
+
+bool FileDB::waitForReady() { return true; }
+bool FileDB::isReady() { return true; }
+
+bool FileDB::save(nlohmann::json &record,bool notifyListeners)
+{
+	char p1[4096],p2[4096],pb[4096];
+	bool modified = false;
+	try {
+		const std::string objtype = record["objtype"];
+		if (objtype == "network") {
+
+			const uint64_t nwid = OSUtils::jsonIntHex(record["id"],0ULL);
+			if (nwid) {
+				nlohmann::json old;
+				get(nwid,old);
+				if ((!old.is_object())||(!_compareRecords(old,record))) {
+					record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
+					OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),nwid);
+					if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) {
+						fprintf(stderr,"WARNING: controller unable to write to path: %s" ZT_EOL_S,p1);
+					}
+					_networkChanged(old,record,notifyListeners);
+					modified = true;
+				}
+			}
+
+		} else if (objtype == "member") {
+
+			const uint64_t id = OSUtils::jsonIntHex(record["id"],0ULL);
+			const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"],0ULL);
+			if ((id)&&(nwid)) {
+				nlohmann::json network,old;
+				get(nwid,network,id,old);
+				if ((!old.is_object())||(!_compareRecords(old,record))) {
+					record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
+					OSUtils::ztsnprintf(pb,sizeof(pb),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member",_networksPath.c_str(),(unsigned long long)nwid);
+					OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.10llx.json",pb,(unsigned long long)id);
+					if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) {
+						OSUtils::ztsnprintf(p2,sizeof(p2),"%s" ZT_PATH_SEPARATOR_S "%.16llx",_networksPath.c_str(),(unsigned long long)nwid);
+						OSUtils::mkdir(p2);
+						OSUtils::mkdir(pb);
+						if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) {
+							fprintf(stderr,"WARNING: controller unable to write to path: %s" ZT_EOL_S,p1);
+						}
+					}
+					_memberChanged(old,record,notifyListeners);
+					modified = true;
+				}
+			}
+
+		}
+	} catch ( ... ) {} // drop invalid records missing fields
+	return modified;
+}
+
+void FileDB::eraseNetwork(const uint64_t networkId)
+{
+	nlohmann::json network,nullJson;
+	get(networkId,network);
+	char p[16384];
+	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),networkId);
+	OSUtils::rm(p);
+	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx",_networksPath.c_str(),(unsigned long long)networkId);
+	OSUtils::rmDashRf(p);
+	_networkChanged(network,nullJson,true);
+	std::lock_guard<std::mutex> l(this->_online_l);
+	this->_online.erase(networkId);
+}
+
+void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId)
+{
+	nlohmann::json network,member,nullJson;
+	get(networkId,network,memberId,member);
+	char p[4096];
+	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member" ZT_PATH_SEPARATOR_S "%.10llx.json",_networksPath.c_str(),networkId,memberId);
+	OSUtils::rm(p);
+	_memberChanged(member,nullJson,true);
+	std::lock_guard<std::mutex> l(this->_online_l);
+	this->_online[networkId].erase(memberId);
+}
+
+void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress)
+{
+	char mid[32],atmp[64];
+	OSUtils::ztsnprintf(mid,sizeof(mid),"%.10llx",(unsigned long long)memberId);
+	physicalAddress.toString(atmp);
+	std::lock_guard<std::mutex> l(this->_online_l);
+	this->_online[networkId][memberId][OSUtils::now()] = physicalAddress;
+}
+
+} // namespace ZeroTier

controller/FileDB.hpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_CONTROLLER_FILEDB_HPP
+#define ZT_CONTROLLER_FILEDB_HPP
+
+#include "DB.hpp"
+
+namespace ZeroTier
+{
+
+class FileDB : public DB
+{
+public:
+	FileDB(const char *path);
+	virtual ~FileDB();
+
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json &record,bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId,const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress);
+
+protected:
+	std::string _path;
+	std::string _networksPath;
+	std::string _tracePath;
+	std::thread _onlineUpdateThread;
+	std::map< uint64_t,std::map<uint64_t,std::map<int64_t,InetAddress> > > _online;
+	std::mutex _online_l;
+	bool _running;
+};
+
+} // namespace ZeroTier
+
+#endif

controller/LFDB.cpp

@@ -0,0 +1,406 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "LFDB.hpp"
+
+#include <thread>
+#include <chrono>
+#include <iostream>
+#include <sstream>
+
+#include "../osdep/OSUtils.hpp"
+#include "../ext/cpp-httplib/httplib.h"
+
+namespace ZeroTier
+{
+
+LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,const char *lfOwnerPublic,const char *lfNodeHost,int lfNodePort,bool storeOnlineState) :
+	DB(),
+	_myId(myId),
+	_lfOwnerPrivate((lfOwnerPrivate) ? lfOwnerPrivate : ""),
+	_lfOwnerPublic((lfOwnerPublic) ? lfOwnerPublic : ""),
+	_lfNodeHost((lfNodeHost) ? lfNodeHost : "127.0.0.1"),
+	_lfNodePort(((lfNodePort > 0)&&(lfNodePort < 65536)) ? lfNodePort : 9980),
+	_running(true),
+	_ready(false),
+	_storeOnlineState(storeOnlineState)
+{
+	_syncThread = std::thread([this]() {
+		char controllerAddress[24];
+		const uint64_t controllerAddressInt = _myId.address().toInt();
+		_myId.address().toString(controllerAddress);
+		std::string networksSelectorName("com.zerotier.controller.lfdb:"); networksSelectorName.append(controllerAddress); networksSelectorName.append("/network");
+
+		// LF record masking key is the first 32 bytes of SHA512(controller private key) in hex,
+		// hiding record values from anything but the controller or someone who has its key.
+		uint8_t sha512pk[64];
+		_myId.sha512PrivateKey(sha512pk);
+		char maskingKey [128];
+		Utils::hex(sha512pk,32,maskingKey);
+
+		httplib::Client htcli(_lfNodeHost.c_str(),_lfNodePort);
+		int64_t timeRangeStart = 0;
+		while (_running.load()) {
+			{
+				std::lock_guard<std::mutex> sl(_state_l);
+				for(auto ns=_state.begin();ns!=_state.end();++ns) {
+					if (ns->second.dirty) {
+						nlohmann::json network;
+						if (get(ns->first,network)) {
+							nlohmann::json newrec,selector0;
+							selector0["Name"] = networksSelectorName;
+							selector0["Ordinal"] = ns->first;
+							newrec["Selectors"].push_back(selector0);
+							newrec["Value"] = network.dump();
+							newrec["OwnerPrivate"] = _lfOwnerPrivate;
+							newrec["MaskingKey"] = maskingKey;
+							newrec["PulseIfUnchanged"] = true;
+							try {
+								auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
+								if (resp) {
+									if (resp->status == 200) {
+										ns->second.dirty = false;
+										//printf("SET network %.16llx %s\n",ns->first,resp->body.c_str());
+									} else {
+										fprintf(stderr,"ERROR: LFDB: %d from node (create/update network): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+									}
+								} else {
+									fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
+								}
+							} catch (std::exception &e) {
+								fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update network): %s" ZT_EOL_S,e.what());
+							} catch ( ... ) {
+								fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update network): unknown exception" ZT_EOL_S);
+							}
+						}
+					}
+
+					for(auto ms=ns->second.members.begin();ms!=ns->second.members.end();++ms) {
+						if ((_storeOnlineState)&&(ms->second.lastOnlineDirty)&&(ms->second.lastOnlineAddress)) {
+							nlohmann::json newrec,selector0,selector1,selectors,ip;
+							char tmp[1024],tmp2[128];
+							OSUtils::ztsnprintf(tmp,sizeof(tmp),"com.zerotier.controller.lfdb:%s/network/%.16llx/online",controllerAddress,(unsigned long long)ns->first);
+							ms->second.lastOnlineAddress.toIpString(tmp2);
+							selector0["Name"] = tmp;
+							selector0["Ordinal"] = ms->first;
+							selector1["Name"] = tmp2;
+							selector1["Ordinal"] = 0;
+							selectors.push_back(selector0);
+							selectors.push_back(selector1);
+							newrec["Selectors"] = selectors;
+							const uint8_t *const rawip = (const uint8_t *)ms->second.lastOnlineAddress.rawIpData();
+							switch(ms->second.lastOnlineAddress.ss_family) {
+								case AF_INET:
+									for(int j=0;j<4;++j)
+										ip.push_back((unsigned int)rawip[j]);
+									break;
+								case AF_INET6:
+									for(int j=0;j<16;++j)
+										ip.push_back((unsigned int)rawip[j]);
+									break;
+								default:
+									ip = tmp2; // should never happen since only IP transport is currently supported
+									break;
+							}
+							newrec["Value"] = ip;
+							newrec["OwnerPrivate"] = _lfOwnerPrivate;
+							newrec["MaskingKey"] = maskingKey;
+							newrec["Timestamp"] = ms->second.lastOnlineTime;
+							newrec["PulseIfUnchanged"] = true;
+							try {
+								auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
+								if (resp) {
+									if (resp->status == 200) {
+										ms->second.lastOnlineDirty = false;
+										//printf("SET member online %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str());
+									} else {
+										fprintf(stderr,"ERROR: LFDB: %d from node (create/update member online status): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+									}
+								} else {
+									fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
+								}
+							} catch (std::exception &e) {
+								fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update member online status): %s" ZT_EOL_S,e.what());
+							} catch ( ... ) {
+								fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update member online status): unknown exception" ZT_EOL_S);
+							}
+						}
+
+						if (ms->second.dirty) {
+							nlohmann::json network,member;
+							if (get(ns->first,network,ms->first,member)) {
+								nlohmann::json newrec,selector0,selector1,selectors;
+								selector0["Name"] = networksSelectorName;
+								selector0["Ordinal"] = ns->first;
+								selector1["Name"] = "member";
+								selector1["Ordinal"] = ms->first;
+								selectors.push_back(selector0);
+								selectors.push_back(selector1);
+								newrec["Selectors"] = selectors;
+								newrec["Value"] = member.dump();
+								newrec["OwnerPrivate"] = _lfOwnerPrivate;
+								newrec["MaskingKey"] = maskingKey;
+								newrec["PulseIfUnchanged"] = true;
+								try {
+									auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
+									if (resp) {
+										if (resp->status == 200) {
+											ms->second.dirty = false;
+											//printf("SET member %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str());
+										} else {
+											fprintf(stderr,"ERROR: LFDB: %d from node (create/update member): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+										}
+									} else {
+										fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
+									}
+								} catch (std::exception &e) {
+									fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update member): %s" ZT_EOL_S,e.what());
+								} catch ( ... ) {
+									fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update member): unknown exception" ZT_EOL_S);
+								}
+							}
+						}
+					}
+				}
+			}
+
+			try {
+				std::ostringstream query;
+				query <<
+					"{"
+						"\"Ranges\":[{"
+							"\"Name\":\"" << networksSelectorName << "\","
+							"\"Range\":[0,18446744073709551615]"
+						"}],"
+						"\"TimeRange\":[" << timeRangeStart << ",9223372036854775807],"
+						"\"MaskingKey\":\"" << maskingKey << "\","
+						"\"Owners\":[\"" << _lfOwnerPublic << "\"]"
+					"}";
+				auto resp = htcli.Post("/query",query.str(),"application/json");
+				if (resp) {
+					if (resp->status == 200) {
+						nlohmann::json results(OSUtils::jsonParse(resp->body));
+						if ((results.is_array())&&(!results.empty())) {
+							for(std::size_t ri=0;ri<results.size();++ri) {
+								nlohmann::json &rset = results[ri];
+								if ((rset.is_array())&&(!rset.empty())) {
+
+									nlohmann::json &result = rset[0];
+									if (result.is_object()) {
+										nlohmann::json &record = result["Record"];
+										if (record.is_object()) {
+											const std::string recordValue = result["Value"];
+											//printf("GET network %s\n",recordValue.c_str());
+											nlohmann::json network(OSUtils::jsonParse(recordValue));
+											if (network.is_object()) {
+												const std::string idstr = network["id"];
+												const uint64_t id = Utils::hexStrToU64(idstr.c_str());
+												if ((id >> 24) == controllerAddressInt) { // sanity check
+
+													nlohmann::json oldNetwork;
+													if ((timeRangeStart > 0)&&(get(id,oldNetwork))) {
+														const uint64_t revision = network["revision"];
+														const uint64_t prevRevision = oldNetwork["revision"];
+														if (prevRevision < revision) {
+															_networkChanged(oldNetwork,network,timeRangeStart > 0);
+														}
+													} else {
+														nlohmann::json nullJson;
+														_networkChanged(nullJson,network,timeRangeStart > 0);
+													}
+
+												}
+											}
+										}
+									}
+
+								}
+							}
+						}
+					} else {
+						fprintf(stderr,"ERROR: LFDB: %d from node (check for network updates): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+					}
+				} else {
+					fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
+				}
+			} catch (std::exception &e) {
+				fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (check for network updates): %s" ZT_EOL_S,e.what());
+			} catch ( ... ) {
+				fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (check for network updates): unknown exception" ZT_EOL_S);
+			}
+
+			try {
+				std::ostringstream query;
+				query <<
+					"{"
+						"\"Ranges\":[{"
+							"\"Name\":\"" << networksSelectorName << "\","
+							"\"Range\":[0,18446744073709551615]"
+						"},{"
+							"\"Name\":\"member\","
+							"\"Range\":[0,18446744073709551615]"
+						"}],"
+						"\"TimeRange\":[" << timeRangeStart << ",9223372036854775807],"
+						"\"MaskingKey\":\"" << maskingKey << "\","
+						"\"Owners\":[\"" << _lfOwnerPublic << "\"]"
+					"}";
+				auto resp = htcli.Post("/query",query.str(),"application/json");
+				if (resp) {
+					if (resp->status == 200) {
+						nlohmann::json results(OSUtils::jsonParse(resp->body));
+						if ((results.is_array())&&(!results.empty())) {
+							for(std::size_t ri=0;ri<results.size();++ri) {
+								nlohmann::json &rset = results[ri];
+								if ((rset.is_array())&&(!rset.empty())) {
+
+									nlohmann::json &result = rset[0];
+									if (result.is_object()) {
+										nlohmann::json &record = result["Record"];
+										if (record.is_object()) {
+											const std::string recordValue = result["Value"];
+											//printf("GET member %s\n",recordValue.c_str());
+											nlohmann::json member(OSUtils::jsonParse(recordValue));
+											if (member.is_object()) {
+												const std::string nwidstr = member["nwid"];
+												const std::string idstr = member["id"];
+												const uint64_t nwid = Utils::hexStrToU64(nwidstr.c_str());
+												const uint64_t id = Utils::hexStrToU64(idstr.c_str());
+												if ((id)&&((nwid >> 24) == controllerAddressInt)) { // sanity check
+
+													nlohmann::json network,oldMember;
+													if ((timeRangeStart > 0)&&(get(nwid,network,id,oldMember))) {
+														const uint64_t revision = member["revision"];
+														const uint64_t prevRevision = oldMember["revision"];
+														if (prevRevision < revision)
+															_memberChanged(oldMember,member,timeRangeStart > 0);
+													} else if (hasNetwork(nwid)) {
+														nlohmann::json nullJson;
+														_memberChanged(nullJson,member,timeRangeStart > 0);
+													}
+
+												}
+											}
+										}
+									}
+
+								}
+							}
+						}
+					} else {
+						fprintf(stderr,"ERROR: LFDB: %d from node (check for member updates): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+					}
+				} else {
+					fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
+				}
+			} catch (std::exception &e) {
+				fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (check for member updates): %s" ZT_EOL_S,e.what());
+			} catch ( ... ) {
+				fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (check for member updates): unknown exception" ZT_EOL_S);
+			}
+
+			timeRangeStart = time(nullptr) - 120; // start next query 2m before now to avoid losing updates
+			_ready.store(true);
+
+			for(int k=0;k<4;++k) { // 2s delay between queries for remotely modified networks or members
+				if (!_running.load())
+					return;
+				std::this_thread::sleep_for(std::chrono::milliseconds(500));
+			}
+		}
+	});
+}
+
+LFDB::~LFDB()
+{
+	_running.store(false);
+	_syncThread.join();
+}
+
+bool LFDB::waitForReady()
+{
+	while (!_ready.load()) {
+		std::this_thread::sleep_for(std::chrono::milliseconds(500));
+	}
+	return true;
+}
+
+bool LFDB::isReady()
+{
+	return (_ready.load());
+}
+
+bool LFDB::save(nlohmann::json &record,bool notifyListeners)
+{
+	bool modified = false;
+	const std::string objtype = record["objtype"];
+	if (objtype == "network") {
+		const uint64_t nwid = OSUtils::jsonIntHex(record["id"],0ULL);
+		if (nwid) {
+			nlohmann::json old;
+			get(nwid,old);
+			if ((!old.is_object())||(!_compareRecords(old,record))) {
+				record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
+				_networkChanged(old,record,notifyListeners);
+				{
+					std::lock_guard<std::mutex> l(_state_l);
+					_state[nwid].dirty = true;
+				}
+				modified = true;
+			}
+		}
+	} else if (objtype == "member") {
+		const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"],0ULL);
+		const uint64_t id = OSUtils::jsonIntHex(record["id"],0ULL);
+		if ((id)&&(nwid)) {
+			nlohmann::json network,old;
+			get(nwid,network,id,old);
+			if ((!old.is_object())||(!_compareRecords(old,record))) {
+				record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
+				_memberChanged(old,record,notifyListeners);
+				{
+					std::lock_guard<std::mutex> l(_state_l);
+					_state[nwid].members[id].dirty = true;
+				}
+				modified = true;
+			}
+		}
+	}
+	return modified;
+}
+
+void LFDB::eraseNetwork(const uint64_t networkId)
+{
+	// TODO
+}
+
+void LFDB::eraseMember(const uint64_t networkId,const uint64_t memberId)
+{
+	// TODO
+}
+
+void LFDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress)
+{
+	std::lock_guard<std::mutex> l(_state_l);
+	auto nw = _state.find(networkId);
+	if (nw != _state.end()) {
+		auto m = nw->second.members.find(memberId);
+		if (m != nw->second.members.end()) {
+			m->second.lastOnlineTime = OSUtils::now();
+			if (physicalAddress)
+				m->second.lastOnlineAddress = physicalAddress;
+			m->second.lastOnlineDirty = true;
+		}
+	}
+}
+
+} // namespace ZeroTier

controller/LFDB.hpp

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_CONTROLLER_LFDB_HPP
+#define ZT_CONTROLLER_LFDB_HPP
+
+#include "DB.hpp"
+
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <atomic>
+
+namespace ZeroTier {
+
+/**
+ * DB implementation for controller that stores data in LF
+ */
+class LFDB : public DB
+{
+public:
+	/**
+	 * @param myId This controller's identity
+	 * @param path Base path for ZeroTier node itself
+	 * @param lfOwnerPrivate LF owner private in PEM format
+	 * @param lfOwnerPublic LF owner public in @base62 format
+	 * @param lfNodeHost LF node host
+	 * @param lfNodePort LF node http (not https) port
+	 * @param storeOnlineState If true, store online/offline state and IP info in LF (a lot of data, only for private networks!)
+	 */
+	LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,const char *lfOwnerPublic,const char *lfNodeHost,int lfNodePort,bool storeOnlineState);
+	virtual ~LFDB();
+
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json &record,bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId,const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress);
+
+protected:
+	const Identity _myId;
+
+	std::string _lfOwnerPrivate,_lfOwnerPublic;
+	std::string _lfNodeHost;
+	int _lfNodePort;
+
+	struct _MemberState
+	{
+		_MemberState() :
+			lastOnlineAddress(),
+			lastOnlineTime(0),
+			dirty(false),
+			lastOnlineDirty(false) {}
+		InetAddress lastOnlineAddress;
+		int64_t lastOnlineTime;
+		bool dirty;
+		bool lastOnlineDirty;
+	};
+	struct _NetworkState
+	{
+		_NetworkState() :
+			members(),
+			dirty(false) {}
+		std::unordered_map<uint64_t,_MemberState> members;
+		bool dirty;
+	};
+	std::unordered_map<uint64_t,_NetworkState> _state;
+	std::mutex _state_l;
+
+	std::atomic_bool _running;
+	std::atomic_bool _ready;
+	std::thread _syncThread;
+	bool _storeOnlineState;
+};
+
+} // namespace ZeroTier
+
+#endif

controller/PostgreSQL.hpp

@@ -0,0 +1,196 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "DB.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#ifndef ZT_CONTROLLER_LIBPQ_HPP
+#define ZT_CONTROLLER_LIBPQ_HPP
+
+#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
+
+#include "ConnectionPool.hpp"
+#include <pqxx/pqxx>
+
+#include <memory>
+#include <redis++/redis++.h>
+
+#include "../node/Metrics.hpp"
+
+extern "C" {
+typedef struct pg_conn PGconn;
+}
+
+namespace smeeclient {
+	struct SmeeClient;
+}
+
+namespace ZeroTier {
+
+struct RedisConfig;
+
+
+class PostgresConnection : public Connection {
+public:
+	virtual ~PostgresConnection() {
+	}
+
+	std::shared_ptr<pqxx::connection> c;
+	int a;
+};
+
+
+class PostgresConnFactory : public ConnectionFactory {
+public:
+	PostgresConnFactory(std::string &connString) 
+		: m_connString(connString)
+	{
+	}
+
+	virtual std::shared_ptr<Connection> create() {
+		Metrics::conn_counter++;
+		auto c = std::shared_ptr<PostgresConnection>(new PostgresConnection());
+		c->c = std::make_shared<pqxx::connection>(m_connString);
+		return std::static_pointer_cast<Connection>(c);
+	}
+private:
+	std::string m_connString;
+};
+
+class PostgreSQL;
+
+class MemberNotificationReceiver : public pqxx::notification_receiver {
+public: 
+	MemberNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
+	virtual ~MemberNotificationReceiver() {
+		fprintf(stderr, "MemberNotificationReceiver destroyed\n");
+	}
+
+	virtual void operator() (const std::string &payload, int backendPid);
+private:
+	PostgreSQL *_psql;
+};
+
+class NetworkNotificationReceiver : public pqxx::notification_receiver {
+public:
+	NetworkNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
+	virtual ~NetworkNotificationReceiver() {
+		fprintf(stderr, "NetworkNotificationReceiver destroyed\n");
+	};
+
+	virtual void operator() (const std::string &payload, int packend_pid);
+private:
+	PostgreSQL *_psql;
+};
+
+/**
+ * A controller database driver that talks to PostgreSQL
+ *
+ * This is for use with ZeroTier Central.  Others are free to build and use it
+ * but be aware that we might change it at any time.
+ */
+class PostgreSQL : public DB
+{
+	friend class MemberNotificationReceiver;
+	friend class NetworkNotificationReceiver;
+public:
+	PostgreSQL(const Identity &myId, const char *path, int listenPort, RedisConfig *rc);
+	virtual ~PostgreSQL();
+
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json &record,bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress);
+	virtual AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL);
+
+protected:
+	struct _PairHasher
+	{
+		inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); }
+	};
+	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners) {
+		DB::_memberChanged(old, memberConfig, notifyListeners);
+	}
+
+	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners) {
+		DB::_networkChanged(old, networkConfig, notifyListeners);
+	}
+
+private:
+	void initializeNetworks();
+	void initializeMembers();
+	void heartbeat();
+	void membersDbWatcher();
+	void _membersWatcher_Postgres();
+	void networksDbWatcher();
+	void _networksWatcher_Postgres();
+
+	void _membersWatcher_Redis();
+	void _networksWatcher_Redis();
+
+	void commitThread();
+	void onlineNotificationThread();
+	void onlineNotification_Postgres();
+	void onlineNotification_Redis();
+	uint64_t _doRedisUpdate(sw::redis::Transaction &tx, std::string &controllerId,
+		std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > &lastOnline);
+
+	void configureSmee();
+	void notifyNewMember(const std::string &networkID, const std::string &memberID);
+
+	enum OverrideMode {
+		ALLOW_PGBOUNCER_OVERRIDE = 0,
+		NO_OVERRIDE = 1
+	};
+
+	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
+
+	const Identity _myId;
+	const Address _myAddress;
+	std::string _myAddressStr;
+	std::string _connString;
+
+	BlockingQueue< std::pair<nlohmann::json,bool> > _commitQueue;
+
+	std::thread _heartbeatThread;
+	std::thread _membersDbWatcher;
+	std::thread _networksDbWatcher;
+	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
+	std::thread _onlineNotificationThread;
+
+	std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > _lastOnline;
+
+	mutable std::mutex _lastOnline_l;
+	mutable std::mutex _readyLock;
+	std::atomic<int> _ready, _connected, _run;
+	mutable volatile bool _waitNoticePrinted;
+
+	int _listenPort;
+	uint8_t _ssoPsk[48];
+
+	RedisConfig *_rc;
+	std::shared_ptr<sw::redis::Redis> _redis;
+	std::shared_ptr<sw::redis::RedisCluster> _cluster;
+    bool _redisMemberStatus;
+
+	smeeclient::SmeeClient *_smee;
+};
+
+} // namespace ZeroTier
+
+#endif // ZT_CONTROLLER_LIBPQ_HPP
+
+#endif // ZT_CONTROLLER_USE_LIBPQ

controller/README.md

@@ -0,0 +1,55 @@
+Network Controller Microservice
+======
+
+Every ZeroTier virtual network has a *network controller* responsible for admitting members to the network, issuing certificates, and issuing default configuration information.
+
+This is our reference controller implementation and is almost the same as the one we use to power our own hosted services at [my.zerotier.com](https://my.zerotier.com/). The only difference is the database backend used. 
+
+Controller data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, rsync'd, placed in `git`, etc. The files under `controller.d` should not be modified in place while the controller is running or data loss may result, and if they are edited directly take care not to save corrupt JSON since that can also lead to data loss when the controller is restarted. Going through the API is strongly preferred to directly modifying these files.
+
+See the API section below for information about controlling the controller.
+
+### Scalability and Reliability
+
+Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons. Since the controller uses the filesystem as its data store we recommend fast filesystems and fast SSD drives for heavily loaded controllers.
+
+Since ZeroTier nodes are mobile and do not need static IPs, implementing high availability fail-over for controllers is easy. Just replicate their working directories from master to backup and have something automatically fire up the backup if the master goes down. Modern orchestration tools like Nomad and Kubernetes can be of help here.
+
+### Dockerizing Controllers
+
+ZeroTier network controllers can easily be run in Docker or other container systems. Since containers do not need to actually join networks, extra privilege options like "--device=/dev/net/tun --privileged" are not needed. You'll just need to map the local JSON API port of the running controller and allow it to access the Internet (over UDP/9993 at a minimum) so things can reach and query it.
+
+### Upgrading from Older (1.1.14 or earlier) Versions
+
+Older versions of this code used a SQLite database instead of in-filesystem JSON. A migration utility called `migrate-sqlite` is included here and *must* be used to migrate this data to the new format. If the controller is started with an old `controller.db` in its working directory it will terminate after printing an error to *stderr*. This is done to prevent "surprises" for those running DIY controllers using the old code.
+
+The migration tool is written in nodeJS and can be used like this:
+
+    cd migrate-sqlite
+    npm install
+    node migrate.js </path/to/controller.db> </path/to/controller.d>
+
+### Network Controller API
+
+The controller API is hosted via the same JSON API endpoint that ZeroTier One uses for local control (usually at 127.0.0.1 port 9993). All controller options are routed under the `/controller` base path.
+
+The controller microservice itself does not implement any fine-grained access control. Access control is via the ZeroTier control interface itself and `authtoken.secret`. This can be sent as the `X-ZT1-Auth` HTTP header field or appended to the URL as `?auth=<token>`. Take care when doing the latter that request URLs are not being logged.
+
+While networks with any valid ID can be added to the controller's database, it will only actually work to control networks whose first 10 hex digits correspond with the network controller's ZeroTier ID. See [section 2.2.1 of the ZeroTier manual](https://zerotier.com/manual.shtml#2_2_1).
+
+The controller JSON API is *very* sensitive about types. Integers must be integers and strings strings, etc. Incorrect types may be ignored, set to default values, or set to undefined values.
+
+Full documentation of the Controller API can be found on our [documentation site](https://docs.zerotier.com/service/v1#tag/controller)
+
+### Prometheus Metrics
+
+Controller specific metrics are available from the `/metrics` endpoint.
+
+| Metric Name | Type | Description |
+| --- | --- | --- |
+| controller_network_count | Gauge | number of networks the controller is serving | 
+| controller_member_count | Gauge | number of network members the controller is serving |
+| controller_network_change_count | Counter | number of times a network configuration is changed |
+| controller_member_change_count | Counter | number of times a network member configuration is changed |
+| controller_member_auth_count | Counter | number of network member auths |
+| controller_member_deauth_count | Counter | number of network member deauths|

controller/Redis.hpp

@@ -0,0 +1,15 @@
+#ifndef ZT_CONTROLLER_REDIS_HPP
+#define ZT_CONTROLLER_REDIS_HPP
+
+#include <string>
+
+namespace ZeroTier {
+struct RedisConfig {
+    std::string hostname;
+    int port;
+    std::string password;
+    bool clusterMode;
+};
+}
+
+#endif