Skip to content
This repository was archived by the owner on Dec 11, 2020. It is now read-only.

Commit 47bbdf6

Browse files
jma127jma127
authored
Merge pull request #104 from jma127/master
Slight logger refactor and memory pressure reduction
2 parents a39e7dc + 7b50f00 commit 47bbdf6

34 files changed

+112
-84
lines changed

‎src_cpp/elf/CMakeLists.txt

+5-5
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ set(ELF_SOURCES
1414
options/Pybind.cc
1515
)
1616

17-
# set(ELF_TEST_SOURCES
18-
# options/OptionMapTest.cc
19-
# options/OptionSpecTest.cc
20-
# )
17+
set(ELF_TEST_SOURCES
18+
options/OptionMapTest.cc
19+
options/OptionSpecTest.cc
20+
)
2121

2222
# Main ELF library
2323

@@ -40,7 +40,7 @@ target_link_libraries(elf PUBLIC
4040
# Tests
4141

4242
enable_testing()
43-
# add_cpp_tests(test_cpp_elf_ elf ${ELF_TEST_SOURCES})
43+
add_cpp_tests(test_cpp_elf_ elf ${ELF_TEST_SOURCES})
4444

4545
# Python bindings
4646

‎src_cpp/elf/ai/tree_search/mcts.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,9 @@ class MCTSAI_T : public AI_T<typename Actor::State, typename Actor::Action> {
4242
const elf::ai::tree_search::TSOptions& options,
4343
std::function<Actor*(int)> gen)
4444
: options_(options),
45-
logger_(
46-
elf::logging::getLogger("elf::ai::tree_search::MCTSAI_T-", "")) {
45+
logger_(elf::logging::getIndexedLogger(
46+
"elf::ai::tree_search::MCTSAI_T-",
47+
"")) {
4748
ts_.reset(new TreeSearch(options_, gen));
4849
}
4950

‎src_cpp/elf/ai/tree_search/tree_search.h

+4-3
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ class TreeSearchSingleThreadT {
7070
TreeSearchSingleThreadT(int thread_id, const TSOptions& options)
7171
: threadId_(thread_id),
7272
options_(options),
73-
logger_(elf::logging::getLogger(
73+
logger_(elf::logging::getIndexedLogger(
7474
"elf::ai::tree_search::TreeSearchSingleThreadT-",
7575
"")) {
7676
if (options_.verbose) {
@@ -334,8 +334,9 @@ class TreeSearchT {
334334
TreeSearchT(const TSOptions& options, std::function<Actor*(int)> actor_gen)
335335
: options_(options),
336336
stopSearch_(false),
337-
logger_(
338-
elf::logging::getLogger("elf::ai::tree_search::TreeSearchT-", "")) {
337+
logger_(elf::logging::getIndexedLogger(
338+
"elf::ai::tree_search::TreeSearchT-",
339+
"")) {
339340
for (int i = 0; i < options.num_threads; ++i) {
340341
treeSearches_.emplace_back(new TreeSearchSingleThread(i, options_));
341342
actors_.emplace_back(actor_gen(i));

‎src_cpp/elf/ai/tree_search/tree_search_base.h

+1-6
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,7 @@ struct EdgeInfo {
116116
child_node(InvalidNodeId),
117117
reward(0),
118118
num_visits(0),
119-
virtual_loss(0),
120-
logger_(
121-
elf::logging::getLogger("elf::ai::tree_search::EdgeInfo-", "")) {}
119+
virtual_loss(0) {}
122120

123121
float getQSA() const {
124122
return reward / num_visits;
@@ -127,9 +125,6 @@ struct EdgeInfo {
127125
// TODO: What is this function doing (ssengupta@fb.com)
128126
void checkValid() const {
129127
if (virtual_loss != 0) {
130-
// TODO: This should be a Google log (ssengupta@fb)
131-
logger_->info(
132-
"Virtual loss is not zero[{}]\n{}", virtual_loss, info(true));
133128
assert(virtual_loss == 0);
134129
}
135130
}

‎src_cpp/elf/base/context.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,8 @@ class Context {
200200
public:
201201
using GameCallback = std::function<void(int game_idx, GameClient*)>;
202202

203-
Context() : logger_(elf::logging::getLogger("elf::base::Context-", "")) {
203+
Context()
204+
: logger_(elf::logging::getIndexedLogger("elf::base::Context-", "")) {
204205
// Wait for the derived class to add entries to extractor_.
205206
server_ = comm_.getServer();
206207
client_.reset(new GameClient(&comm_, this));

‎src_cpp/elf/base/dispatcher.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@ class ThreadedDispatcherT : public ThreadedCtrlBase {
2121
ThreadedDispatcherT(Ctrl& ctrl, int num_games)
2222
: ThreadedCtrlBase(ctrl, 500),
2323
num_games_(num_games),
24-
logger_(
25-
elf::logging::getLogger("elf::base::ThreadedDispatcherT-", "")) {}
24+
logger_(elf::logging::getIndexedLogger(
25+
"elf::base::ThreadedDispatcherT-",
26+
"")) {}
2627

2728
void Start(ServerReply replier, ServerFirstSend first_send = nullptr) {
2829
server_replier_ = replier;

‎src_cpp/elf/base/extractor.h

+4-2
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,8 @@ class ClassFieldT;
471471
//
472472
class Extractor {
473473
public:
474-
Extractor() : logger_(elf::logging::getLogger("elf::base::Extractor-", "")) {}
474+
Extractor()
475+
: logger_(elf::logging::getIndexedLogger("elf::base::Extractor-", "")) {}
475476

476477
template <typename T>
477478
FuncMapT<T>& addField(const std::string& key) {
@@ -579,7 +580,8 @@ class ClassFieldT {
579580

580581
ClassFieldT(Extractor* ext)
581582
: ext_(ext),
582-
logger_(elf::logging::getLogger("elf::base::ClassFieldT-", "")) {}
583+
logger_(elf::logging::getIndexedLogger("elf::base::ClassFieldT-", "")) {
584+
}
583585

584586
template <typename T>
585587
ClassField& addFunction(

‎src_cpp/elf/base/sharedmem.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ class SharedMem {
133133
const std::unordered_map<std::string, AnyP>& mem)
134134
: opts_(smem_opts),
135135
mem_(mem),
136-
logger_(elf::logging::getLogger("elf::base::SharedMem-", "")) {
136+
logger_(elf::logging::getIndexedLogger("elf::base::SharedMem-", "")) {
137137
opts_.setIdx(idx);
138138
}
139139

‎src_cpp/elf/comm/comm.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ class CommT : public CommInternalT<
327327
: CommInternal::Client(pp),
328328
pp_(pp),
329329
rng_(time(NULL)),
330-
logger_(elf::logging::getLogger("elf::comm::Client-", "")) {}
330+
logger_(elf::logging::getIndexedLogger("elf::comm::Client-", "")) {}
331331

332332
ReplyStatus sendWait(Data data, const std::vector<std::string>& labels) {
333333
return CommInternal::Client::sendWait(

‎src_cpp/elf/distributed/shared_reader.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -164,8 +164,9 @@ class ReaderQueuesT {
164164
ReaderQueuesT(const RQCtrl& reader_ctrl)
165165
: min_size_satisfied_(false),
166166
parity_sizes_(2, 0),
167-
logger_(
168-
elf::logging::getLogger("elf::distributed::ReaderQueuesT-", "")) {
167+
logger_(elf::logging::getIndexedLogger(
168+
"elf::distributed::ReaderQueuesT-",
169+
"")) {
169170
// Make sure this is an even number.
170171
assert(reader_ctrl.num_reader % 2 == 0);
171172
min_size_per_queue_ = reader_ctrl.ctrl.queue_min_size;

‎src_cpp/elf/distributed/shared_rw_buffer2.h

+4-2
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@ class Writer {
5959
Writer(const Options& opt)
6060
: rng_(time(NULL)),
6161
options_(opt),
62-
logger_(elf::logging::getLogger("elf::distributed::Writer-", "")) {
62+
logger_(
63+
elf::logging::getIndexedLogger("elf::distributed::Writer-", "")) {
6364
identity_ = options_.identity + "-" + get_id(rng_);
6465
sender_.reset(new elf::distri::ZMQSender(
6566
identity_, options_.addr, options_.port, options_.use_ipv6));
@@ -151,7 +152,8 @@ class Reader {
151152
db_name_(filename),
152153
rng_(time(NULL)),
153154
done_(false),
154-
logger_(elf::logging::getLogger("elf::distributed::Reader-", "")) {}
155+
logger_(
156+
elf::logging::getIndexedLogger("elf::distributed::Reader-", "")) {}
155157

156158
void startReceiving(
157159
ProcessFunc proc_func,

‎src_cpp/elf/distributed/zmq_util.h

+10-5
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,9 @@ class SegmentedRecv {
8080
public:
8181
SegmentedRecv(zmq::socket_t& socket)
8282
: socket_(socket),
83-
logger_(
84-
elf::logging::getLogger("elf::distributed::SegmentedRecv-", "")) {}
83+
logger_(elf::logging::getIndexedLogger(
84+
"elf::distributed::SegmentedRecv-",
85+
"")) {}
8586

8687
/*
8788
void recvBlocked(size_t n, std::vector<std::string>* p_msgs) {
@@ -179,7 +180,7 @@ class SegmentedRecv {
179180
class SameThreadChecker {
180181
public:
181182
SameThreadChecker()
182-
: logger_(elf::logging::getLogger(
183+
: logger_(elf::logging::getIndexedLogger(
183184
"elf::distributed::SameThreadChecker-",
184185
"")) {
185186
id_ = std::this_thread::get_id();
@@ -210,7 +211,9 @@ class ZMQReceiver : public SameThreadChecker {
210211
public:
211212
ZMQReceiver(int port, bool use_ipv6)
212213
: context_(1),
213-
logger_(elf::logging::getLogger("elf::distributed::ZMQReceiver-", "")) {
214+
logger_(elf::logging::getIndexedLogger(
215+
"elf::distributed::ZMQReceiver-",
216+
"")) {
214217
broker_.reset(new zmq::socket_t(context_, ZMQ_ROUTER));
215218
if (use_ipv6) {
216219
int ipv6 = 1;
@@ -286,7 +289,9 @@ class ZMQSender : public SameThreadChecker {
286289
int port,
287290
bool use_ipv6)
288291
: context_(1),
289-
logger_(elf::logging::getLogger("elf::distributed::ZMQSender-", "")) {
292+
logger_(elf::logging::getIndexedLogger(
293+
"elf::distributed::ZMQSender-",
294+
"")) {
290295
sender_.reset(new zmq::socket_t(context_, ZMQ_DEALER));
291296
if (use_ipv6) {
292297
int ipv6 = 1;

‎src_cpp/elf/legacy/python_options_utils_cpp.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ struct ContextOptions {
3232
std::shared_ptr<spdlog::logger> _logger;
3333

3434
ContextOptions()
35-
: _logger(elf::logging::getLogger("elf::legacy::ContextOptions-", "")) {}
35+
: _logger(elf::logging::getIndexedLogger(
36+
"elf::legacy::ContextOptions-",
37+
"")) {}
3638

3739
void print() const {
3840
_logger->info("JobId: {}", job_id);

‎src_cpp/elf/logging/IndexedLoggerFactory.cc

+16
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212

1313
#include "IndexedLoggerFactory.h"
1414

15+
#include <iostream>
16+
1517
namespace elf {
1618
namespace logging {
1719

@@ -22,15 +24,29 @@ void IndexedLoggerFactory::registerPy(pybind11::module& m) {
2224
.def(py::init<CreatorT>())
2325
.def(py::init<CreatorT, size_t>())
2426
.def("makeLogger", &IndexedLoggerFactory::makeLogger);
27+
28+
m.def("getIndexedLogger", getIndexedLogger);
2529
}
2630

2731
std::shared_ptr<spdlog::logger> IndexedLoggerFactory::makeLogger(
2832
const std::string& prefix,
2933
const std::string& suffix) {
3034
size_t curCount = counter_++;
3135
std::string loggerName = prefix + std::to_string(curCount) + suffix;
36+
3237
return creator_(loggerName);
3338
}
3439

40+
std::shared_ptr<spdlog::logger> getIndexedLogger(
41+
const std::string& prefix,
42+
const std::string& suffix) {
43+
static IndexedLoggerFactory factory([](const std::string& name) {
44+
auto ptr = spdlog::stderr_color_mt(name);
45+
spdlog::drop(name);
46+
return ptr;
47+
});
48+
return factory.makeLogger(prefix, suffix);
49+
}
50+
3551
} // namespace logging
3652
} // namespace elf

‎src_cpp/elf/logging/IndexedLoggerFactory.h

+6-6
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@
3434
* };
3535
*
3636
* }
37+
*
38+
* WARNING: Use this *only* when you are able to guarantee a bounded number of
39+
* object instantiations. This class will automatically enforce an upper bound
40+
* of a few thousand.
3741
*/
3842

3943
#pragma once
@@ -69,13 +73,9 @@ class IndexedLoggerFactory {
6973
std::atomic_size_t counter_;
7074
};
7175

72-
inline std::shared_ptr<spdlog::logger> getLogger(
76+
std::shared_ptr<spdlog::logger> getIndexedLogger(
7377
const std::string& prefix,
74-
const std::string& suffix) {
75-
static IndexedLoggerFactory factory(
76-
[](const std::string& name) { return spdlog::stderr_color_mt(name); });
77-
return factory.makeLogger(prefix, suffix);
78-
}
78+
const std::string& suffix);
7979

8080
} // namespace logging
8181
} // namespace elf

‎src_cpp/elfgames/go/base/board_feature.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,9 @@ class BoardFeature {
6666
: s_(s),
6767
_rot(rot),
6868
_flip(flip),
69-
logger_(
70-
elf::logging::getLogger("elfgames::go::base::BoardFeature-", "")) {}
69+
logger_(elf::logging::getIndexedLogger(
70+
"elfgames::go::base::BoardFeature-",
71+
"")) {}
7172
BoardFeature(const GoState& s) : s_(s), _rot(NONE), _flip(false) {}
7273

7374
static BoardFeature RandomShuffle(const GoState& s, std::mt19937* rng) {

‎src_cpp/elfgames/go/common/dispatcher_callback.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ class DispatcherCallback {
1313
public:
1414
DispatcherCallback(ThreadedDispatcher* dispatcher, elf::GameClient* client)
1515
: client_(client),
16-
logger_(elf::logging::getLogger(
16+
logger_(elf::logging::getIndexedLogger(
1717
"elfgames::go::common::DispatcherCallback-",
1818
"")) {
1919
using std::placeholders::_1;

‎src_cpp/elfgames/go/common/game_base.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ class GoGameBase {
2626
_game_idx(game_idx),
2727
_options(options),
2828
_context_options(context_options),
29-
_logger(
30-
elf::logging::getLogger("elfgames::go::common::GoGameBase-", "")) {
29+
_logger(elf::logging::getIndexedLogger(
30+
"elfgames::go::common::GoGameBase-",
31+
"")) {
3132
if (options.seed == 0) {
3233
_seed = elf_utils::get_seed(
3334
game_idx ^ std::hash<std::string>{}(context_options.job_id));

‎src_cpp/elfgames/go/common/game_selfplay.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ GoGameSelfPlay::GoGameSelfPlay(
2323
dispatcher_(dispatcher),
2424
notifier_(notifier),
2525
_state_ext(game_idx, options),
26-
logger_(elf::logging::getLogger(
26+
logger_(elf::logging::getIndexedLogger(
2727
"elfgames::go::GoGameSelfPlay-" + std::to_string(game_idx) + "-",
2828
"")) {}
2929

‎src_cpp/elfgames/go/common/game_stats.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@
2121
class GameStats {
2222
public:
2323
GameStats()
24-
: _logger(
25-
elf::logging::getLogger("elfgames::go::common::GameStats-", "")) {}
24+
: _logger(elf::logging::getIndexedLogger(
25+
"elfgames::go::common::GameStats-",
26+
"")) {}
2627

2728
void feedMoveRanking(int ranking) {
2829
std::lock_guard<std::mutex> lock(_mutex);

‎src_cpp/elfgames/go/common/go_state_ext.h

+4-3
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,9 @@ struct GoStateExt {
3939
_last_value(0.0),
4040
_resign_check(options.resign_thres, options.resign_prob_never),
4141
_options(options),
42-
_logger(
43-
elf::logging::getLogger("elfgames::go::common::GoStateExt-", "")) {
42+
_logger(elf::logging::getIndexedLogger(
43+
"elfgames::go::common::GoStateExt-",
44+
"")) {
4445
restart();
4546
}
4647

@@ -263,7 +264,7 @@ class GoStateExtOffline {
263264
: _game_idx(game_idx),
264265
_bf(_state),
265266
_options(options),
266-
_logger(elf::logging::getLogger(
267+
_logger(elf::logging::getIndexedLogger(
267268
"elfgames::go::common::GoStateExtOffline-",
268269
"")) {}
269270

‎src_cpp/elfgames/go/inference/game_context.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ class GameContext {
3131
GameContext(const ContextOptions& contextOptions, const GameOptions& options)
3232
: contextOptions_(contextOptions),
3333
goFeature_(options),
34-
logger_(elf::logging::getLogger("GameContext-", "")) {
34+
logger_(elf::logging::getIndexedLogger("GameContext-", "")) {
3535
context_.reset(new elf::Context);
3636

3737
// Only works for online setting.

‎src_cpp/elfgames/go/mcts/mcts.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,9 @@ class MCTSActor {
4747
MCTSActor(elf::GameClient* client, const MCTSActorParams& params)
4848
: params_(params),
4949
rng_(params.seed),
50-
logger_(elf::logging::getLogger("elfgames::go::mcts::MCTSActor-", "")) {
50+
logger_(elf::logging::getIndexedLogger(
51+
"elfgames::go::mcts::MCTSActor-",
52+
"")) {
5153
ai_.reset(new AI(client, {params_.actor_name}));
5254
}
5355

‎src_cpp/elfgames/go/sgf/sgf.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,8 @@ class Sgf {
260260

261261
Sgf()
262262
: _num_moves(0),
263-
_logger(elf::logging::getLogger("elfgames::go::sgf::Sgf-", "")) {}
263+
_logger(elf::logging::getIndexedLogger("elfgames::go::sgf::Sgf-", "")) {
264+
}
264265
bool load(const std::string& filename);
265266
bool load(const std::string& filename, const std::string& game);
266267

0 commit comments

Comments
 (0)