sarsoo/KnowledgeCrawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tweaking logging to allow folder specification, adding logging

Browse Source
This commit is contained in:
Andy Pack 2025-01-28 17:09:59 +00:00
parent ab89424ad1
commit c8205adb50
Signed by: sarsoo
GPG Key ID: A55BA3536A5E0ED7
9 changed files with 106 additions and 71 deletions

9
src/CMakeLists.txt

@ -14,7 +14,6 @@
task/current_tasks.cpp
net/ntfy.cpp
net/http.cpp
# image/img.cpp
)
FetchContent_Declare(
@ -27,7 +26,11 @@ FetchContent_MakeAvailable(Boost)
find_package(OpenSSL REQUIRED)
include_directories(${OPENSSL_INCLUDE_DIR})
#find_package( OpenCV REQUIRED )
#include_directories( ${OpenCV_INCLUDE_DIRS} )
if(OPENCV)
add_compile_definitions(WITH_OPENCV)
find_package( OpenCV REQUIRED )
include_directories( ${OpenCV_INCLUDE_DIRS} )
target_sources(kc PRIVATE image/img.cpp)
endif()
target_link_libraries(kc PRIVATE Boost::program_options Boost::log Boost::date_time Boost::filesystem Boost::system Boost::thread Boost::log_setup Boost::chrono Boost::atomic Boost::asio Boost::beast Boost::json Boost::algorithm ${OPENSSL_LIBRARIES} ${OpenCV_LIBS})

5
src/appcontext.cpp

@ -22,6 +22,11 @@ void AppContext::load_and_parse_cache() {
void AppContext::load_and_parse_cache(ParseOperations operations)
{
if (!config->contains("path")) {
print_and_log_error("No paths provided");
return;
}
const auto env_paths = (*config)["path"].as<std::vector<std::string>>();
for (const auto& env_path : env_paths) {

9
src/config.cpp

@ -7,6 +7,7 @@
#include <boost/program_options.hpp>
#include "const.hpp"
#include "logging.hpp"
po::options_description get_current_tasks_options() {
po::options_description options("Current Tasks");
@ -15,6 +16,7 @@ po::options_description get_current_tasks_options() {
(CONFIG_HOST.c_str(), po::value<std::string>(), "ntfy hostname")
(CONFIG_TOPIC.c_str(), po::value<std::string>(), "ntfy topic name")
(CONFIG_TITLE.c_str(), po::value<std::string>(), "title for notifications")
(CONFIG_TAGS.c_str(), po::value<std::vector<std::string>>(), "tags to add to notification")
;
return options;
@ -28,6 +30,7 @@ std::shared_ptr<po::variables_map> init_config(int argc, const char *argv[])
("help,h", "produce help message")
("path,p", po::value<std::vector<std::string>>(), "set root path of knowledge base")
("config", po::value<std::string>()->default_value("kc.ini"), "config file location")
("logpath", po::value<std::string>()->default_value("."), "log folder location")
;
po::options_description hidden_general("Hidden");
@ -75,10 +78,10 @@ std::shared_ptr<po::variables_map> init_config(int argc, const char *argv[])
if (vm->contains("config"))
{
auto config_path = (*vm)["config"].as<std::string>();
BOOST_LOG_TRIVIAL(info) << "Attempting file config load for " << config_path;
// BOOST_LOG_TRIVIAL(info) << "Attempting file config load for " << config_path;
if (std::ifstream ifs{config_path.c_str()}) {
BOOST_LOG_TRIVIAL(info) << "File opened, loading...";
// BOOST_LOG_TRIVIAL(info) << "File opened, loading...";
po::store(po::parse_config_file(ifs, config_file_options), *vm);
}
}
@ -107,7 +110,7 @@ std::shared_ptr<po::variables_map> init_config(int argc, const char *argv[])
}
catch (const po::error &ex)
{
BOOST_LOG_TRIVIAL(error) << ex.what();
std::cout << ex.what() << std::endl;
return nullptr;
}

2
src/const.hpp

@ -10,7 +10,6 @@ static const std::string CMD_VALIDATE_TASKS = "validate";
static const std::string CMD_IMG_TASKS = "img";
static const std::string CMD_PRINT_TASKS = "print";
static const std::string CMD_CURRENT_TASKS = "current";
static const std::string CMD_NET_TASKS = "net";
///////////////
@ -21,6 +20,7 @@ static const std::string CONFIG_NOTIFY = "notify";
static const std::string CONFIG_HOST = "host";
static const std::string CONFIG_TOPIC = "topic";
static const std::string CONFIG_TITLE = "title";
static const std::string CONFIG_TAGS = "tag";
///////////////

8
src/logging.cpp

@ -1,18 +1,22 @@
#include "logging.hpp"
#include <filesystem>
namespace logging = boost::log;
namespace src = boost::log::sources;
namespace sinks = boost::log::sinks;
namespace keywords = boost::log::keywords;
namespace fs = std::filesystem;
void init_logging()
void init_logging(const std::string &log_path)
{
logging::register_simple_formatter_factory<logging::trivial::severity_level, char>("Severity");
auto log_file = std::string(fs::path(log_path) / fs::path("kc_%N.log"));
logging::add_file_log
(
keywords::file_name = "kc_%N.log",
keywords::file_name = log_file,
keywords::time_based_rotation = sinks::file::rotation_at_time_point(0, 0, 0),
keywords::format = "[%TimeStamp%] [%ThreadID%] [%Severity%] %Message%",
keywords::open_mode = std::ios::app

2
src/logging.hpp

@ -9,7 +9,7 @@
#include <boost/log/utility/setup/common_attributes.hpp>
#include <boost/log/utility/setup/console.hpp>
void init_logging();
void init_logging(const std::string &log_path);
inline void print_and_log(std::string log_line)
{

116
src/main.cpp

@ -3,78 +3,85 @@
#include <iostream>
#include <fstream>
#include <memory>
#include <regex>
#include "const.hpp"
#include "logging.hpp"
#include "config.hpp"
#include "appcontext.hpp"
#include "fs/fs.hpp"
#include "net/http.hpp"
#include "net/ntfy.hpp"
#include "parse/FileContextCache.hpp"
#include "print/print.hpp"
#include "task/current_tasks.hpp"
#include "valid/link.hpp"
//#include "image/img.hpp"
#ifdef WITH_OPENCV
#include "image/img.hpp"
#endif
void run_validate(const kc::AppContext &app_context);
void run_img(const kc::AppContext &app_context);
void run_print(const kc::AppContext &app_context);
int run_current_tasks(const kc::AppContext &app_context);
void run_test_net(const kc::AppContext &app_context);
int main(int argc, const char *argv[]) {
init_logging();
BOOST_LOG_TRIVIAL(info) << "================================";
BOOST_LOG_TRIVIAL(info) << " kc";
BOOST_LOG_TRIVIAL(info) << "================================";
BOOST_LOG_TRIVIAL(info) << "Starting up....";
kc::AppContext app_context;
app_context.load_config(argc, argv);
if(app_context.config)
{
const auto command = app_context.command();
if (!command.empty())
{
if (command == CMD_VALIDATE_TASKS)
{
app_context.load_and_parse_cache(kc::LINKS);
run_validate(app_context);
}
else if (command == CMD_IMG_TASKS)
{
app_context.load_and_parse_cache(kc::IMAGES);
run_img(app_context);
}
else if (command == CMD_PRINT_TASKS)
{
app_context.load_and_parse_cache();
run_print(app_context);
}
else if (command == CMD_CURRENT_TASKS)
{
app_context.load_and_parse_cache(kc::TASKS);
return run_current_tasks(app_context);
}
else if (command == CMD_NET_TASKS)
{
// app_context.load_and_parse_cache(kc::TASKS);
run_test_net(app_context);
}
}
else
{
print_and_log_error("Command not found, exiting");
return 1;
}
init_logging((*app_context.config)["logpath"].as<std::string>());
return 0;
try {
BOOST_LOG_TRIVIAL(info) << "================================";
BOOST_LOG_TRIVIAL(info) << " kc";
BOOST_LOG_TRIVIAL(info) << "================================";
BOOST_LOG_TRIVIAL(info) << "Starting up....";
#ifdef __cpp_lib_execution
BOOST_LOG_TRIVIAL(debug) << "Compiled with parallel loops";
#else
BOOST_LOG_TRIVIAL(debug) << "Compiled WITHOUT parallel loops";
#endif
if(app_context.config)
{
const auto command = app_context.command();
if (!command.empty())
{
if (command == CMD_VALIDATE_TASKS)
{
app_context.load_and_parse_cache(kc::LINKS);
run_validate(app_context);
}
#ifdef WITH_OPENCV
else if (command == CMD_IMG_TASKS)
{
app_context.load_and_parse_cache(kc::IMAGES);
run_img(app_context);
}
#endif
else if (command == CMD_PRINT_TASKS)
{
app_context.load_and_parse_cache();
run_print(app_context);
}
else if (command == CMD_CURRENT_TASKS)
{
app_context.load_and_parse_cache(kc::TASKS);
return run_current_tasks(app_context);
}
}
else
{
print_and_log_error("Command not found, exiting");
return 1;
}
return 0;
}
}
catch (const std::exception &e) {
print_and_log_error(std::format("Exception occurred - {}", e.what()));
}
return 1;
@ -91,11 +98,14 @@ void run_validate(const kc::AppContext &app_context)
void run_img(const kc::AppContext &app_context)
{
#ifdef WITH_OPENCV
BOOST_LOG_TRIVIAL(info) << "Running \"Image Processing\" command";
for(const auto& file_cache : app_context.file_caches) {
BOOST_LOG_TRIVIAL(info) << "Image processing in " << file_cache->get_root_path();
// kc::image_proc(file_cache->get());
kc::image_proc(file_cache->get());
}
#endif
}
void run_print(const kc::AppContext &app_context)
@ -110,10 +120,4 @@ int run_current_tasks(const kc::AppContext &app_context)
{
BOOST_LOG_TRIVIAL(info) << "Running \"Current Tasks\" command";
return kc::current_tasks(app_context);
}
void run_test_net(const kc::AppContext &app_context)
{
kc::notify("ntfy.sheep-ghoul.ts.net",
kc::Notification("todo", "next test!"));
}

5
src/net/http.cpp

@ -2,6 +2,7 @@
#include <boost/beast/version.hpp>
#include <boost/log/trivial.hpp>
#include <utility>
// https://www.boost.org/doc/libs/1_87_0/libs/beast/example/http/client/sync-ssl/http_client_sync_ssl.cpp
@ -38,11 +39,11 @@ void shutdown_stream(const std::shared_ptr<ssl::stream<beast::tcp_stream>> &stre
}
void request(http::verb method, const std::string &host, std::string target, std::string body, std::unique_ptr<std::unordered_map<std::string, std::string>> headers) {
request(method, host, target, 443, body, std::move(headers));
request(std::move(method), std::move(host), std::move(target), 443, std::move(body), std::move(headers));
}
void request(http::verb method, const std::string &host, std::string target, std::string body) {
request(method, host, target, 443, body, nullptr);
request(std::move(method), std::move(host), std::move(target), 443, std::move(body), nullptr);
}
void request(http::verb method, const std::string &host, std::string target, int port, std::string body, std::unique_ptr<std::unordered_map<std::string, std::string>> headers) {

21
src/task/current_tasks.cpp

@ -33,7 +33,7 @@ int current_tasks(const kc::AppContext &app_context)
}
}
std::ranges::sort(tasks, [](Task a, Task b)
std::ranges::sort(tasks, [](const Task &a, const Task &b)
{
return a.get_due_date() < b.get_due_date();
});
@ -42,7 +42,9 @@ int current_tasks(const kc::AppContext &app_context)
std::cout << task.get_content() << " (" << task.get_due_date() << ")" << std::endl;
}
if (app_context.config->contains(CONFIG_NOTIFY)) {
if (tasks.size() > 0 && app_context.config->contains(CONFIG_NOTIFY)) {
print_and_log(std::format("Sending notification for {} tasks", tasks.size()));
if (!app_context.config->contains(CONFIG_HOST)) {
print_and_log_error("No NTFY host provided");
return 1;
@ -59,7 +61,20 @@ int current_tasks(const kc::AppContext &app_context)
auto notif = kc::Notification(topic_name, payload);
kc::notify(host_name, notif);
if (app_context.config->contains(CONFIG_TAGS)) {
for (auto const& tag : (*app_context.config)[CONFIG_TAGS].as<std::vector<std::string>>()) {
BOOST_LOG_TRIVIAL(info) << "Tagging notification with " << tag;
notif.add_tag(tag);
}
}
try{
kc::notify(host_name, notif);
print_and_log("Notification sent");
}
catch (const std::exception &e) {
print_and_log_error(std::format("Exception occurred while sending notification - {}", e.what()));
}
}
return 0;