add support for include in datalog scanner

- add option `--no-preprocessor`
- new directive `.include "path"`
- new directive `.once`
- scanner treats `__FILE__` and `__LINE__`.
- scanner harvests comments (for future usage)

cmake/SouffleTests.cmake

@@ -142,7 +142,7 @@ function(SOUFFLE_RUN_TEST_HELPER)
 #Usually just "facts" but can be different when running multi - tests
     cmake_parse_arguments(
         PARAM
-        "COMPILED;FUNCTORS;NEGATIVE;MULTI_TEST" # Options
+        "COMPILED;FUNCTORS;NEGATIVE;MULTI_TEST;NO_PREPROCESSOR" # Options
         "TEST_NAME;CATEGORY;FACTS_DIR_NAME;EXTRA_DATA" #Single valued options
         ""
         ${ARGV}
@@ -159,8 +159,12 @@ function(SOUFFLE_RUN_TEST_HELPER)
         set(SHORT_EXEC_STYLE "")
     endif()
 
-    if (MSVC)
-        list(APPEND EXTRA_FLAGS "--preprocessor" "cl -nologo -TC -E")
+    if (PARAM_NO_PREPROCESSOR)
+      list(APPEND EXTRA_FLAGS "--no-preprocessor")
+    else ()
+      if (MSVC)
+          list(APPEND EXTRA_FLAGS "--preprocessor" "cl -nologo -TC -E")
+      endif()
     endif()
 
     if (PARAM_FUNCTORS)

src/main.cpp

@@ -206,6 +206,154 @@ void compileToBinary(const std::string& command, std::string_view sourceFilename
         throw std::invalid_argument(tfm::format("failed to compile C++ source <%s>", sourceFilename));
 }
 
+class InputProvider {
+public:
+    virtual ~InputProvider() {}
+    virtual FILE* getInputStream() = 0;
+    virtual bool endInput() = 0;
+};
+
+class FileInput : public InputProvider {
+public:
+    FileInput(const std::filesystem::path& path) : Path(path) {}
+
+    ~FileInput() {
+        if (Stream) {
+            fclose(Stream);
+        }
+    }
+
+    FILE* getInputStream() override {
+        if (std::filesystem::exists(Path)) {
+            Stream = fopen(Path.string().c_str(), "rb");
+            return Stream;
+        } else {
+            return nullptr;
+        }
+    }
+
+    bool endInput() override {
+        if (Stream == nullptr) {
+            return false;
+        } else {
+            fclose(Stream);
+            Stream = nullptr;
+            return true;
+        }
+    }
+
+private:
+    const std::filesystem::path Path;
+    FILE* Stream = nullptr;
+};
+
+class PreprocInput : public InputProvider {
+public:
+    PreprocInput(const std::filesystem::path& path, MainConfig& conf, const std::string& exec,
+            const std::string& options)
+            : Exec(which(exec)), Options(options), InitCmd(), Path(path), Conf(conf) {}
+
+    PreprocInput(const std::filesystem::path& path, MainConfig& conf, const std::string& cmd)
+            : Exec(), Options(), InitCmd(cmd), Path(path), Conf(conf) {}
+
+    virtual ~PreprocInput() {
+        if (Stream) {
+            pclose(Stream);
+        }
+    }
+
+    FILE* getInputStream() override {
+        Cmd.str("");
+
+        if (Exec) {
+            if (Exec->empty()) {
+                return nullptr;
+            }
+            Cmd << *Exec;
+        } else if (InitCmd) {
+            Cmd << *InitCmd;
+        } else {
+            return nullptr;
+        }
+
+        if (Options && !Options->empty()) {
+            Cmd << " ";
+            Cmd << *Options;
+        }
+
+        Cmd << " ";
+        Cmd << toString(join(Conf.getMany("include-dir"), " ",
+                [&](auto&& os, auto&& dir) { tfm::format(os, "-I \"%s\"", dir); }));
+
+        if (Conf.has("macro")) {
+            Cmd << " " << Conf.get("macro");
+        }
+        // Add RamDomain size as a macro
+        Cmd << " -DRAM_DOMAIN_SIZE=" << std::to_string(RAM_DOMAIN_SIZE);
+        Cmd << " \"" + Path.string() + "\"";
+
+#if defined(_MSC_VER)
+        // cl.exe prints the input file name on the standard error stream,
+        // we must silent it in order to preserve an empty error output
+        // because Souffle test-suite is sensible to error outputs.
+        Cmd << " 2> nul";
+#endif
+
+        Stream = popen(Cmd.str().c_str(), "r");
+        return Stream;
+    }
+
+    bool endInput() {
+        const int Status = pclose(Stream);
+        Stream = nullptr;
+        if (Status == -1) {
+            perror(nullptr);
+            throw std::runtime_error("failed to close pre-processor pipe");
+        } else if (Status != 0) {
+            std::cerr << "Pre-processors command failed with code " << Status << ": '" << Cmd.str() << "'\n";
+            throw std::runtime_error("Pre-processor command failed");
+        }
+        return true;
+    }
+
+    static bool available(const std::string& Exec) {
+        return !which(Exec).empty();
+    }
+
+private:
+    std::optional<std::string> Exec;
+    std::optional<std::string> Options;
+    std::optional<std::string> InitCmd;
+    std::filesystem::path Path;
+    MainConfig& Conf;
+    std::stringstream Cmd;
+    FILE* Stream = nullptr;
+};
+
+class GCCPreprocInput : public PreprocInput {
+public:
+    GCCPreprocInput(const std::filesystem::path& mainSource, MainConfig& conf)
+            : PreprocInput(mainSource, conf, "gcc", "-x c -E") {}
+
+    virtual ~GCCPreprocInput() {}
+
+    static bool available() {
+        return PreprocInput::available("gcc");
+    }
+};
+
+class MCPPPreprocInput : public PreprocInput {
+public:
+    MCPPPreprocInput(const std::filesystem::path& mainSource, MainConfig& conf)
+            : PreprocInput(mainSource, conf, "mcpp", "-e utf8 -W0") {}
+
+    virtual ~MCPPPreprocInput() {}
+
+    static bool available() {
+        return PreprocInput::available("mcpp");
+    }
+};
+
 int main(int argc, char** argv) {
     /* Time taking for overall runtime */
     auto souffle_start = std::chrono::high_resolution_clock::now();
@@ -298,7 +446,8 @@ int main(int argc, char** argv) {
                 {"parse-errors", '\5', "", "", false, "Show parsing errors, if any, then exit."},
                 {"help", 'h', "", "", false, "Display this help message."},
                 {"legacy", '\6', "", "", false, "Enable legacy support."},
-                {"preprocessor", '\7', "CMD", "", false, "C preprocessor to use."}};
+                {"preprocessor", '\7', "CMD", "", false, "C preprocessor to use."},
+                {"no-preprocessor", 10, "", "", false, "Do not use a C preprocessor."}};
         Global::config().processArgs(argc, argv, header.str(), versionFooter, options);
 
         // ------ command line arguments -------------
@@ -424,44 +573,30 @@ int main(int argc, char** argv) {
         throw std::runtime_error("failed to determine souffle executable path");
     }
 
-    /* Create the pipe to establish a communication between cpp and souffle */
-
-    std::string cmd;
-
-    if (Global::config().has("preprocessor")) {
-        cmd = Global::config().get("preprocessor");
-    } else {
-        cmd = which("mcpp");
-        if (isExecutable(cmd)) {
-            cmd += " -e utf8 -W0";
-        } else {
-            cmd = which("gcc");
-            if (isExecutable(cmd)) {
-                cmd += " -x c -E";
+    const std::filesystem::path InputPath(Global::config().get(""));
+    std::unique_ptr<InputProvider> Input;
+    const bool use_preprocessor = !Global::config().has("no-preprocessor");
+    if (use_preprocessor) {
+        if (Global::config().has("preprocessor")) {
+            auto cmd = Global::config().get("preprocessor");
+            if (cmd == "gcc") {
+                Input = std::make_unique<GCCPreprocInput>(InputPath, Global::config());
+            } else if (cmd == "mcpp") {
+                Input = std::make_unique<MCPPPreprocInput>(InputPath, Global::config());
             } else {
-                std::cerr << "failed to locate mcpp or gcc pre-processors\n";
-                throw std::runtime_error("failed to locate mcpp or gcc pre-processors");
+                Input = std::make_unique<PreprocInput>(InputPath, Global::config(), cmd);
             }
+        } else if (MCPPPreprocInput::available()) {  // mcpp fallback
+            Input = std::make_unique<MCPPPreprocInput>(InputPath, Global::config());
+        } else if (GCCPreprocInput::available()) {  // gcc fallback
+            Input = std::make_unique<GCCPreprocInput>(InputPath, Global::config());
+        } else {
+            throw std::runtime_error("failed to locate mcpp or gcc pre-processors");
         }
+    } else {
+        Input = std::make_unique<FileInput>(Global::config().get(""));
     }
 
-    cmd += " " + toString(join(Global::config().getMany("include-dir"), " ",
-                         [&](auto&& os, auto&& dir) { tfm::format(os, "-I \"%s\"", dir); }));
-
-    if (Global::config().has("macro")) {
-        cmd += " " + Global::config().get("macro");
-    }
-    // Add RamDomain size as a macro
-    cmd += " -DRAM_DOMAIN_SIZE=" + std::to_string(RAM_DOMAIN_SIZE);
-    cmd += " \"" + Global::config().get("") + "\"";
-#if defined(_MSC_VER)
-    // cl.exe prints the input file name on the standard error stream,
-    // we must silent it in order to preserve an empty error output
-    // because Souffle test-suite is sensible to error outputs.
-    cmd += " 2> nul";
-#endif
-    FILE* in = popen(cmd.c_str(), "r");
-
     /* Time taking for parsing */
     auto parser_start = std::chrono::high_resolution_clock::now();
 
@@ -470,19 +605,9 @@ int main(int argc, char** argv) {
     // parse file
     ErrorReport errReport(Global::config().has("no-warn"));
     DebugReport debugReport;
-    Own<ast::TranslationUnit> astTranslationUnit =
-            ParserDriver::parseTranslationUnit("<stdin>", in, errReport, debugReport);
-
-    // close input pipe
-    int preprocessor_status = pclose(in);
-    if (preprocessor_status == -1) {
-        perror(nullptr);
-        throw std::runtime_error("failed to close pre-processor pipe");
-    } else if (preprocessor_status != 0) {
-        std::cerr << "Pre-processors command failed with code " << preprocessor_status << ": '" << cmd
-                  << "'\n";
-        throw std::runtime_error("Pre-processor command failed");
-    }
+    Own<ast::TranslationUnit> astTranslationUnit = ParserDriver::parseTranslationUnit(
+            InputPath.string(), Input->getInputStream(), errReport, debugReport);
+    Input->endInput();
 
     /* Report run-time of the parser if verbose flag is set */
     if (Global::config().has("verbose")) {

src/parser/ParserDriver.cpp

@@ -54,7 +54,8 @@ Own<ast::TranslationUnit> ParserDriver::parse(
     translationUnit = mk<ast::TranslationUnit>(mk<ast::Program>(), errorReport, debugReport);
     yyscan_t scanner;
     ScannerInfo data;
-    data.yyfilename = filename;
+    SrcLocation emptyLoc;
+    data.push(std::filesystem::weakly_canonical(filename).string(), emptyLoc);
     yylex_init_extra(&data, &scanner);
     yyset_debug(0, scanner);
     yyset_in(in, scanner);
@@ -72,7 +73,8 @@ Own<ast::TranslationUnit> ParserDriver::parse(
     translationUnit = mk<ast::TranslationUnit>(mk<ast::Program>(), errorReport, debugReport);
 
     ScannerInfo data;
-    data.yyfilename = "<in-memory>";
+    SrcLocation emptyLoc;
+    data.push("<in-memory>", emptyLoc);
     yyscan_t scanner;
     yylex_init_extra(&data, &scanner);
     yyset_debug(0, scanner);
@@ -258,4 +260,36 @@ void ParserDriver::error(const std::string& msg) {
             Diagnostic(Diagnostic::Type::ERROR, DiagnosticMessage(msg)));
 }
 
+std::optional<std::filesystem::path> ParserDriver::searchIncludePath(
+        const std::string& IncludeString, const SrcLocation& Loc) {
+    std::filesystem::path Candidate(IncludeString);
+
+    if (Candidate.is_absolute()) {
+        if (std::filesystem::exists(Candidate)) {
+            return std::filesystem::canonical(Candidate);
+        } else {
+            return std::nullopt;
+        }
+    }
+
+    // search relative from current input file
+    Candidate = std::filesystem::path(Loc.file->Physical).parent_path() / IncludeString;
+    if (std::filesystem::exists(Candidate)) {
+        return std::filesystem::canonical(Candidate);
+    } else if (Candidate.is_absolute()) {
+        return std::nullopt;
+    }
+
+    return std::nullopt;
+}
+
+bool ParserDriver::canEnterOnce(const SrcLocation& onceLoc) {
+    const auto Inserted = VisitedLocations.emplace(onceLoc.file->Physical, onceLoc.start.line);
+    return Inserted.second;
+}
+
+void ParserDriver::addComment(const SrcLocation& Loc, const std::stringstream& Content) {
+    ScannedComments.emplace_back(Loc, Content.str());
+}
+
 }  // end of namespace souffle

src/parser/ParserDriver.h

@@ -31,7 +31,9 @@
 #include "ast/Type.h"
 #include "parser/SrcLocation.h"
 #include "reports/DebugReport.h"
+
 #include <cstdio>
+#include <filesystem>
 #include <memory>
 #include <set>
 #include <string>
@@ -43,8 +45,6 @@ class ParserDriver {
 public:
     virtual ~ParserDriver() = default;
 
-    Own<ast::TranslationUnit> translationUnit;
-
     void addRelation(Own<ast::Relation> r);
     void addFunctorDeclaration(Own<ast::FunctorDeclaration> f);
     void addDirective(Own<ast::Directive> d);
@@ -66,8 +66,6 @@ class ParserDriver {
 
     Own<ast::Counter> addDeprecatedCounter(SrcLocation tagLoc);
 
-    bool trace_scanning = false;
-
     Own<ast::TranslationUnit> parse(
             const std::string& filename, FILE* in, ErrorReport& errorReport, DebugReport& debugReport);
     Own<ast::TranslationUnit> parse(
@@ -80,6 +78,21 @@ class ParserDriver {
     void warning(const SrcLocation& loc, const std::string& msg);
     void error(const SrcLocation& loc, const std::string& msg);
     void error(const std::string& msg);
+
+    std::optional<std::filesystem::path> searchIncludePath(
+            const std::string& IncludeString, const SrcLocation& IncludeLoc);
+
+    bool canEnterOnce(const SrcLocation& onceLoc);
+
+    void addComment(const SrcLocation& Loc, const std::stringstream& Content);
+
+    Own<ast::TranslationUnit> translationUnit;
+
+    bool trace_scanning = false;
+
+    std::set<std::pair<std::filesystem::path, int>> VisitedLocations;
+
+    std::deque<std::pair<SrcLocation, std::string>> ScannedComments;
 };
 
 }  // end of namespace souffle