Windows port, including zipping (#29)

.github/workflows/build.yml

@@ -110,3 +110,45 @@ jobs:
     - name: run demo
       run: ./ci-default --unity --run-demo
 
+  build-windows:
+    runs-on: windows-latest
+    env:
+      CCACHE_TEMPDIR: C:\Users\runneradmin\.ccache-temp
+      CCACHE_DIR: C:\Users\runneradmin\.ccache
+    defaults:
+      run:
+        shell: msys2 {0}
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: recursive
+    # Volatile cache
+    - name: ccache
+      uses: actions/cache@v2
+      with:
+        path: ~/.ccache
+        key:          ccache03-${{ runner.os }}-build-${{ github.sha }}
+        restore-keys: ccache03-${{ runner.os }}-build-
+    # Less volatile cache
+    - name: data cache
+      uses: actions/cache@v2
+      with:
+        path: build/data/data/txt
+        key:          data-${{ hashFiles('build/data/data/txt/**/*') }}
+        restore-keys: data-${{ hashFiles('build/data/data/txt/**/*') }}
+    - uses: eine/setup-msys2@v2
+      with:
+        update: true
+        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-eigen3 mingw-w64-x86_64-unittest-cpp mingw-w64-x86_64-boost mingw-w64-x86_64-gsl git
+    - name: prepare environment
+      run: util/prep-env.sh
+# TODO: Use python later
+#    - name: install dependencies
+#      run: util/deps-pull.sh
+#    - name: build & install the unmanaged dependencies
+#      run: util/deps-build.sh
+    - name: build
+      run: ./ci-default --unity
+    - name: run demo
+      run: ./ci-default --unity --run-demo        
+

cmake/util.cmake

@@ -146,10 +146,22 @@ function (enjoCopyDir projName source dest)
 endfunction()
 
 function (enjoSymlink projName linkSrc linkDst)
+  if (WIN32)
+  	#add_custom_command(TARGET ${projName} POST_BUILD
+	       #COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/bin
+#	)
+#	add_custom_command(TARGET ${projName} POST_BUILD
+#	       COMMAND ${CMAKE_COMMAND} -E copy_directory
+#	       ${linkSrc}
+#	       ${linkDst}
+#	)
+	# TODO: Won't work for files
+   else()
     add_custom_command(TARGET ${projName} POST_BUILD
        COMMAND ${CMAKE_COMMAND} -E create_symlink ${linkSrc} ${linkDst}
        )
     #install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${target} bin/${link})")
+  endif()
 endfunction()
 function (enjoSymlinkData projName)
 	enjoSymlink(${projName} ${CMAKE_CURRENT_SOURCE_DIR}/data data)

src/lib-base/src/CacheDynamicNonTpl.cpp

@@ -5,6 +5,8 @@
 #include "ConfigMan.h"
 #include "ConfigTF2.h"
 
+#include <EnjoLibBoost/Filesystem.hpp>
+
 #include <Util/FileUtils.hpp>
 #include <Util/Str.hpp>
 #include <Util/Except.hpp>
@@ -33,13 +35,13 @@ EnjoLib::Str CacheDynamicNonTpl::GetFilename() const
 {
     const ConfigDirs cfgDirs;
     const EnjoLib::Str dirFinal = cfgDirs.DIR_CACHE + '/' + m_name + '/';
-    FileUtils().CreateDirIfNotExistsLinux(cfgDirs.DIR_CACHE);
-    FileUtils().CreateDirIfNotExistsLinux(dirFinal);
+    Filesystem().CreateDirIfNotExists(cfgDirs.DIR_CACHE);
+    Filesystem().CreateDirIfNotExists(dirFinal);
     const char d = '_';
     EnjoLib::Osstream ossMT;
     if (m_isMT)
         ossMT << "MT" << d;
-        
+
     const EnjoLib::Str extension = ".bin" + cfgDirs.EXT_ARCHIVE;
     //const string extension = ".txt";
     return dirFinal + ossMT.str() + m_sym + d + m_per + extension;

src/lib-base/src/ConfigTxt.cpp

@@ -3,6 +3,8 @@
 #include "ConfigTxt.h"
 #include "ConfigDirs.h"
 
+#include <EnjoLibBoost/Filesystem.hpp>
+
 #include <Util/Tokenizer.hpp>
 #include <Util/CharManipulationsTplVec.hpp>
 #include <Ios/Ofstream.hpp>
@@ -33,7 +35,7 @@ ConfigTxt::~ConfigTxt()
 {
     const EnjoLib::Str delim = EnjoLib::Str(" ") + m_delim + ' ';
     const EnjoLib::Str outFile = GetFileName() + ".out";
-    EnjoLib::FileUtils().CreateDirIfNotExistsLinux(ConfigDirs().DIR_CFG);
+    EnjoLib::Filesystem().CreateDirIfNotExists(ConfigDirs().DIR_CFG);
     EnjoLib::Ofstream f(outFile);
 
     bool hadMissing = false;

src/lib-base/src/IConfig.cpp

@@ -2,6 +2,8 @@
 
 #include "ConfigDirs.h"
 
+#include <EnjoLibBoost/Filesystem.hpp>
+
 #include <Util/FileUtils.hpp>
 #include <Ios/Ofstream.hpp>
 #include <Ios/Ifstream.hpp>
@@ -33,7 +35,7 @@ bool IConfig::PrepareReadStream(EnjoLib::Istream & ifs) const
 
 void IConfig::Write() const
 {
-    EnjoLib::FileUtils().CreateDirIfNotExistsLinux(ConfigDirs().DIR_CFG);
+    EnjoLib::Filesystem().CreateDirIfNotExists(ConfigDirs().DIR_CFG);
     EnjoLib::Ofstream f(GetFileName());
     f << GetConfigString();
     //cout << GetConfigString() << endl;

src/lib-base/src/OptiVar.cpp

@@ -5,6 +5,8 @@
 #include "ConfigOpti.h"
 #include "ConfigGlob.h"
 
+#include <EnjoLibBoost/Filesystem.hpp>
+
 #include <Ios/Ofstream.hpp>
 #include <Ios/Ifstream.hpp>
 #include <Math/RandomMath.hpp>
@@ -76,7 +78,7 @@ void OptiVarF::Init()
                 {
                     LOGL << "OptiVar: FOUND! = " << pathVar << ", val = " << val << Nl;
                 }
-                    
+
 
                 f.close(); // For debugging val
             }
@@ -142,7 +144,7 @@ EnjoLib::VecF OptiVarF::GetSpaceRandom(float valMin, float valMax, int number)
 {
     EnjoLib::VecF ret;
     const RandomMath rmath;
-    
+
     for (int i = 0; i < number; ++i)
     {
         const double rnd = rmath.Rand(valMin, valMax);
@@ -153,13 +155,13 @@ EnjoLib::VecF OptiVarF::GetSpaceRandom(float valMin, float valMax, int number)
 
 void OptiVarF::StoreVariable(const IDataProvider & period) const
 {
-    FileUtils fu;
+    Filesystem fu;
     OptimizerUtils ou;
     bool release = gcfgMan.cfgOpti->OPTI_USE_RELEASE;;
     const EnjoLib::Str & baseDir = ou.GetVarBaseDir(release);
     const EnjoLib::Str & dir = ou.GetVarDir(period, release);
-    fu.CreateDirIfNotExistsLinux(baseDir);
-    fu.CreateDirIfNotExistsLinux(dir);
+    fu.CreateDirIfNotExists(baseDir);
+    fu.CreateDirIfNotExists(dir);
     const EnjoLib::Str & pathVar = ou.GetVarPath(period, varId, release);
     Ofstream f(pathVar);
     f << val << '\n';

src/lib-base/src/OptimizerUtils.cpp

@@ -1,10 +1,12 @@
 #include "OptimizerUtils.h"
 #include "ConfigDirs.h"
 #include "IDataProvider.h"
-#include <Util/FileUtils.hpp>
 #include "TradeUtil.h"
 #include "SymbolAlternatives.h"
 
+#include <EnjoLibBoost/Filesystem.hpp>
+
+#include <Util/FileUtils.hpp>
 
 using namespace std;
 using namespace EnjoLib;
@@ -19,7 +21,7 @@ EnjoLib::Str OptimizerUtils::GetVarBaseDir(bool release) const
         return cfg.DIR_OPTI_RELEASE;
     else
     {
-        FileUtils().CreateDirIfNotExistsLinux(cfg.DIR_OPTI_DEVEL);
+        Filesystem().CreateDirIfNotExists(cfg.DIR_OPTI_DEVEL);
         return cfg.DIR_OPTI_DEVEL;
     }
 

src/lib-base/src/TicksFilesExt.cpp

@@ -1,9 +1,12 @@
 #include "TicksFilesExt.h"
-#include <Util/FileUtils.hpp>
 #include "ConfigDirs.h"
 #include "ConfigSym.h"
 #include "ConfigMan.h"
 
+#include <EnjoLibBoost/Filesystem.hpp>
+
+#include <Util/FileUtils.hpp>
+
 using namespace std;
 
 TicksFilesExt::TicksFilesExt(EnjoLib::Str symbolName, EnjoLib::Str fileName, EnjoLib::Str periodName, EnjoLib::Str typeName)
@@ -27,10 +30,11 @@ TicksFilesExt::TicksFilesExt(EnjoLib::Str symbolName, EnjoLib::Str fileName, Enj
     EnjoLib::Str dirBin = dirBase + symbolName + "/";
     binaryFile = dirBin   + symbolPer + ".bin";
     textFile   = fileBase + fileExt;
-    txtZipFile = textFile + ".gz";
+    txtGZipFile = textFile + ".gz";
+    txtZipFile  = textFile + ".zip";
     binZipFile = binaryFile + archiveFmt;
 
-    EnjoLib::FileUtils().CreateDirIfNotExistsLinux(dirBin);
+    EnjoLib::Filesystem().CreateDirIfNotExists(dirBin);
 }
 
 TicksFilesExt::~TicksFilesExt(){}

src/lib-base/src/TicksFilesExt.h

@@ -11,6 +11,7 @@ class TicksFilesExt
 
         EnjoLib::Str binaryFile;
         EnjoLib::Str textFile;
+        EnjoLib::Str txtGZipFile;
         EnjoLib::Str txtZipFile;
         EnjoLib::Str binZipFile;
     protected:

src/lib-base/src/TicksProviderBinary.cpp

@@ -14,6 +14,7 @@
 #include <Util/CoutBuf.hpp>
 #include <Ios/Osstream.hpp>
 #include <Ios/Sstream.hpp>
+#include <Ios/Ifstream.hpp>
 
 #include <STD/Iostream.hpp>
 
@@ -157,13 +158,24 @@ CorPtr<ITicks> TicksProviderBinary::StoreOrRestore(const EnjoLib::Str & symbol,
     {
         return RestoreZipped(tfx.binZipFile);
     }
+    else if (fu.FileExists(tfx.textFile))
+    {
+        Ifstream ifs(tfx.textFile);
+        const auto & ticks = ReadFile(fileName, ifs); /// TODO: Join with UnzipTxt(). Used for Windows for now.
+        Archive(fileName, ticks);
+        return CorPtr<ITicks>(new Ticks(ticks));
+    }
     else if (fu.FileExists(tfx.txtZipFile))
     {
         return UnzipTxt(symbol, fileName, tfx.txtZipFile);
     }
+    else if (fu.FileExists(tfx.txtGZipFile))
+    {
+        return UnzipTxt(symbol, fileName, tfx.txtGZipFile);
+    }
     else
     {
-        throw ExceptInvalidArg("File " + tfx.txtZipFile + " doesn't exist");
+        throw ExceptInvalidArg("File " + tfx.txtGZipFile + " doesn't exist");
     }
 }
 

src/qt-lib/src/Monster.cpp

@@ -210,7 +210,7 @@ void Monster::InitBufs(const IPeriod & per) const
             Ofstream fopens(baseDir + period.GetSymbolPeriodId() + "-opens.dat");
             Ofstream fcloses(baseDir + period.GetSymbolPeriodId() + "-closes.dat");
             Ofstream fatr   (baseDir + period.GetSymbolPeriodId() + "-atr.dat");
-            FileUtils().CreateDirIfNotExistsLinux(baseDir);
+            Filesystem().CreateDirIfNotExists(baseDir);
             fopens << "\"Opens\"\n";
             fcloses << "\"Closes\"\n";
             fatr    << "\"ATR\"\n";

util/build.py

@@ -9,6 +9,7 @@
 import os
 import sys
 import time
+import shutil
 import platform
 import subprocess
 import argparse
@@ -77,11 +78,22 @@ def build(args):
 
     prefix = "set -e\n"
     print("System = " + platform.system())
-    if platform.system() == 'Darwin':
+    #if platform.system() == 'Darwin':
         #prefix += 'PATH="$(brew --prefix qt5)/bin:$PATH"' # https://github.com/leela-zero/leela-zero/issues/2177 
-        pass
+    #    pass
+    if platform.system() == 'Windows':
+        prefix = ""
+        #args.generator = "MinGW Makefiles"
+        args.generator = "MSYS Makefiles"
+        MINGW_PREFIX="x86_64-w64-mingw32"
+        cccompiler=MINGW_PREFIX + "-gcc"
+        cpcompiler=MINGW_PREFIX + "-g++"
+        make = " mingw32-make "
+        args.compiler = cccompiler
 
-    cmd = prefix + " cmake -G '{}'".format(args.generator) + ' -S {} -B .'.format(path)
+    cmd = prefix + ' cmake  -S {} -B .'.format(path)
+    if args.generator:
+        cmd += NL + '-G "{}"'.format(args.generator)
     cmd += NL + '-DUSE_STATIC={}' .format(OFF if args.shared else ON)
     cmd += NL + '-DUSE_DEBUG={}'  .format(ON  if args.debug  else OFF)
     cmd += NL + '-DUSE_UNITY={}'  .format(ON  if args.unity  else OFF)
@@ -90,14 +102,23 @@ def build(args):
     cmd += NL + '-DBUILD_BOOST={}'.format(ON) # Required
     if args.compiler:
         cmd += NL + '-DCMAKE_C_COMPILER="{}"'.format(cccompiler) 
-        cmd += NL + '-DCMAKE_CXX_COMPILER="{}"'.format(cpcompiler)    
-    cmd += NL + '&&' +  make + proc + '||' + make + '&&' + make + proc_local + 'install'
-    cmd += NL + '&& (' + ctest + proc_local + '||' + ctest + ')'
+        cmd += NL + '-DCMAKE_CXX_COMPILER="{}"'.format(cpcompiler)
+        if platform.system() == 'Windows':
+            cmd += NL + '-DCMAKE_MAKE_PROGRAM="{}"'.format(make.strip())
+
+    if platform.system() == 'Windows':
+        cmd += NL + '&&' +  make + proc
+        subprocess.run(cmd, shell=True, check=True)
+        subprocess.run(make + " install", shell=True, check=True)
+        subprocess.run(ctest, shell=True, check=True)   
+    else:
+        cmd += NL + '&&' +  make + proc + '||' + make + '&&' + make + proc_local + 'install'
+        cmd += NL + '&& (' + ctest + proc_local + '||' + ctest + ')'
 
-    print('Build command:\n')
-    print(cmd)
-    print('')
-    proc = subprocess.run(cmd, shell=True, check=True)
+        print('Build command:\n')
+        print(cmd)
+        print('')
+        proc = subprocess.run(cmd, shell=True, check=True)
 
     #print(cmd)
 def run_demo(args):
@@ -107,6 +128,12 @@ def run_demo(args):
     cmd += '&& ./tsqsim --help'
     cmd += '&& ./tsqsim'
 
+    if platform.system() == 'Windows':
+        #subprocess.run("ls ../..", shell=True, check=True)
+        shutil.move('../../data', '.') # TODO: Ugly
+        shutil.move('../../../src/tsqsim/scripts', '.') # Even uglier
+        cmd = "tsqsim.exe --help && tsqsim.exe"
+
     proc = subprocess.run(cmd, shell=True, check=True)
     proc = subprocess.run(cmd, shell=True, check=True) # Run again to test the deserialization
 

util/deps-build.sh

@@ -2,10 +2,13 @@
 
 echo "Building dependencies"
 # externals/URT/build/build-urt.sh # Not needed anymore. Using CMakeLists.txt
-if [ "$(uname)" != "Darwin" ]; then
+if [ "$(uname)" == "Linux" ]; then
 	externals/scripts/build-qcustomplot.sh
 fi
-
+#elif [ "$(uname)" == "MinGw" ]; then
+#	echo "TODO: install deps"	
+#else
+#elif [ "$(uname)" == "Dawrin" ]; then
 
 # Other dependencies here...
 

util/deps-pull.sh

@@ -10,6 +10,8 @@ if [ "$(uname)" == "Darwin" ]; then
 	#brew link -v qt5 --force
 	HOMEBREW_NO_AUTO_UPDATE=1 brew install boost openssl ccache gnuplot eigen gsl unittest-cpp
 	# TODO: install qt instead of qt5, after moving to latest QCustomPlot. Then the "--prefix" and "link" aren't needed.
+elif [ "$(uname)" == "MinGw" ]; then
+	echo "TODO: install deps"	
 else
 	sudo apt update; sudo apt -y install build-essential cmake ccache gnuplot libssl-dev libeigen3-dev libunittest++-dev libgsl-dev libboost-system-dev libboost-filesystem-dev libboost-iostreams-dev libboost-serialization-dev libboost-program-options-dev libboost-date-time-dev libboost-atomic-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools
 fi