From 0584bcba589c9a013212c6c7aaf307f85e6a2361 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Thu, 11 Jun 2020 16:40:10 -0400 Subject: [PATCH 01/13] v2.0.0 OOD Branch --- etc/Notes | 124 ---- etc/PlotTest.R | 39 +- etc/Test.cc | 6 +- etc/ccm.cc | 80 +++ etc/check | 27 + etc/dataFrame.cc | 72 ++ etc/edm.cc | 142 ++++ etc/embed.cc | 66 ++ etc/eval.cc | 147 ++++ etc/mview.cc | 107 +++ etc/test | 8 + src/API.cc | 547 +++++++++++++++ src/API.h | 327 +++++++++ src/AuxFunc.cc | 496 ------------- src/AuxFunc.h | 52 -- src/CCM.cc | 856 +++++++---------------- src/CCM.h | 36 + src/Common.cc | 94 +-- src/Common.h | 335 +-------- src/DataFrame.h | 226 +++--- src/{DateTimeUtil.cc => DateTime.cc} | 33 +- src/DateTime.h | 15 +- src/EDM.cc | 88 +++ src/EDM.h | 67 ++ src/EDM_Formatting.cc | 347 +++++++++ src/EDM_Neighbors.cc | 469 +++++++++++++ src/EDM_Neighbors.h | 19 + src/Eval.cc | 415 ++++++----- src/Interface.cc | 7 - src/Multiview.cc | 702 +++++++++---------- src/Multiview.h | 34 + src/Neighbors.cc | 357 ---------- src/Neighbors.h | 40 -- src/Parameter.cc | 185 ++--- src/Parameter.h | 144 ++-- src/SMap.cc | 389 +++------- src/SMap.h | 37 + src/Simplex.cc | 304 +++----- src/Simplex.h | 29 + src/Version.h | 3 + src/makefile | 88 +-- src/makefile.windows | 74 -- tests/DateTimeTest.cc | 10 +- tests/MultiviewTest.cc | 2 +- tests/TestCommon.h | 7 +- tests/data/CCM_anch_sst_cppEDM_valid.csv | 28 +- tests/data/CCM_anch_sst_pyEDM.csv | 15 - tests/data/Multiview_combos_valid.csv | 12 +- tests/data/Multiview_pred_valid.csv | 196 +++--- tests/data/Smap_circle_coef.csv | 100 --- 50 files changed, 4181 insertions(+), 3822 deletions(-) delete mode 100644 etc/Notes create mode 100644 etc/ccm.cc create mode 100755 etc/check create mode 100644 etc/dataFrame.cc create mode 100644 etc/edm.cc create mode 100644 etc/embed.cc create mode 100644 etc/eval.cc create mode 100644 etc/mview.cc create mode 100755 etc/test create mode 100644 src/API.cc create mode 100644 src/API.h delete mode 100644 src/AuxFunc.cc delete mode 100644 src/AuxFunc.h create mode 100644 src/CCM.h rename src/{DateTimeUtil.cc => DateTime.cc} (86%) create mode 100644 src/EDM.cc create mode 100644 src/EDM.h create mode 100644 src/EDM_Formatting.cc create mode 100644 src/EDM_Neighbors.cc create mode 100644 src/EDM_Neighbors.h delete mode 100644 src/Interface.cc create mode 100644 src/Multiview.h delete mode 100644 src/Neighbors.cc delete mode 100644 src/Neighbors.h create mode 100644 src/SMap.h create mode 100644 src/Simplex.h delete mode 100644 src/makefile.windows delete mode 100644 tests/data/CCM_anch_sst_pyEDM.csv delete mode 100644 tests/data/Smap_circle_coef.csv diff --git a/etc/Notes b/etc/Notes deleted file mode 100644 index 9e6cf9a..0000000 --- a/etc/Notes +++ /dev/null @@ -1,124 +0,0 @@ - -General notes - -1) The OSX XCode compiler/linker seems to be incompatible with the C++11 - standard implementation allowing template classes to be distributed into - declarations (.h) and implementation (.cc). However, this does entail - explicit declaration of specialised template types in the .cc file. - To support OSX, DataFrame.h contains both declarations and implementations. - See: libstdc++_Notes.txt. - -2) The code relies heavily on class and data containers without explicit - heap allocation. This facilitates garbage collection. It may be that - copy-on-return for large data objects creates a performance issue. - If the code encounters massive data objects/large problems, this may - pose a limitation. The use of object references may alleviate this. - -3) The LAPACK library is used for the S-map solver. It calls the dgelss_() - FORTRAN function. On nix systems, OS-installed libraries are used. - On Windows, the Windows for LAPACK libraries are needed. These are - built with mingw, so mingw libraries must be available. It further - seems that the mingw 32-bit libraries have to be used. See below. - -4) pyEDM implements a wrapper for cppEDM using PyBind11. This requires - a MSVC build since Python extensions must be compiled with the same - compiler as the Python interpreter. - -5) rEDM implements a wrapper for cppEDM using Rcpp. R includes the - RTools package for building Windows binaries, and, it's own - LAPACK libraries. - - - --------------------------------------------------------------------- -CCM vs Simplex : lib, pred and libSize compatibility --------------------------------------------------------------------- - -Time delay embedding removes the first tau(E-1) rows of data to exclude -partial data vectors. Accordingly, the maximum index of lib or pred -rows allowed is the number of data observation rows minus tau(E-1). - -If Tp > 0, then Tp additional rows are forecast beyond pred. - -Simplex() is a wrapper for SimplexProjection(). With default embedded = false -it embeds the data and performs cross mapping from lib NN at each pred -observation with a call to SimplexProjection(). The lib and pred indices are -adjusted to account for the removal of partial data vectors so that output -data align with the observations. - -Simplex with E = 3, Tp = 0, lib = pred = "1 10" on a dataset with 12 -observation rows will have a NN library of size 8 = 10 - 2 from embedding -of the first 10 observation rows. Predictions will be output at rows -3 - 10 since the max pred is 10 and the first two observations were deleted -as partial data E-dimensional vectors where no neighbors can be computed. - -CCM() embeds the entire data, then uses the embedding size (rows) as both -lib and pred to perform cross mapping with a call to SimplexProjection() -using embedded = true (no embedding performed on data). If random = false -then lib is assigned sequentially up to the current library size. There -is no external notion of lib & pred. - -CCM with E = 3 and libSizes = "10 10 1", random = false, will read the 12 -observation rows, perform embedding discarding the first two rows, then -predict 10 points at a library size of 10 using sequential lib indices. - -CCM was not intended to be used for non-convergent cross mapping and has -no notion of pred, only library size. As a result, it does not have the -same alignment of lib & pred as in Simplex(), and, forms the embedding -based on all available observations (not just lib). - -Continuing with the 12 observation data, 10 analysis row example, the -following parameter combinations give congruent rho results: - -******** Tp = 0 ********** -Simplex: data.csv lib = "1 10" pred = "3 12" columns = x target = y -E = 3 Tp = 0 -rho 0.911001 RMSE 0.259461 MAE 0.198365 - -CCM: data.csv columns = x target = y E = 3 Tp = 0 libSizes = "10 10 1" ----------------------------------------------- - 10.0000 0.9110 0.5470 ----------------------------------------------- - -******** Tp = 1 ********** -Simplex: data.csv lib = "1 10" pred = "3 11" columns = x target = y -E = 3 Tp = 1 -rho 0.868089 RMSE 0.198611 MAE 0.15669 - -CCM: data.csv columns = x target = y E = 3 Tp = 1 libSizes = "9 9 1" ----------------------------------------------- - 9.0000 0.8681 0.7016 ----------------------------------------------- - -******** Tp = 2 ********** -Simplex: data.csv lib = "1 10" pred = "3 10" columns = x target = y -E = 3 Tp = 2 -rho 0.831373 RMSE 0.170693 MAE 0.132672 - -CCM: data.csv columns = x target = y E = 3 Tp = 2 libSizes = "8 8 1" - - - --------------------------------------------------------------------- -C++ std::sort() for finding nearest neighbors --------------------------------------------------------------------- -Investigation was made into using std::sort instead of the heuristic -used in FindNeighbors() and CCMNeighbors(). Since we seek to sort -coupled pairs, this was done using a vector -of distance, index pairs: std::vector< std::pair >. - -This is then sorted with std::sort(). Any elements that were not inserted -into the vector, e.g. pred = lib degenerate rows, will have 0 distance -and get sorted to the front. These are removed by tracking how many -elements were not inserted into the vector. - -Ties in distances are sorted as one would expect, following the strict -weak odering rules. This works fine for FindNeighbors() used for -Simplex() & SMap(). - -In CCMNeighbors() used for CCM(), it seems that subtle differences in -handling ties (which neighbor out of a set of equal distances) creates -substantial problems in CCM results. This could be resolved with the -use of a std::sort() comparison function invoking: x.first <= y.first, -however, this <= does not follow the strict weak odering mandated by -std::sort(), which can cause faults if used with std::sort(). diff --git a/etc/PlotTest.R b/etc/PlotTest.R index 2dbca2c..71530cd 100644 --- a/etc/PlotTest.R +++ b/etc/PlotTest.R @@ -1,11 +1,37 @@ +## Run() and Clean() are executed at the bottom of this code. + +library(tcltk) + #--------------------------------------------------------------------- #--------------------------------------------------------------------- Run = function( path = './' ) { + prompt = "press to close" + extra = "" + + X11() + par( mar = c(2, 3.8, 0.5, 1), mgp = c(2.2, 0.8, 0), cex = 1.3, + cex.axis = 1.5, cex.lab = 1.5, mfrow = c(7, 1) ) PlotSimplexSmap( path ) - PlotSMapCircle ( path ) - PlotCCM ( path ) - PlotEval ( path ) + capture = tk_messageBox(message = prompt, detail = extra) + + X11() + par( mar = c(2, 3.8, 0.5, 1), mgp = c(2.2, 0.8, 0), cex = 1.3, + cex.axis = 1.5, cex.lab = 1.5, mfrow = c(3, 1) ) + PlotSMapCircle( path ) + capture = tk_messageBox(message = prompt, detail = extra) + + X11() + par( mar = c(3.5, 3.8, 0.5, 1), mgp = c(2.2, 0.8, 0), cex = 1.3, + cex.axis = 1.5, cex.lab = 1.5, mfrow = c(1, 1) ) + PlotCCM( path ) + capture = tk_messageBox(message = prompt, detail = extra) + + X11() + par( mar = c(3.5, 3.8, 0.5, 1), mgp = c(2.2, 0.8, 0), cex = 1.3, + cex.axis = 1.5, cex.lab = 1.5, mfrow = c(3, 1) ) + PlotEval( path ) + capture = tk_messageBox(message = prompt, detail = extra) } #--------------------------------------------------------------------- @@ -18,7 +44,7 @@ Clean = function( path = './' ) { "smap_3sp_coeff.csv", "smap_circ_coeff.csv", "smap_circle.csv", - "ccm.csv", + "ccm-out.csv", "EmbedDimOut.csv", "PredictIntervalOut.csv", "PredictNonlinearOut.csv", @@ -134,7 +160,7 @@ PlotSMapCircle = function( path = './' ) { #--------------------------------------------------------------------- PlotCCM = function( path = './', - file = 'ccm.csv', + file = 'ccm-out.csv', col_i = 2, target_i = 3 ) { @@ -184,3 +210,6 @@ PlotEval = function( xlab='Theta', ylab='rho', col = 'blue' ) } + +Run() +Clean() diff --git a/etc/Test.cc b/etc/Test.cc index 7fa9eca..7002767 100644 --- a/etc/Test.cc +++ b/etc/Test.cc @@ -2,9 +2,7 @@ // g++ Test.cc -o Test -std=c++11 -I../src -L../lib -lstdc++ -lEDM -lpthread -llapack -O3 // -g -DDEBUG -#include "Common.h" -#include "Neighbors.h" -#include "Embed.h" +#include "API.h" //---------------------------------------------------------------- // Suite of tests for API functionality @@ -248,7 +246,7 @@ int main( int argc, char *argv[] ) { CCM( "../data/", // pathIn "sardine_anchovy_sst.csv", // dataFile "./", // pathOut - "ccm.csv", // predictFile + "ccm-out.csv", // predictFile 3, // E 0, // Tp 0, // knn diff --git a/etc/ccm.cc b/etc/ccm.cc new file mode 100644 index 0000000..db063e0 --- /dev/null +++ b/etc/ccm.cc @@ -0,0 +1,80 @@ + +// g++ ccm.cc -o ccm -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -llapack -lpthread + +#include "API.h" + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +int main( int argc, char *argv[] ) { + + std::string dataFile = "../data/sardine_anchovy_sst.csv"; + std::string columns = "anchovy"; + std::string target = "np_sst"; + std::string fileOut = "ccm-out.csv"; + int E = 3; + int Tp = 0; + std::string libSizes = "10 75 5"; + int sample = 0; + bool random = false; // 'y' = true + bool replacement = false; // 'y' = true + bool verbose = false; // 'y' = true + + if ( argc > 1 ) { dataFile = argv[1]; } + if ( argc > 2 ) { columns = argv[2]; } + if ( argc > 3 ) { target = argv[3]; } + if ( argc > 4 ) { fileOut = argv[4]; } + if ( argc > 5 ) { std::stringstream ss( argv[5] ); ss >> E; } + if ( argc > 6 ) { std::stringstream ss( argv[6] ); ss >> Tp; } + if ( argc > 7 ) { libSizes = argv[7]; } + if ( argc > 8 ) { std::stringstream ss( argv[8] ); ss >> sample; } + if ( argc > 9 ){ random = ( *argv[9] == 'y' ? true : false ); } + if ( argc > 10 ){ replacement = ( *argv[10] == 'y' ? true : false ); } + if ( argc > 11 ){ verbose = ( *argv[11] == 'y' ? true : false ); } + + if ( verbose ) { + std::cout << dataFile << " " << columns << " " << target << " " + << E << " " << Tp << " " << libSizes << std::endl; + } + + try { + CCMValues ccmOut = CCM( "./", // pathIn, + dataFile, + "./", // pathOut, + fileOut, // predictFile, + E, + Tp, + 0, // knn, + -1, // tau, + columns, + target, + libSizes, // libSizes_str, + sample, + random, + replacement, + 0, // seed, + false, // includeData + verbose ); + + DataFrame dataFrame = ccmOut.AllLibStats; + dataFrame.MaxRowPrint() = dataFrame.NRows(); + + if ( verbose ) { + std::cout << dataFrame; + } + } + + catch ( const std::exception& e ) { + std::cout << "Exception caught in main:\n"; + std::cout << e.what() << std::endl; + return -1; + } + catch (...) { + std::cout << "Unknown exception caught in main.\n"; + return -1; + } + + std::cout << "normal termination\n"; + + return 0; +} diff --git a/etc/check b/etc/check new file mode 100755 index 0000000..149aed3 --- /dev/null +++ b/etc/check @@ -0,0 +1,27 @@ +#!/bin/bash +tput reset +echo "------- Building -------------------------------------------" +g++ edm.cc -o edm -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -llapack -pthread +g++ embed.cc -o embed -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -llapack -pthread +g++ ccm.cc -o ccm -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -llapack -lpthread +g++ mview.cc -o mview -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -lpthread -llapack +g++ eval.cc -o eval -std=c++11 -I../src -L../lib -lstdc++ -lEDM -lpthread -llapack +echo "------- Embed ----------------------------------------------" +./embed +# head embed_out.csv -n 4 +echo "------- Simplex ---------------------------------------------" +./edm +# head out.csv -n 3 +# tail out.csv -n 2 +echo "------- CCM -------------------------------------------------" +./ccm ../data/sardine_anchovy_sst.csv anchovy np_sst ccm-out.csv 3 0 "10 75 5" 100 y n n +head ccm-out.csv -n 2 +tail ccm-out.csv -n 2 +echo "------- Multiview -------------------------------------------" +./mview ../data/block_3sp.csv "1 100" "101 195" "x_t y_t z_t" x_t mview-out.csv 0 3 1 0 0 n n 1 +echo "------- EmbedDimension --------------------------------------" +./eval E ../data/TentMap_rEDM.csv "1 100" "201 500" TentMap TentMap +./eval Tp ../data/TentMap_rEDM.csv "1 100" "201 500" TentMap TentMap +./eval theta ../data/TentMapNoise_rEDM.csv "1 100" "201 500" TentMap TentMap + +rm -f eval mview ccm edm embed *.csv diff --git a/etc/dataFrame.cc b/etc/dataFrame.cc new file mode 100644 index 0000000..f9898b3 --- /dev/null +++ b/etc/dataFrame.cc @@ -0,0 +1,72 @@ + +// g++ dataFrame.cc -o dataFrame -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -llapack -pthread + +#include "API.h" +#include "DataFrame.h" + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +int main( int argc, char *argv[] ) { + + std::string dataFile = "../data/block_3sp.csv"; + std::string columns = "x_t"; + std::string fileOut = "out.csv"; + bool verbose = false; // 'y' = true + + if ( argc > 1 ) { dataFile = argv[1]; } + if ( argc > 2 ) { columns = argv[2]; } + if ( argc > 3 ) { fileOut = argv[3]; } + if ( argc > 4 ){ verbose = ( *argv[4] == 'y' ? true : false ); } + + if ( verbose ) { + std::cout << dataFile << " cols " << columns << std::endl; + } + + try { + //---------------------------------------------------------- + //---------------------------------------------------------- + + DataFrame< double > DF; + + DF = DataFrame < double >( "", // pathIn + dataFile ); // dataFile + + if ( verbose ) { + DF.MaxRowPrint() = DF.NRows(); + std::cout << DF; + } + + size_t rowi[] = { 0,2,4,6,8,10 }; + std::vector< size_t > rows( rowi, rowi + sizeof(rowi)/sizeof(size_t) ); + DataFrame < double > DFrows = DF.DataFrameFromRowIndex( rows ); + std::cout << DFrows; + + size_t coli[] = { 1 }; + std::vector< size_t > cols( coli, coli + sizeof(coli)/sizeof(size_t) ); + DataFrame < double > DFcols = DF.DataFrameFromColumnIndex( cols ); + std::cout << DFcols; + + DataFrame < double > DFrowscols = + DFrows.DataFrameFromColumnIndex( cols ); + std::cout << DFrowscols; + + if ( fileOut.size() ) { + DF.WriteData( "", fileOut ); + } + } + + catch ( const std::exception& e ) { + std::cout << "Exception caught in main:\n"; + std::cout << e.what() << std::endl; + return -1; + } + catch (...) { + std::cout << "Unknown exception caught in main.\n"; + return -1; + } + + std::cout << "normal termination\n"; + + return 0; +} diff --git a/etc/edm.cc b/etc/edm.cc new file mode 100644 index 0000000..ef7d8b4 --- /dev/null +++ b/etc/edm.cc @@ -0,0 +1,142 @@ + +// g++ edm.cc -o edm -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -llapack -pthread + +#include "API.h" + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +int main( int argc, char *argv[] ) { + + std::string dataFile = "../data/block_3sp.csv"; + std::string lib = "1 100"; + std::string pred = "101 195"; + std::string columns = "x_t"; + std::string target = "x_t"; + std::string fileOut = "out.csv"; + int E = 3; + int Tp = 1; + int tau = -1; + std::string method = "simplex"; // or smap + int theta = 2; + bool embedded = false; // 'y' = true + bool verbose = false; // 'y' = true + bool const_pred = false; // 'y' = true + int exclusionR = 0; + + if ( argc > 1 ) { dataFile = argv[1]; } + if ( argc > 2 ) { lib = argv[2]; } + if ( argc > 3 ) { pred = argv[3]; } + if ( argc > 4 ) { columns = argv[4]; } + if ( argc > 5 ) { target = argv[5]; } + if ( argc > 6 ) { fileOut = argv[6]; } + if ( argc > 7 ) { std::stringstream ss( argv[7] ); ss >> E; } + if ( argc > 8 ) { std::stringstream ss( argv[8] ); ss >> Tp; } + if ( argc > 9 ) { std::stringstream ss( argv[9] ); ss >> tau; } + if ( argc > 10 ) { method = argv[10]; } + if ( argc > 11 ){ std::stringstream ss( argv[11] ); ss >> theta; } + if ( argc > 12 ){ embedded = ( *argv[12] == 'y' ? true : false ); } + if ( argc > 13 ){ verbose = ( *argv[13] == 'y' ? true : false ); } + if ( argc > 14 ){ const_pred = ( *argv[14] == 'y' ? true : false ); } + if ( argc > 15 ){ std::stringstream ss( argv[15] ); ss >> exclusionR; } + + if ( verbose ) { + std::cout << method << " " << dataFile << " lib " << lib << " pred " + << pred << " cols " << columns << " target " << target + << " E " << E << " Tp " << Tp << " tau " << tau + << " theta " << theta << std::endl; + } + + try { + + //---------------------------------------------------------- + //---------------------------------------------------------- + bool simplex = true; + if ( method.find( "smap" ) != std::string::npos ) { simplex = false; } + + DataFrame dataFrame; + DataFrame coef; + + if ( simplex ) { + dataFrame = Simplex( "", // pathIn + dataFile, // dataFile + "./", // pathOut + fileOut, // predictFile + lib, // lib + pred, // pred + E, // E + Tp, // Tp + 0, // knn <<<<< CONSTANT + tau, // tau + exclusionR, // exclusionRadius + columns, // columns + target, // target + embedded, // embedded + const_pred, // const_predict + verbose ); // verbose + } + else { + SMapValues SM = SMap( "", // pathIn + dataFile, // dataFile + "./", // pathOut + fileOut, // predictFile + lib, // lib + pred, // pred + E, // E + Tp, // Tp + 0, // knn <<<<< CONSTANT + tau, // tau + theta, // theta + exclusionR, // exclusionRadius + columns, // columns + target, // target + "smap_coeff.csv", // smapFile + "", // derivatives + embedded, // embedded + const_pred, // const_predict + verbose ); // verbose + + dataFrame = SM.predictions; + coef = SM.coefficients; + } + + VectorError ve = ComputeError( + dataFrame.VectorColumnName( "Observations" ), + dataFrame.VectorColumnName( "Predictions" ) ); + + std::cout << method << " on " << dataFile << ":\n"; + std::cout << "rho " << ve.rho << " RMSE " << ve.RMSE + << " MAE " << ve.MAE << std::endl; + + if ( const_pred ) { + ve = ComputeError( + dataFrame.VectorColumnName( "Observations" ), + dataFrame.VectorColumnName( "Const_Predictions" ) ); + + std::cout << "rho_c " << ve.rho << " RMSE_c " << ve.RMSE + << " MAE_c " << ve.MAE << std::endl << std::endl; + } + + if ( verbose ) { + dataFrame.MaxRowPrint() = dataFrame.NRows(); + std::cout << dataFrame; + if ( not simplex ) { + std::cout << coef; + } + } + } + + catch ( const std::exception& e ) { + std::cout << "Exception caught in main:\n"; + std::cout << e.what() << std::endl; + return -1; + } + catch (...) { + std::cout << "Unknown exception caught in main.\n"; + return -1; + } + + std::cout << "normal termination\n"; + + return 0; +} diff --git a/etc/embed.cc b/etc/embed.cc new file mode 100644 index 0000000..e61a67c --- /dev/null +++ b/etc/embed.cc @@ -0,0 +1,66 @@ + +// g++ embed.cc -o embed -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -llapack -pthread + +#include "API.h" + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +int main( int argc, char *argv[] ) { + + std::string dataFile = "../data/block_3sp.csv"; + std::string columns = "x_t"; + std::string fileOut = "embed_out.csv"; + int E = 3; + int tau = -1; + bool verbose = false; // 'y' = true + + if ( argc > 1 ) { dataFile = argv[1]; } + if ( argc > 2 ) { columns = argv[2]; } + if ( argc > 3 ) { fileOut = argv[3]; } + if ( argc > 4 ) { std::stringstream ss( argv[4] ); ss >> E; } + if ( argc > 5 ) { std::stringstream ss( argv[5] ); ss >> tau; } + if ( argc > 6 ){ verbose = ( *argv[6] == 'y' ? true : false ); } + + if ( verbose ) { + std::cout << dataFile << " cols " << columns + << " E " << E << " tau " << tau << std::endl; + } + + try { + //---------------------------------------------------------- + //---------------------------------------------------------- + + DataFrame dataFrame; + + dataFrame = Embed( "", // pathIn + dataFile, // dataFile + E, // E + tau, // tau + columns, // columns + verbose ); // verbose + + if ( verbose ) { + dataFrame.MaxRowPrint() = dataFrame.NRows(); + std::cout << dataFrame; + } + + if ( fileOut.size() ) { + dataFrame.WriteData( "", fileOut ); + } + } + + catch ( const std::exception& e ) { + std::cout << "Exception caught in main:\n"; + std::cout << e.what() << std::endl; + return -1; + } + catch (...) { + std::cout << "Unknown exception caught in main.\n"; + return -1; + } + + std::cout << "normal termination\n"; + + return 0; +} diff --git a/etc/eval.cc b/etc/eval.cc new file mode 100644 index 0000000..638fdef --- /dev/null +++ b/etc/eval.cc @@ -0,0 +1,147 @@ + +// g++ eval.cc -o eval -std=c++11 -I../src -L../lib -lstdc++ -lEDM -lpthread -llapack + +#include "API.h" + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +int main( int argc, char *argv[] ) { + + std::string method = "E"; // E, Tp or theta + std::string dataFile = "../data/block_3sp.csv"; + std::string lib = "1 100"; + std::string pred = "101 180"; + std::string columns = "x_t"; + std::string target = "x_t"; + std::string fileOut = "out.csv"; + int E = 3; + int Tp = 1; + std::string theta = ""; + bool embedded = false; // 'y' = true + bool verbose = false; // 'y' = true + bool const_pred = false; // 'y' = true + int exclusionR = 0; + int nThreads = 1; + + if ( argc > 1 ) { method = argv[1]; } + if ( argc > 2 ) { dataFile = argv[2]; } + if ( argc > 3 ) { lib = argv[3]; } + if ( argc > 4 ) { pred = argv[4]; } + if ( argc > 5 ) { columns = argv[5]; } + if ( argc > 6 ) { target = argv[6]; } + if ( argc > 7 ) { fileOut = argv[7]; } + if ( argc > 8 ) { std::stringstream ss( argv[8] ); ss >> E; } + if ( argc > 9 ) { std::stringstream ss( argv[9] ); ss >> Tp; } + if ( argc > 10 ){ theta = argv[10]; } + if ( argc > 11 ){ embedded = ( *argv[11] == 'y' ? true : false ); } + if ( argc > 12 ){ verbose = ( *argv[12] == 'y' ? true : false ); } + if ( argc > 13 ){ const_pred = ( *argv[13] == 'y' ? true : false ); } + if ( argc > 14 ){ std::stringstream ss( argv[14] ); ss >> exclusionR; } + if ( argc > 15 ){ std::stringstream ss( argv[15] ); ss >> nThreads; } + + if ( verbose ) { + std::cout << method << " " << dataFile << " lib: " << lib + << " pred: " << pred << " " << columns << " " + << target << " E: " << E << " Tp: " << Tp + << " theta: " << theta << std::endl; + } + + try { + if ( method.compare( "E" ) == 0 ) { + + // EmbedDimension + DataFrame< double > EMBD = + EmbedDimension( "", // pathIn + dataFile, // dataFile + "./", // pathOut + fileOut, // predictFile + lib, // lib + pred, // pred + 10, // maxE <<<<< CONSTANT + Tp, // Tp + -1, // tau <<<<< CONSTANT + columns, // colNames + target, // targetName + embedded, // embedded + verbose, // verbose + nThreads ); // nThreads + + std::cout << "EmbedDimension:\n"; + if ( verbose ) { + std::cout << EMBD; + } + } + + else if ( method.compare( "Tp" ) == 0 ) { + + // PredictInterval + DataFrame< double > PD = + PredictInterval( "", // pathIn + dataFile, // dataFile + "./", // pathOut + fileOut, // predictFile + lib, // lib + pred, // pred + 10, // maxTp <<<<< CONSTANT + E, // E + -1, // tau <<<<< CONSTANT + columns, // colNames + target, // targetName + embedded, // embedded + verbose, // verbose + nThreads ); // nThreads + + std::cout << "PredictInterval:\n"; + if ( verbose ) { + std::cout << PD; + } + } + + else if ( method.compare( "theta" ) == 0 ) { + + // PredictNonlinear + DataFrame< double > NL = + PredictNonlinear( "", // pathIn + dataFile, // dataFile, + "./", // pathOut + fileOut, // predictFile + lib, // lib + pred, // pred + theta, // theta + E, // E + Tp, // Tp + 0, // knn + -1, // tau <<<<< CONSTANT + columns, // colNames + target, // targetName + embedded, // embedded + verbose, // verbose + nThreads ); // nThreads + + NL.MaxRowPrint() = 15; + std::cout << "PredictNonlinear:\n"; + if ( verbose ) { + std::cout << NL; + } + } + + else { + std::cout << "No method found" << std::endl; + } + } + + catch ( const std::exception& e ) { + std::cout << "Exception caught in main:\n"; + std::cout << e.what() << std::endl; + return -1; + } + catch (...) { + std::cout << "Unknown exception caught in main.\n"; + return -1; + } + + std::cout << "normal termination\n"; + + return 0; +} diff --git a/etc/mview.cc b/etc/mview.cc new file mode 100644 index 0000000..7f2a707 --- /dev/null +++ b/etc/mview.cc @@ -0,0 +1,107 @@ + +// g++ mview.cc -o mview -std=c++11 -g -I../src -L../lib -lstdc++ -lEDM -lpthread -llapack + +#include "API.h" + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +int main( int argc, char *argv[] ) { + + std::string dataFile = "../data/block_3sp.csv"; + std::string lib = "1 100"; + std::string pred = "101 195"; + std::string columns = "x_t y_t z_t"; + std::string target = "x_t"; + std::string fileOut = "mview-out.csv"; + int D = 0; + int E = 3; + int Tp = 1; + int multiview = 0; + int exclusionR = 0; + bool trainLib = false; // 'y' = true + bool verbose = true; // 'y' = true + int nThreads = 1; + + if ( argc > 1 ) { dataFile = argv[1]; } + if ( argc > 2 ) { lib = argv[2]; } + if ( argc > 3 ) { pred = argv[3]; } + if ( argc > 4 ) { columns = argv[4]; } + if ( argc > 5 ) { target = argv[5]; } + if ( argc > 6 ) { fileOut = argv[6]; } + if ( argc > 7 ) { std::stringstream ss( argv[7] ); ss >> D; } + if ( argc > 8 ) { std::stringstream ss( argv[8] ); ss >> E; } + if ( argc > 9 ) { std::stringstream ss( argv[9] ); ss >> Tp; } + if ( argc > 10) { std::stringstream ss( argv[10]); ss >> multiview; } + if ( argc > 11) { std::stringstream ss( argv[11]); ss >> exclusionR; } + if ( argc > 12) { trainLib = ( *argv[12] == 'y' ? true : false ); } + if ( argc > 13) { verbose = ( *argv[13] == 'y' ? true : false ); } + if ( argc > 14) { std::stringstream ss( argv[14] ); ss >> nThreads; } + + if ( verbose ) { + std::cout << dataFile << " lib: " << lib << " pred: " << pred + << " columns: " << columns << " target: " << target + << " D: " << D << " E: " << E << " Tp: " << Tp + << " multiview: " << multiview + << " exclusionR: " << exclusionR + << " trainLib: " << trainLib << std::endl; + } + + try { + //---------------------------------------------------------- + // + //---------------------------------------------------------- + MultiviewValues MV = + Multiview( "", // pathIn + dataFile, // dataFile + "./", // pathOut + fileOut, // predictFile + lib, // lib + pred, // pred + D, // D + E, // E + Tp, // Tp + 0, // knn <<<<< CONSTANT + -1, // tau <<<<< CONSTANT + columns, // columns + target, // target, + multiview, // multiview + exclusionR,// exclusionRadius + trainLib, // trainLib + false, // verbose, + nThreads );// nThreads + + DataFrame< double > MVPredictions = MV.Predictions; + + VectorError vemv = + ComputeError( MVPredictions.VectorColumnName( "Observations" ), + MVPredictions.VectorColumnName( "Predictions" ) ); + + std::cout << "Multiview()" << dataFile << "\nrho " << vemv.rho + << " MAE " << vemv.MAE + << " RMSE " << vemv.RMSE << std::endl; + + if ( verbose ) { + // Table of columns, names, rho, MAE RMSE + std::cout << std::endl; + std::vector< std::string > table = MV.ComboRhoTable; + for ( size_t row = 0; row < table.size(); row++ ) { + std::cout << table[ row ] << std::endl; + } + } + } + + catch ( const std::exception& e ) { + std::cout << "Exception caught in main:\n"; + std::cout << e.what() << std::endl; + return -1; + } + catch (...) { + std::cout << "Unknown exception caught in main.\n"; + return -1; + } + + std::cout << "normal termination\n"; + + return 0; +} diff --git a/etc/test b/etc/test new file mode 100755 index 0000000..d1a1aef --- /dev/null +++ b/etc/test @@ -0,0 +1,8 @@ +#!/bin/bash +tput reset +echo "------- Building -------------------------------------------" +g++ Test.cc -o Test -std=c++11 -I../src -L../lib -lstdc++ -lEDM -lpthread -llapack -O3 +echo "------- Test -----------------------------------------------" +./Test +echo "------- PlotTest -------------------------------------------" +Rscript PlotTest.R diff --git a/src/API.cc b/src/API.cc new file mode 100644 index 0000000..fc3e6b1 --- /dev/null +++ b/src/API.cc @@ -0,0 +1,547 @@ + +//---------------------------------------------------------------- +// Functions implemented here: +// Embed(), MakeBlock(), Simplex(), SMap(), CCM(), Multiview() +// +// Functions implemented in Eval.cc: +// EmbedDimension(), PredictInterval(), PredictNonlinear() +//---------------------------------------------------------------- + +#include "API.h" + +//---------------------------------------------------------------- +// Embed with file path/file input +//---------------------------------------------------------------- +DataFrame< double > Embed( std::string path, + std::string dataFile, + int E, // embedding dimension + int tau, // time step delay + std::string columns, // column names or indices + bool verbose ) { + + DataFrame< double > dataFrame( path, dataFile ); + DataFrame< double > embedded = Embed( std::ref( dataFrame ), + E, tau, columns, verbose ); + return embedded; +} + +//---------------------------------------------------------------- +// Embed with DataFrame input +//---------------------------------------------------------------- +DataFrame< double > Embed( DataFrame< double > & dataFrameIn, + int E, + int tau, + std::string columns, + bool verbose ) { + + // Parameter.Validate will convert columns into a vector of names + // or a vector of column indices + Parameters parameters = Parameters( Method::Embed, "", "", "", "", + "1 1", "1 1", E, 0, 0, tau, 0, 0, + columns, "", false, false, verbose ); + // Instantiate EDM object + EDM EDM_Embed = EDM( dataFrameIn, std::ref( parameters ) ); + + // Perform embedding : calls MakeBlock() API function + EDM_Embed.EmbedData(); + + return EDM_Embed.embedding; +} + +//------------------------------------------------------------------------ +// MakeBlock from dataFrame :: API function +// Ignores the first (or last) tau * (E-1) dataFrame rows of partial data. +// Does not validate parameters or columns, use EmbedData() +//------------------------------------------------------------------------ +DataFrame< double > MakeBlock( DataFrame< double > & dataFrame, + int E, + int tau, + std::vector columnNames ) +{ + if ( columnNames.size() != dataFrame.NColumns() ) { + std::stringstream errMsg; + errMsg << "MakeBlock: The number of columns in the dataFrame (" + << dataFrame.NColumns() << ") is not equal to the number " + << "of columns specified (" << columnNames.size() << ").\n";; + throw std::runtime_error( errMsg.str() ); + } + + if ( E < 1 ) { + std::stringstream errMsg; + errMsg << "MakeBlock(): E = " << E << " is invalid.\n" ; + throw std::runtime_error( errMsg.str() ); + } + + size_t NRows = dataFrame.NRows(); // number of input rows + size_t NColOut = dataFrame.NColumns() * E; // number of output columns + size_t NPartial = abs( tau ) * (E-1); // rows to shift & delete + + // Create embedded data frame column names X(t-0) X(t-1)... + std::vector< std::string > newColumnNames( NColOut ); + size_t newCol_i = 0; + for ( size_t col = 0; col < columnNames.size(); col ++ ) { + for ( int e = 0; e < E; e++ ) { + std::stringstream ss; + if ( tau < 0 ) { + ss << columnNames[ col ] << "(t-" << e << ")"; + } + else { + ss << columnNames[ col ] << "(t+" << e << ")"; + } + newColumnNames[ newCol_i ] = ss.str(); + newCol_i++; + } + } + + // Ouput data frame with tau * E-1 fewer rows + DataFrame< double > embedding( NRows - NPartial, NColOut, newColumnNames ); + + // To keep track of where to insert column in new data frame + size_t colCount = 0; + + // Slice to ignore rows with partial data + std::slice slice_i; + if ( tau < 0 ) { + slice_i = std::slice( NPartial, NRows - NPartial, 1 ); + } + else { + slice_i = std::slice( 0, NRows - NPartial, 1 ); + } + + // Shift column data and write to embedding data frame + for ( size_t col = 0; col < dataFrame.NColumns(); col++ ) { + // for each embedding dimension + for ( int e = 0; e < E; e++ ) { + + std::valarray< double > column = dataFrame.Column( col ); + + // Returns a copy of the valarray object with its elements + // shifted left n spaces (or right if n is negative). + std::valarray< double > tmp = column.shift( e * tau ); + + // Write shifted columns to the output embedding DataFrame + embedding.WriteColumn( colCount, tmp[ slice_i ] ); + + colCount++; + } + } + + return embedding; +} + +//---------------------------------------------------------------------- +// Simplex with path/file input +//---------------------------------------------------------------------- +DataFrame< double > Simplex( std::string pathIn, + std::string dataFile, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int E, + int Tp, + int knn, + int tau, + int exclusionRadius, + std::string colNames, + std::string targetName, + bool embedded, + bool const_predict, + bool verbose ) +{ + // DataFrame constructor loads data + DataFrame< double > DF( pathIn, dataFile ); + + // Pass data frame to Simplex + DataFrame< double > simplexProjection = Simplex( std::ref( DF ), + pathOut, + predictFile, + lib, + pred, + E, + Tp, + knn, + tau, + exclusionRadius, + colNames, + targetName, + embedded, + const_predict, + verbose ); + + return simplexProjection; +} + +//---------------------------------------------------------------------- +// Simplex with DataFrame input +//---------------------------------------------------------------------- +DataFrame Simplex( DataFrame< double > & DF, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int E, + int Tp, + int knn, + int tau, + int exclusionRadius, + std::string colNames, + std::string targetName, + bool embedded, + bool const_predict, + bool verbose ) +{ + // Instantiate Parameters + Parameters parameters = Parameters( Method::Simplex, "", "", + pathOut, predictFile, + lib, pred, E, Tp, knn, tau, 0, + exclusionRadius, + colNames, targetName, embedded, + const_predict, verbose ); + + // Instantiate EDM::SimplexClass object + SimplexClass SimplexModel = SimplexClass( DF, std::ref( parameters ) ); + + SimplexModel.Project(); + + return SimplexModel.projection; +} + +//---------------------------------------------------------------------------- +// 1) SMap with path/file input +// Default SVD (LAPACK) assigned in SMap() overload 2) +//---------------------------------------------------------------------------- +SMapValues SMap( std::string pathIn, + std::string dataFile, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int E, + int Tp, + int knn, + int tau, + double theta, + int exclusionRadius, + std::string columns, + std::string target, + std::string smapFile, + std::string derivatives, + bool embedded, + bool const_predict, + bool verbose ) +{ + // DataFrame constructor loads data + DataFrame< double > DF( pathIn, dataFile ); + + // Call overload 2) with DataFrame + SMapValues SMapOutput = SMap( std::ref( DF ), pathOut, predictFile, + lib, pred, E, Tp, knn, tau, theta, + exclusionRadius, + columns, target, smapFile, derivatives, + embedded, const_predict, verbose ); + return SMapOutput; +} + +//---------------------------------------------------------------------------- +// 2) SMap with DataFrame +// Default SVD (LAPACK) assigned in Smap.cc overload 2) +//---------------------------------------------------------------------------- +SMapValues SMap( DataFrame< double > & DF, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int E, + int Tp, + int knn, + int tau, + double theta, + int exclusionRadius, + std::string columns, + std::string target, + std::string smapFile, + std::string derivatives, + bool embedded, + bool const_predict, + bool verbose ) +{ + // Call overload 4) with default SVD function + SMapValues SMapOutput = SMap( DF, pathOut, predictFile, + lib, pred, E, Tp, knn, tau, theta, + exclusionRadius, + columns, target, smapFile, derivatives, + & SVD, // LAPACK SVD default + embedded, const_predict, verbose); + + return SMapOutput; +} + +//---------------------------------------------------------------------------- +// 3) Data path/file with external solver object +//---------------------------------------------------------------------------- +SMapValues SMap( std::string pathIn, + std::string dataFile, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int E, + int Tp, + int knn, + int tau, + double theta, + int exclusionRadius, + std::string columns, + std::string target, + std::string smapFile, + std::string derivatives, + std::valarray< double > (*solver)(DataFrame < double >, + std::valarray < double >), + bool embedded, + bool const_predict, + bool verbose ) +{ + // DataFrame constructor loads data + DataFrame< double > DF( pathIn, dataFile ); + + // Call overload 4) with DataFrame and solver object + SMapValues SMapOutput = SMap( std::ref( DF ), pathOut, predictFile, + lib, pred, E, Tp, knn, tau, theta, + exclusionRadius, + columns, target, smapFile, derivatives, + solver, embedded, const_predict, verbose ); + return SMapOutput; +} + +//---------------------------------------------------------------------------- +// 4) DataFrame with external solver object +//---------------------------------------------------------------------------- +SMapValues SMap( DataFrame< double > & DF, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int E, + int Tp, + int knn, + int tau, + double theta, + int exclusionRadius, + std::string columns, + std::string target, + std::string smapFile, + std::string derivatives, + std::valarray< double > (*solver)(DataFrame < double >, + std::valarray < double >), + bool embedded, + bool const_predict, + bool verbose ) +{ + if ( derivatives.size() ) {} // -Wunused-parameter + + Parameters parameters = Parameters( Method::SMap, "", "", + pathOut, predictFile, + lib, pred, E, Tp, knn, tau, theta, + exclusionRadius, + columns, target, embedded, + const_predict, verbose, + smapFile ); + + // Instantiate EDM::SMapClass object + SMapClass SMapModel = SMapClass( DF, std::ref( parameters ) ); + + SMapModel.Project( solver ); + + SMapValues values = SMapValues(); + values.predictions = SMapModel.projection; + values.coefficients = SMapModel.coefficients; + + return values; +} + +//---------------------------------------------------------------------- +// CCM with path/file input +//---------------------------------------------------------------------- +CCMValues CCM( std::string pathIn, + std::string dataFile, + std::string pathOut, + std::string predictFile, + int E, + int Tp, + int knn, + int tau, + std::string colNames, + std::string targetName, + std::string libSizes_str, + int sample, + bool random, + bool replacement, + unsigned seed, + bool includeData, + bool verbose ) +{ + // DataFrame constructor loads data + DataFrame< double > DF( pathIn, dataFile ); + + CCMValues ccmValues = CCM( std::ref( DF ), pathOut, predictFile, + E, Tp, knn, tau, colNames, targetName, + libSizes_str, sample, random, replacement, + seed, includeData, verbose ); + + return ccmValues; +} + +//---------------------------------------------------------------------- +// CCM with DataFrame input +//---------------------------------------------------------------------- +CCMValues CCM( DataFrame< double > & DF, + std::string pathOut, + std::string predictFile, + int E, + int Tp, + int knn, + int tau, + std::string colNames, + std::string targetName, + std::string libSizes_str, + int sample, + bool random, + bool replacement, + unsigned seed, + bool includeData, + bool verbose ) +{ + // Set library and prediction indices to entire library + std::stringstream ss; + ss << "1 " << DF.NRows(); + + Parameters parameters = Parameters( Method::CCM, + "", // pathIn + "", // dataFile + pathOut, // + predictFile, // + ss.str(), // lib_str + ss.str(), // pred_str + E, // + Tp, // + knn, // + tau, // + 0, // theta + 0, // exclusionRadius + colNames, // + targetName, // + false, // embedded + false, // const_predict + verbose, // + "", // SmapFile + "", // blockFile + 0, // multiviewEnsemble + 0, // multiviewD + false, // multiviewTrainLib + libSizes_str, // + sample, // + random, // + replacement, // + seed, // + includeData );// + + // Instantiate EDM::Simplex::CCM object + CCMClass CCMModel = CCMClass( DF, std::ref( parameters ) ); + + CCMModel.Project(); + + CCMValues values = CCMValues(); + values.AllLibStats = CCMModel.allLibStats; + values.CrossMap1 = CCMModel.colToTarget; + values.CrossMap2 = CCMModel.targetToCol; + + return values; +} + +//---------------------------------------------------------------------- +// Multiview with path/file input +//---------------------------------------------------------------------- +MultiviewValues Multiview( std::string pathIn, + std::string dataFile, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int D, + int E, + int Tp, + int knn, + int tau, + std::string columns, + std::string target, + int multiview, + int exclusionRadius, + bool trainLib, + bool verbose, + unsigned nThreads ) +{ + // DataFrame constructor loads data + DataFrame< double > DF( pathIn, dataFile ); + + MultiviewValues mvValues = Multiview( std::ref( DF ), pathOut, predictFile, + lib, pred, D, E, Tp, knn, tau, + columns, target, multiview, + exclusionRadius, trainLib, + verbose, nThreads); + + return mvValues; +} + +//---------------------------------------------------------------------- +// Multiview with DataFrame input +//---------------------------------------------------------------------- +MultiviewValues Multiview( DataFrame< double > & DF, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int D, + int E, + int Tp, + int knn, + int tau, + std::string columns, + std::string target, + int multiview, + int exclusionRadius, + bool trainLib, + bool verbose, + unsigned nThreads ) +{ + Parameters parameters = Parameters( Method::Simplex, + "", // pathIn + "", // dataFile + pathOut, // + predictFile, // + lib, // lib_str + pred , // pred_str + E, // + Tp, // + knn, // + tau, // + 0, // theta + exclusionRadius, + columns, // + target, // + true, // embedded true + false, // const_predict + verbose, // + "", // SmapFile + "", // blockFile + multiview, // multiviewEnsemble, + D, // multiviewD + trainLib ); // multiviewTrainLib + + // Instantiate EDM::Simplex::Multiview object + MultiviewClass MultiviewModel = MultiviewClass( DF, std::ref( parameters ) ); + + MultiviewModel.Project( nThreads ); + + return MultiviewModel.MVvalues; +} diff --git a/src/API.h b/src/API.h new file mode 100644 index 0000000..90365a9 --- /dev/null +++ b/src/API.h @@ -0,0 +1,327 @@ +#ifndef EDM_API_H +#define EDM_API_H + +#include "Common.h" // Non DataFrame return struct definitions +#include "Parameter.h" +#include "Simplex.h" +#include "SMap.h" +#include "CCM.h" +#include "Multiview.h" + +//------------------------------------------------------------- +// API function declarations. +// +// API functions generally have two call-signatures. +// The first takes a (path, file name) pair specifying the data +// file image on disk to be loaded and converted to a data frame. +// The second replaces these two arguments with a DataFrame object. +// +// NOTE: These are the first declarations seen by the compiler +// for the API and provide default argument values +//------------------------------------------------------------- + +DataFrame< double > Embed( std::string path = "", + std::string dataFile = "", + int E = 0, + int tau = 0, + std::string columns = "", + bool verbose = false ); + +DataFrame< double > Embed( DataFrame< double > & dataFrame, + int E = 0, + int tau = 0, + std::string columns = "", + bool verbose = false ); + +DataFrame< double > MakeBlock( DataFrame< double > & dataFrame, + int E, + int tau, + std::vector columnNames ); + +DataFrame< double > Simplex( std::string pathIn = "./data/", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int E = 0, + int Tp = 1, + int knn = 0, + int tau = -1, + int exclusionRadius = 0, + std::string colNames = "", + std::string targetName = "", + bool embedded = false, + bool const_predict = false, + bool verbose = true ); + +DataFrame< double > Simplex( DataFrame< double > & dataFrameIn, + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int E = 0, + int Tp = 1, + int knn = 0, + int tau = -1, + int exclusionRadius = 0, + std::string colNames = "", + std::string targetName = "", + bool embedded = false, + bool const_predict = false, + bool verbose = true ); + +// SMap is a special case since it can be called with a function pointer +// to the SVD solver. This is done so that interfaces such as pybind11 +// can provide their own object for the solver. +// 1) Data path/file with default SVD (LAPACK) assigned in Smap.cc 2) +SMapValues SMap( std::string pathIn = "./data/", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int E = 0, + int Tp = 1, + int knn = 0, + int tau = -1, + double theta = 0, + int exclusionRadius = 0, + std::string columns = "", + std::string target = "", + std::string smapFile = "", + std::string derivatives = "", + bool embedded = false, + bool const_predict = false, + bool verbose = true ); + +// 2) DataFrame with default SVD (LAPACK) assigned in Smap.cc 2) +SMapValues SMap( DataFrame< double > &dataFrameIn, + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int E = 0, + int Tp = 1, + int knn = 0, + int tau = -1, + double theta = 0, + int exclusionRadius = 0, + std::string columns = "", + std::string target = "", + std::string smapFile = "", + std::string derivatives = "", + bool embedded = false, + bool const_predict = false, + bool verbose = true ); + +// 3) Data path/file with external solver object, init to default SVD +SMapValues SMap( std::string pathIn = "./data/", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int E = 0, + int Tp = 1, + int knn = 0, + int tau = -1, + double theta = 0, + int exclusionRadius = 0, + std::string columns = "", + std::string target = "", + std::string smapFile = "", + std::string derivatives = "", + std::valarray< double > (*solver) + (DataFrame < double >, + std::valarray < double >) = & SVD, + bool embedded = false, + bool const_predict = false, + bool verbose = true ); + +// 4) DataFrame with external solver object, init to default SVD +SMapValues SMap( DataFrame< double > &dataFrameIn, + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int E = 0, + int Tp = 1, + int knn = 0, + int tau = -1, + double theta = 0, + int exclusionRadius = 0, + std::string columns = "", + std::string target = "", + std::string smapFile = "", + std::string derivatives = "", + std::valarray< double > (*solver) + (DataFrame < double >, + std::valarray < double >) = & SVD, + bool embedded = false, + bool const_predict = false, + bool verbose = true ); + +CCMValues CCM( std::string pathIn = "./data/", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictFile = "", + int E = 0, + int Tp = 0, + int knn = 0, + int tau = -1, + std::string colNames = "", + std::string targetName = "", + std::string libSizes_str = "", + int sample = 0, + bool random = true, + bool replacement = false, + unsigned seed = 0, // seed=0: use RNG + bool includeData = false, + bool verbose = true ); + +CCMValues CCM( DataFrame< double > & dataFrameIn, + std::string pathOut = "./", + std::string predictFile = "", + int E = 0, + int Tp = 0, + int knn = 0, + int tau = -1, + std::string colNames = "", + std::string targetName = "", + std::string libSizes_str = "", + int sample = 0, + bool random = true, + bool replacement = false, + unsigned seed = 0, // seed=0: use RNG + bool includeData = false, + bool verbose = true ); + +MultiviewValues Multiview( std::string pathIn = "./", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int D = 0, + int E = 1, + int Tp = 1, + int knn = 0, + int tau = -1, + std::string columns = "", + std::string target = "", + int multiview = 0, + int exclusionRadius = 0, + bool trainLib = true, + bool verbose = false, + unsigned nThreads = 4 ); + +MultiviewValues Multiview( DataFrame< double > & dataFrameIn, + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int D = 0, + int E = 1, + int Tp = 1, + int knn = 0, + int tau = -1, + std::string columns = "", + std::string target = "", + int multiview = 0, + int exclusionRadius = 0, + bool trainLib = true, + bool verbose = false, + unsigned nThreads = 4 ); + +DataFrame< double > EmbedDimension( std::string pathIn = "./data/", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int maxE = 10, + int Tp = 1, + int tau = -1, + std::string colNames = "", + std::string targetName = "", + bool embedded = false, + bool verbose = true, + unsigned nThreads = 4 ); + +DataFrame< double > EmbedDimension( DataFrame< double > & dataFrameIn, + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int maxE = 10, + int Tp = 1, + int tau = -1, + std::string colNames = "", + std::string targetName = "", + bool embedded = false, + bool verbose = true, + unsigned nThreads = 4 ); + +DataFrame< double > PredictInterval( std::string pathIn = "./data/", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int maxTp = 10, + int E = 0, + int tau = -1, + std::string colNames = "", + std::string targetName = "", + bool embedded = false, + bool verbose = true, + unsigned nThreads = 4 ); + +DataFrame< double > PredictInterval( DataFrame< double > & dataFrameIn, + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + int maxTp = 10, + int E = 0, + int tau = -1, + std::string colNames = "", + std::string targetName = "", + bool embedded = false, + bool verbose = true, + unsigned nThreads = 4 ); + +DataFrame< double > PredictNonlinear( std::string pathIn = "./data/", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + std::string theta = "", + int E = 0, + int Tp = 1, + int knn = 0, + int tau = -1, + std::string colNames = "", + std::string targetName = "", + bool embedded = false, + bool verbose = true, + unsigned nThreads = 4 ); + +DataFrame< double > PredictNonlinear( DataFrame< double > & dataFrameIn, + std::string pathOut = "./", + std::string predictFile = "", + std::string lib = "", + std::string pred = "", + std::string theta = "", + int E = 0, + int Tp = 1, + int knn = 0, + int tau = -1, + std::string colNames = "", + std::string targetName = "", + bool embedded = false, + bool verbose = true, + unsigned nThreads = 4 ); +#endif diff --git a/src/AuxFunc.cc b/src/AuxFunc.cc deleted file mode 100644 index 9b1a457..0000000 --- a/src/AuxFunc.cc +++ /dev/null @@ -1,496 +0,0 @@ - -#include "AuxFunc.h" -#include "DateTime.h" - -namespace EDM_AuxFunc { - std::mutex mtx; -} - -//--------------------------------------------------------------- -// Common code for Simplex and Smap: -// 1) Extract or Embed() data into dataBlock -// 2) Get target (library) vector -// 3) DeletePartialDataRows() -// 4) Adjust param.library and param.prediction indices -// 5) FindNeighbors() -// -// NOTE: time column is not returned in the embedding dataBlock. -// -// NOTE: If data is embedded by Embed(), the returned dataBlock -// has tau * (E-1) fewer rows than data. Since data is -// included in the returned DataEmbedNN struct, the first -// (or last) tau * (E-1) data rows are deleted to match -// dataBlock. The target vector is also reduced. -// -// NOTE: If rows are deleted, then the library and prediction -// vectors in Parameters are updated to reflect this. -//--------------------------------------------------------------- -DataEmbedNN EmbedNN( DataFrame *data, - Parameters ¶m, - bool checkDataRows ) -{ - DataFrame &dataIn = std::ref( *data ); - - if ( checkDataRows ) { - CheckDataRows( param, dataIn, "EmbedNN: Input data" ); - } - - //---------------------------------------------------------- - // Extract or embed dataIn into dataBlock - //---------------------------------------------------------- - DataFrame dataBlock; // Multivariate or embedded DataFrame - - if ( param.embedded ) { - // dataIn is a multivariable block, no embedding needed - // Select the specified columns into dataBlock - if ( param.columnNames.size() ) { - dataBlock = dataIn.DataFrameFromColumnNames( param.columnNames ); - } - else if ( param.columnIndex.size() ) { - dataBlock = dataIn.DataFrameFromColumnIndex( param.columnIndex ); - } - else { - throw std::runtime_error( "EmbedNN(): colNames and " - " colIndex are empty.\n" ); - } - } - else { - // embedded = false: Create the embedding dataBlock via Embed() - // dataBlock will have tau * (E-1) fewer rows than dataIn - dataBlock = Embed( dataIn, param.E, param.tau, - param.columns_str, param.verbose ); - } - - //---------------------------------------------------------- - // Get target (library) vector - //---------------------------------------------------------- - std::valarray targetIn; - if ( param.targetIndex ) { - targetIn = dataIn.Column( param.targetIndex ); - } - else if ( param.targetName.size() ) { - targetIn = dataIn.VectorColumnName( param.targetName ); - } - else { - // Default to first column - targetIn = dataIn.Column( 0 ); - } - - //------------------------------------------------------------ - // embedded = false: Embed() was called on dataIn - // Remove target, dataIn rows as needed - // Adjust param.library and param.prediction indices - //------------------------------------------------------------ - if ( not param.embedded ) { - - if ( param.E < 1 ) { - std::stringstream errMsg; - errMsg << "EmbedNN(): E = " << param.E << " is invalid.\n" ; - throw std::runtime_error( errMsg.str() ); - } - - size_t shift = abs( param.tau ) * ( param.E - 1 ); - - // Copy targetIn excluding partial data into targetEmbed - std::valarray targetEmbed( dataIn.NRows() - shift ); - - // Bogus cast to ( std::valarray ) for MSVC - // as it doesn't export its own slice_array applied to [] - if ( param.tau < 0 ) { - targetEmbed = ( std::valarray ) - targetIn[ std::slice( shift, targetIn.size() - shift, 1 ) ]; - } - else { - targetEmbed = ( std::valarray ) - targetIn[ std::slice( 0, targetIn.size() - shift, 1 ) ]; - } - - // Resize targetIn to ignore partial data rows - targetIn.resize( targetEmbed.size() ); - - // Copy target without partial data into resized targetIn - std::slice targetEmbed_i = std::slice( 0, targetEmbed.size(), 1 ); - targetIn[ targetEmbed_i ] = ( std::valarray ) - targetEmbed[ targetEmbed_i ]; - - // Delete dataIn top or bottom rows of partial data - if ( not dataIn.PartialDataRowsDeleted() ) { - // Not thread safe - std::lock_guard lck( EDM_AuxFunc::mtx ); - - dataIn.DeletePartialDataRows( shift, param.tau ); - } - - // Adjust param.library and param.prediction vectors of indices - if ( shift > 0 ) { - param.DeleteLibPred(); - } - - // Check boundaries again since rows were removed - if ( checkDataRows ) { - CheckDataRows( param, dataIn, "EmbedNN: Embedded data" ); - } - } - - //---------------------------------------------------------- - // Nearest neighbors - //---------------------------------------------------------- - Neighbors neighbors = FindNeighbors( dataBlock, param ); - - // Create struct to return the objects - DataEmbedNN dataEmbedNN = DataEmbedNN( &dataIn, dataBlock, - targetIn, neighbors ); - return dataEmbedNN; -} - -//---------------------------------------------------------- -// Common code for Simplex and Smap output generation -//---------------------------------------------------------- -DataFrame FormatOutput( Parameters param, - std::valarray predictions, - std::valarray const_predictions, - std::valarray variance, - std::valarray target_vec, - std::vector time, - std::string timeName ) -{ - //---------------------------------------------------- - // TimeOut vector with additional Tp points - //---------------------------------------------------- - size_t N_time = time.size(); - size_t N_row = predictions.size(); - size_t Tp_magnitude = abs( param.Tp ); - - std::vector timeOut( N_row + Tp_magnitude ); - - // Populate timeOut vector with strings for output - if ( N_time ) { - FillTimes( param, time, std::ref( timeOut ) ); - } - - //---------------------------------------------------- - // Observations: Insert data; add Tp nan at end/start - //---------------------------------------------------- - std::valarray observations( N_row + Tp_magnitude ); - - if ( param.Tp > -1 ) { // Positive Tp --------------------------- - std::slice pred_i = std::slice( param.prediction[0], N_row, 1 ); - - observations[ std::slice( 0, N_row, 1 ) ] = - ( std::valarray ) target_vec[ pred_i ]; - - for ( size_t i = N_row; i < N_row + param.Tp; i++ ) { - observations[ i ] = NAN; // assign nan at end - } - } - else { // Negative Tp ------------------------------------------- - std::slice pred_i; - - if ( param.prediction[0] >= Tp_magnitude ) { - pred_i = std::slice( param.prediction[ 0 ] - Tp_magnitude, - N_row + Tp_magnitude, 1 ); - - observations[ std::slice( 0, N_row + Tp_magnitude, 1 ) ] = - ( std::valarray ) target_vec[ pred_i ]; - } - else { - // Edge case where -Tp preceeds available record pred - pred_i = std::slice( 0, N_row + Tp_magnitude, 1 ); - - observations[std::slice( Tp_magnitude, N_row, 1 )] = - ( std::valarray ) target_vec[ pred_i ]; - - for ( size_t i = 0; i < Tp_magnitude; i++ ) { - observations[ i ] = NAN; // assign nan at start - } - } - } - - //------------------------------------------------------------------ - // Predictions & variance: Assign values; insert Tp nan at start/end - //------------------------------------------------------------------ - std::valarray predictionsOut ( N_row + Tp_magnitude ); - std::valarray constPredictionsOut( N_row + Tp_magnitude ); - std::valarray varianceOut ( N_row + Tp_magnitude ); - - if ( param.Tp > -1 ) { // Positive Tp --------------------------- - std::slice predOut_i = std::slice( param.Tp, N_row, 1 ); - - for ( size_t i = 0; i < param.Tp; i++ ) { - predictionsOut[ i ] = NAN; // assign nan at start - varianceOut [ i ] = NAN; // assign nan at start - } - predictionsOut[ predOut_i ] = predictions; - varianceOut [ predOut_i ] = variance; - - if ( param.const_predict ) { - for ( size_t i = 0; i < param.Tp; i++ ) { - constPredictionsOut[ i ] = NAN; // assign nan at start - } - constPredictionsOut[ predOut_i ] = const_predictions; - } - } - else { // Negative Tp -------------------------------------------- - std::slice predOut_i = std::slice( 0, N_row - Tp_magnitude, 1 ); - std::slice predIn_i = std::slice( 0, N_row, 1 ); - - predictionsOut[ predOut_i ] = predictions[ predIn_i ]; - varianceOut [ predOut_i ] = variance [ predIn_i ]; - - for ( size_t i = N_row; i < N_row + Tp_magnitude; i++ ) { - predictionsOut[ i ] = NAN; // assign nan at end - varianceOut [ i ] = NAN; // assign nan at end - } - - if ( param.const_predict ) { - constPredictionsOut[ predOut_i ] = const_predictions[ predIn_i ]; - - for ( size_t i = N_row; i < N_row + Tp_magnitude; i++ ) { - constPredictionsOut[ i ] = NAN; // assign nan at end - } - } - } - - //---------------------------------------------------- - // Create output DataFrame - //---------------------------------------------------- - size_t dataFrameColumms = param.const_predict ? 4 : 3; - - DataFrame dataFrame( N_row + Tp_magnitude, dataFrameColumms ); - - if ( param.const_predict ) { - dataFrame.ColumnNames() = { "Observations", "Predictions", - "Pred_Variance", "Const_Predictions" }; - } - else { - dataFrame.ColumnNames() = {"Observations","Predictions","Pred_Variance"}; - } - - if ( N_time ) { - dataFrame.TimeName() = timeName; - dataFrame.Time() = timeOut; - } - - dataFrame.WriteColumn( 0, observations ); - dataFrame.WriteColumn( 1, predictionsOut ); - dataFrame.WriteColumn( 2, varianceOut ); - - if ( param.const_predict ) { - dataFrame.WriteColumn( 3, constPredictionsOut ); - } - -#ifdef DEBUG_ALL - std::cout << "FormatOutput() time " << timeOut.size() - << " pred " << predictionsOut.size() - << " obs " << observations.size() << std::endl; - std::cout << "FormatOutput() dataFrame -------------------" << std::endl; - std::cout << dataFrame; -#endif - - return dataFrame; -} - -//---------------------------------------------------------- -// Copy strings of time values into timeOut. -// If prediction times exceed times from the data, -// create new entries for the additional times. -//---------------------------------------------------------- -void FillTimes( Parameters param, - std::vector time, - std::vector &timeOut ) -{ - size_t N_time = time.size(); - size_t N_row = param.prediction.size(); - size_t max_pred_i = param.prediction[ N_row - 1 ]; - size_t min_pred_i = param.prediction[ 0 ]; - size_t Tp_magnitude = abs( param.Tp ); - - if ( max_pred_i >= N_time ) { - // If tau > 0 end rows were deleted. max_pred_i might exceed time bounds - max_pred_i = N_time - 1; - } - - if ( timeOut.size() != N_row + Tp_magnitude ) { - std::stringstream errMsg; - errMsg << "FillTimes(): timeOut vector length " << timeOut.size() - << " is not equal to the number of predictions + Tp " - << N_row + Tp_magnitude << std::endl; - throw std::runtime_error( errMsg.str() ); - } - - // Positive Tp ----------------------------------------------------- - if ( param.Tp > -1 ) { - // Fill in times guaranteed to be in param.prediction indices - for ( size_t i = 0; i < N_row; i++ ) { - size_t pred_i = param.prediction[ i ]; - if ( pred_i < N_time ) { - timeOut[ i ] = time[ pred_i ]; - } - } - - // Now fill in times beyond param.prediction indices - if ( max_pred_i + param.Tp < N_time ) { - // All prediction times are available in time, get the rest - for ( size_t i = 0; i < param.Tp; i++ ) { - timeOut[ N_row + i ] = time[ max_pred_i + i + 1 ]; - } - } - else { - // Tp introduces time values beyond the range of time - bool time_format_warning_printed = false; - - // Try to parse the last time vector string as a date or datetime - // if dtinfo.unrecognized_fmt = true; it is not a date or datetime - datetime_info dtinfo = parse_datetime( time[ max_pred_i ] ); - - for ( size_t i = 0; i < param.Tp; i++ ) { - std::stringstream tss; - - if ( dtinfo.unrecognized_fmt ) { - // Numeric so add Tp - tss << std::stod( time[ max_pred_i ] ) + i + 1; - } - else { - int time_delta = i + 1; - // Last two datetimes to compute time diff to add time delta - std::string time_new( time[ max_pred_i ] ); - std::string time_old( time[ max_pred_i - 1 ] ); - std::string new_time = - increment_datetime_str( time_old, time_new, time_delta ); - - // Add +ti if not recognized format(datetime util returns "") - if ( new_time.size() ) { - tss << new_time; - } - else { - tss << time[ max_pred_i ] << " +" << i + 1; - - if ( not time_format_warning_printed ) { - std::cout << "FillTimes(): " - << "time column unrecognized time format." - << "\n\tManually adding + tp to the last" - << " time column available." << std::endl; - time_format_warning_printed = true; - } - } - } - - timeOut[ N_row + i ] = tss.str(); - } - } - } - // Negative Tp ----------------------------------------------------- - else { - // Fill in times guaranteed to be in param.prediction indices - for ( size_t i = 0; i < N_row; i++ ) { - size_t pred_i = param.prediction[ i ]; - if ( pred_i < N_time ) { - // param.Tp is negative, start at timeOut[ 0 - param.Tp ] - // timeOut is shifted forward to accomodate the preceeding Tp - timeOut[ i + Tp_magnitude ] = time[ pred_i ]; - } - } - - // Now fill in times before param.prediction indices - if ( (int) min_pred_i + param.Tp >= 0 ) { - // All prediction times are available in time, get the rest - for ( size_t i = 0; i < Tp_magnitude; i++ ) { - timeOut[ i ] = time[ param.prediction[ i ] - Tp_magnitude ]; - } - } - else { - // Tp introduces time values before the range of time - bool time_format_warning_printed = false; - - // Try to parse the first time vector string as a date or datetime - // if dtinfo.unrecognized_fmt = true; it is not a date or datetime - datetime_info dtinfo = parse_datetime( time[ 0 ] ); - - for ( size_t i = 0; i < Tp_magnitude; i++ ) { - std::stringstream tss; - - if ( dtinfo.unrecognized_fmt ) { - // Numeric so subtract i Tp - tss << std::stod( time[ Tp_magnitude - 1 ] ) - (i + 1); - } - else { - int time_delta = i - 1; - // Get first two datetimes to compute time diff - // to add time delta - std::string time_new( time[ 1 ] ); - std::string time_old( time[ 0 ] ); - std::string new_time = - increment_datetime_str( time_old, time_new, time_delta ); - - // Subtract +ti if not a recognized format - // (datetime util returns "") - if ( new_time.size() ) { - tss << new_time; - } - else { - tss << time[ max_pred_i ] << " -" << i + 1; - - if ( not time_format_warning_printed ) { - std::cout << "FillTimes(): " - << "time column unrecognized time format." - << "\n\tManually adding - tp to the first" - << " time column available." << std::endl; - time_format_warning_printed = true; - } - } - } // else not dtinfo.unrecognized_fmt - - timeOut[ i ] = tss.str(); - - } // for ( size_t i = 0; i < Tp_magnitude; i++ ) - } // else Tp introduces time values before the range of time - } // else Negative Tp ------------------------------------------------ -} - -//---------------------------------------------------------- -// Validate dataFrameIn rows against lib and pred indices -//---------------------------------------------------------- -void CheckDataRows( Parameters param, - DataFrame &dataFrameIn, - std::string call ) -{ - // param.prediction & library have been zero-offset in Validate() - // to convert from user specified data row to array indicies - size_t prediction_max_i = param.prediction[ param.prediction.size() - 1 ]; - size_t library_max_i = param.library [ param.library.size() - 1 ]; - - size_t shift; - if ( param.embedded ) { - shift = 0; - } - else { - if ( param.E < 1 ) { - std::stringstream errMsg; - errMsg << "CheckDataRows(): E = " << param.E << " is invalid.\n" ; - throw std::runtime_error( errMsg.str() ); - } - - shift = abs( param.tau ) * ( param.E - 1 ); - } - - if ( dataFrameIn.NRows() <= prediction_max_i ) { - std::stringstream errMsg; - errMsg << "CheckDataRows(): " << call - << ": The prediction index " - << prediction_max_i + 1 - << " exceeds the number of data rows " - << dataFrameIn.NRows(); - throw std::runtime_error( errMsg.str() ); - } - - if ( dataFrameIn.NRows() <= library_max_i + shift ) { - std::stringstream errMsg; - errMsg << "CheckDataRows(): " << call - << ": The library index " << library_max_i + 1 - << " + tau(E-1) " << shift << " = " - << library_max_i + 1 + shift - << " exceeds the number of data rows " - << dataFrameIn.NRows(); - throw std::runtime_error( errMsg.str() ); - } -} diff --git a/src/AuxFunc.h b/src/AuxFunc.h deleted file mode 100644 index 63c7ad6..0000000 --- a/src/AuxFunc.h +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef AUXFUNC -#define AUXFUNC - -#include -#include // std::ref - -#include "Common.h" -#include "Neighbors.h" -#include "Embed.h" - -//---------------------------------------------------------------- -// Data Input, embedding and NN structure to accommodate -// common initial processing in Simplex and Smap -//---------------------------------------------------------------- -struct DataEmbedNN { - DataFrame *dataIn; - DataFrame dataFrame; - std::valarray targetVec; - Neighbors neighbors; - - // Constructors - DataEmbedNN() {} - - DataEmbedNN( DataFrame *dataIn, - DataFrame dataFrame, - std::valarray targetVec, - Neighbors neighbors ) : - dataIn( dataIn ), dataFrame( dataFrame ), targetVec( targetVec ), - neighbors( neighbors ) {} -}; - -// Prototypes -DataEmbedNN EmbedNN( DataFrame *dataIn, - Parameters ¶m, - bool checkDataRows = true ); - -DataFrame FormatOutput( Parameters param, - std::valarray predictions, - std::valarray const_predictions, - std::valarray variance, - std::valarray target_vec, - std::vector time, - std::string timeName ); - -void FillTimes( Parameters param, - std::vector time, - std::vector &timeOut ); - -void CheckDataRows( Parameters param, - DataFrame &dataFrameIn, - std::string call ); -#endif diff --git a/src/CCM.cc b/src/CCM.cc index 4312ace..3527c55 100644 --- a/src/CCM.cc +++ b/src/CCM.cc @@ -1,386 +1,121 @@ -#include -#include -#include -#include -#include - -#ifdef CCM_THREADED // Defined in makefile -// Two explicit CrossMap() threads are invoked. -// One for forward mapping, one for inverse mapping. -#include -#endif - -#include "Common.h" -#include "Embed.h" -#include "AuxFunc.h" +#include "CCM.h" -namespace EDM_CCM { +namespace EDM_CCM_Lock { std::mutex mtx; std::mutex q_mtx; std::queue< std::exception_ptr > exceptionQ; - // Define the initial maximum distance for neigbors to avoid sort() - // DBL_MAX is a Macro equivalent to: std::numeric_limits::max() - // The issue with std::sort is that it ignores ties... - double DistanceMax = std::numeric_limits::max(); - double DistanceLimit = std::numeric_limits::max() / ( 1 + 1E-9 ); } //---------------------------------------------------------------- -// forward declarations +// forward declaration //---------------------------------------------------------------- -void CrossMap( Parameters param, - DataFrame< double > dataFrameIn, - bool includeData, - const CrossMapValues &crossMapValues ); - -DataFrame< double > CCMDistances( const DataFrame< double > &dataBlock, - Parameters param ); - -Neighbors CCMNeighbors( const DataFrame< double > &Distances, - std::vector< size_t > lib_i, - Parameters param ); - -DataFrame SimplexProjection( Parameters param, - DataEmbedNN embedNN, - bool checkDataRows ); +void CrossMap( SimplexClass & S ); //---------------------------------------------------------------- -// API Overload 1: Explicit data file path/name -// Implemented as a wrapper to API Overload 2: -// which is a wrapper for CrossMap() +// Constructor +// data & parameters initialise EDM::SimplexClass parent, and, +// both mapping objects to the same initial parameters. //---------------------------------------------------------------- -CCMValues CCM( std::string pathIn, - std::string dataFile, - std::string pathOut, - std::string predictFile, - int E, - int Tp, - int knn, - int tau, - std::string columns, - std::string target, - std::string libSizes_str, - int sample, - bool random, - bool replacement, - unsigned seed, - bool includeData, - bool verbose ) -{ - //---------------------------------------------------------- - // Load data to dataFrameIn - //---------------------------------------------------------- - DataFrame< double > dataFrameIn( pathIn, dataFile ); - - CCMValues ccmValues = CCM( dataFrameIn, - pathOut, - predictFile, - E, - Tp, - knn, - tau, - columns, - target, - libSizes_str, - sample, - random, - replacement, - seed, - includeData, - verbose ); - return ccmValues; -} +CCMClass::CCMClass ( + DataFrame< double > & data, + Parameters & parameters ) : + SimplexClass ( data, parameters ), // base class initialise + colToTargetCCM( data, parameters ), // forward mapping object + targetToColCCM( data, parameters ) // reverse mapping object +{} //---------------------------------------------------------------- -// API Overload 2: DataFrame passed in -// Implemented a wrapper for CrossMap() +// Project : Polymorphic implementation //---------------------------------------------------------------- -CCMValues CCM( DataFrame< double > dataFrameIn, - std::string pathOut, - std::string predictFile, - int E, - int Tp, - int knn, - int tau, - std::string columns, - std::string target, - std::string libSizes_str, - int sample, - bool random, - bool replacement, - unsigned seed, - bool includeData, - bool verbose ) -{ - if ( not columns.size() ) { - throw std::runtime_error("CCM() must specify the column to embed."); - } - if ( not target.size() ) { - throw std::runtime_error("CCM() must specify the target."); - } - if ( not libSizes_str.size() ) { - throw std::runtime_error("CCM() must specify library sizes."); - } +void CCMClass::Project () { - Parameters param = Parameters( Method::CCM, - "", // pathIn - "", // dataFile - pathOut, // - predictFile, // - "", // lib_str - "", // pred_str - E, // - Tp, // - knn, // - tau, // - 0, // theta - 0, // exclusionRadius - columns, // - target, // - false, // embedded - false, // const_predict - verbose, // - "", // SmapFile - "", // blockFile - "", // derivatives_str - 0, // svdSig - 0, // tikhonov - 0, // elasticNet - 0, // multi - libSizes_str, // - sample, // - random, // - replacement, // - seed ); // - - if ( param.columnNames.size() > 1 ) { - std::cout << "WARNING: CCM() Only the first column will be mapped.\n"; - } + SetupParameters(); // Forward and reverse mapping objects - // Create Parameters object that switches column[0] and target - // for the inverse mapping - Parameters inverseParam( param ); // copy constructor - std::string newTarget( param.columns_str ); - inverseParam.columns_str = param.target_str; - inverseParam.target_str = newTarget; - - // Validate converts column_str & target_str to columnNames, targetName - inverseParam.Validate(); - -#ifdef DEBUG_ALL - std::cout << "CCM() params:\n"; - std::cout << param; - std::cout << "CCM() inverseParams:\n"; - std::cout << inverseParam; -#endif + CCM(); // PrepareEmbedding, Distances, FindNeighbors, Simplex - //------------------------------------------------------------ - // Setup DataFrames for output CrossMapValues structs - //------------------------------------------------------------ - DataFrame LibStats1( param.librarySizes.size(), 4, - "LibSize rho RMSE MAE" ); - DataFrame LibStats2( param.librarySizes.size(), 4, - "LibSize rho RMSE MAE" ); - size_t maxSamples; - if ( param.randomLib ) { - // Random samples from library - maxSamples = param.librarySizes.size() * param.subSamples; - } - else { - // Contiguous samples up to the size of the library - maxSamples = param.librarySizes.size(); - } - - DataFrame PredictionStats1( maxSamples, 8, - "N E nn tau LibSize rho RMSE MAE" ); - DataFrame PredictionStats2( maxSamples, 8, - "N E nn tau LibSize rho RMSE MAE" ); + FormatOutput(); - // Instantiate CrossMapValues output structs and insert DataFrames - CrossMapValues col_to_target = CrossMapValues(); - CrossMapValues target_to_col = CrossMapValues(); - - col_to_target.LibStats = LibStats1; - target_to_col.LibStats = LibStats2; + WriteOutput(); +} - if ( includeData ) { - col_to_target.PredictStats = PredictionStats1; - target_to_col.PredictStats = PredictionStats2; +//---------------------------------------------------------------- +// CCM +// To accomodate two threads running forward & inverse mapping +// CrossMap() is called with separate EDM::Simplex objects: +// SimplexClass colToTargetCCM; column to target mapping +// SimplexClass targetToColCCM; target to column mapping +// These fill the respective EDM object CrossMapValues structs: +// CrossMapValues colToTarget; +// CrossMapValues targetToCol; +//---------------------------------------------------------------- +void CCMClass::CCM () { + + if ( parameters.columnNames.size() > 1 ) { + std::cout << "WARNING: CCM() Only the first column will be mapped.\n"; } - + #ifdef CCM_THREADED - std::thread CrossMapColTarget( CrossMap, param, dataFrameIn, includeData, - std::ref( col_to_target ) ); + std::thread CrossMapColTarget( CrossMap, std::ref( colToTargetCCM ) ); + std::thread CrossMapTargetCol( CrossMap, std::ref( targetToColCCM ) ); - std::thread CrossMapTargetCol( CrossMap, inverseParam, dataFrameIn, - includeData, std::ref( target_to_col ) ); CrossMapColTarget.join(); CrossMapTargetCol.join(); // If thread threw exception, get from queue and rethrow - if ( not EDM_CCM::exceptionQ.empty() ) { - std::lock_guard lck( EDM_CCM::q_mtx ); + if ( not EDM_CCM_Lock::exceptionQ.empty() ) { + std::lock_guard lck( EDM_CCM_Lock::q_mtx ); // Take the first exception in the queue - std::exception_ptr exceptionPtr = EDM_CCM::exceptionQ.front(); + std::exception_ptr exceptionPtr = EDM_CCM_Lock::exceptionQ.front(); // Unroll all other exception from the thread/loops - while( not EDM_CCM::exceptionQ.empty() ) { - // JP When do these exception_ptr get deleted? Is it a leak? - EDM_CCM::exceptionQ.pop(); + while( not EDM_CCM_Lock::exceptionQ.empty() ) { + EDM_CCM_Lock::exceptionQ.pop(); } std::rethrow_exception( exceptionPtr ); } #else - CrossMap( param, dataFrameIn, includeData, std::ref( col_to_target)); - CrossMap( inverseParam, dataFrameIn, includeData, std::ref( target_to_col)); + CrossMap( std::ref( colToTargetCCM ) ); + CrossMap( std::ref( targetToColCCM ) ); #endif - - //----------------------------------------------------------------- - // Output - //----------------------------------------------------------------- - // Create unified column names of output DataFrame - std::stringstream libRhoNames; - libRhoNames << "LibSize " - << param.columnNames[0] << ":" << param.targetName << " " - << param.targetName << ":" << param.columnNames[0]; - - // Unified LibStats output DataFrame - DataFrame PredictLibRho( param.librarySizes.size(), 3, - libRhoNames.str() ); - - PredictLibRho.WriteColumn( 0, col_to_target.LibStats.Column( 0 ) ); - PredictLibRho.WriteColumn( 1, col_to_target.LibStats.Column( 1 ) ); - PredictLibRho.WriteColumn( 2, target_to_col.LibStats.Column( 1 ) ); - - if ( param.predictOutputFile.size() ) { - // Write to disk - PredictLibRho.WriteData( param.pathOut, param.predictOutputFile ); - } - - // Output struct - CCMValues ccmValues; - ccmValues.AllLibStats = PredictLibRho; - - if ( includeData ) { - // Now handle data for each prediction instance - ccmValues.CrossMap1.PredictStats = col_to_target.PredictStats; - ccmValues.CrossMap2.PredictStats = target_to_col.PredictStats; - ccmValues.CrossMap1.Predictions = col_to_target.Predictions; - ccmValues.CrossMap2.Predictions = target_to_col.Predictions; - } - - return ccmValues; } //---------------------------------------------------------------- // CrossMap() -// Worker function for CCM. -// Return DataFrame of rho, RMSE, MAE values for param.librarySizes -// -// NOTE: This is a bit of a kludge at the moment... would be nice -// to pass in a reference to dataFrameIn, however... Embed() -// returns a dataBlock with (E-1)*tau fewer rows since -// partial data vectors are removed. To maintain proper -// alignment with the target in the dataFrameIn, -// dataFrameIn.DeletePartialDataRows() is called. If we use -// a reference to dataFrameIn, then the second thread will -// receive a reference to dataFrameIn that has rows deleted -// and Embed() will then not be correct. So at the moment -// we'll pass a copy of the dataFrameIn to each thread. +// Thread worker function for CCM. //---------------------------------------------------------------- -void CrossMap( Parameters paramCCM, - DataFrame< double > dataFrameIn, - bool includeData, - const CrossMapValues &crossMapValuesIn ) { - - // Get a local reference for CrossMapValues - CrossMapValues &crossMapValues = - const_cast< CrossMapValues & >( crossMapValuesIn ); - - if ( paramCCM.verbose ) { - std::lock_guard lck( EDM_CCM::mtx ); +void CrossMap( SimplexClass & S ) { + + if ( S.parameters.verbose ) { + std::lock_guard lck( EDM_CCM_Lock::mtx ); std::stringstream msg; msg << "CrossMap(): Simplex cross mapping from " - << paramCCM.columnNames[0] - << " to " << paramCCM.targetName << " E=" << paramCCM.E - << " knn=" << paramCCM.knn << " Library range: [" - << paramCCM.libSizes_str << "] "; - for ( size_t i = 0; i < paramCCM.librarySizes.size(); i++ ) { - msg << paramCCM.librarySizes[ i ] << " "; + << S.parameters.columnNames[0] + << " to " << S.parameters.targetName << " E=" << S.parameters.E + << " knn=" << S.parameters.knn << " Library range: [" + << S.parameters.libSizes_str << "] "; + for ( size_t i = 0; i < S.parameters.librarySizes.size(); i++ ) { + msg << S.parameters.librarySizes[ i ] << " "; } msg << std::endl << std::endl; std::cout << msg.str(); } try { - //------------------------------------------------------------ - // Generate embedding on data to be cross mapped (-c column) - // dataBlock will have tau * (E-1) fewer rows than dataFrameIn - // JP: Should this be allocated on the heap? - //------------------------------------------------------------ - DataFrame dataBlock = Embed( dataFrameIn, - paramCCM.E, - paramCCM.tau, - paramCCM.columnNames[0], - paramCCM.verbose ); - - size_t N_row = dataBlock.NRows(); - - // NOTE: No need to adjust param.library and param.prediction indices - // with call to param.DeleteLibPred(); since pred will - // be created below based on N_row of dataBlock. - - //-------------------------------------------------------------- - // Remove dataFrameIn rows to match embedded dataBlock with - // partial data rows ignored: CrossMap() -> Embed() -> MakeBlock() - // This removal of partial data rows is also done in EmbedNN() - //-------------------------------------------------------------- - if ( paramCCM.E < 1 ) { + if ( S.parameters.E < 1 ) { + std::lock_guard lck( EDM_CCM_Lock::mtx ); std::stringstream errMsg; - errMsg << "CrossMap(): E = " << paramCCM.E << " is invalid.\n" ; + errMsg << "CrossMap(): E = " << S.parameters.E << " is invalid.\n" ; throw std::runtime_error( errMsg.str() ); } - - size_t shift = abs( paramCCM.tau ) * ( paramCCM.E - 1 ); - { - std::lock_guard lck( EDM_CCM::mtx ); - if ( not dataFrameIn.PartialDataRowsDeleted() ) { - // Not thread safe. - dataFrameIn.DeletePartialDataRows( shift, paramCCM.tau ); - } - } - -#ifdef DEBUG_ALL - { - std::lock_guard lck( EDM_CCM::mtx ); - std::cout << ">>>> CrossMap() dataFrameIn-----------------------\n"; - std::cout << dataFrameIn; - std::cout << "<<<< dataFrameIn----------------------------------\n"; - std::cout << ">>>> dataBlock------------------------------------\n"; - std::cout << dataBlock; - std::cout << "<<<< dataBlock------------------------------------\n"; - } -#endif - //----------------------------------------------------------------- - // Create Parameters for SimplexProjection - // Add library and prediction indices for the entire library - //----------------------------------------------------------------- - std::stringstream ss; - ss << "1 " << N_row; - paramCCM.lib_str = ss.str(); - paramCCM.pred_str = ss.str(); - // Validate converts lib_str, pred_str to library & prediction vectors - paramCCM.Validate(); - //----------------------------------------------------------------- // Set number of samples //----------------------------------------------------------------- size_t maxSamples; - if ( paramCCM.randomLib ) { + if ( S.parameters.randomLib ) { // Random samples from library - maxSamples = paramCCM.subSamples; + maxSamples = S.parameters.subSamples; } else { // Contiguous samples up to the size of the library @@ -388,137 +123,120 @@ void CrossMap( Parameters paramCCM, } //----------------------------------------------------------------- - // Create random number generator: DefaultRandEngine + // Create random number generator: DefaultRandomEngine //----------------------------------------------------------------- - if ( paramCCM.randomLib ) { - if ( paramCCM.seed == 0 ) { + if ( S.parameters.randomLib ) { + if ( S.parameters.seed == 0 ) { // Select a random seed typedef std::chrono::high_resolution_clock CCMclock; CCMclock::time_point beginning = CCMclock::now(); CCMclock::duration duration = CCMclock::now() - beginning; - paramCCM.seed = duration.count(); + S.parameters.seed = duration.count(); } } - std::default_random_engine DefaultRandomEngine( paramCCM.seed ); - - //----------------------------------------------------------------- - // Distance for all possible pred : lib E-dimensional vector pairs - // Distances is a square Matrix of all row to to row distances - //----------------------------------------------------------------- - DataFrame< double > Distances = CCMDistances( std::ref( dataBlock ), - paramCCM ); - -#ifdef DEBUG_ALL - { - std::lock_guard lck( EDM_CCM::mtx ); - std::cout << "CrossMap() " << paramCCM.columnNames[0] << " to " - << paramCCM.targetName << " Distances: " << Distances.NRows() - << " x " << Distances.NColumns() << std::endl; - } -#endif - + std::default_random_engine DefaultRandomEngine( S.parameters.seed ); + //---------------------------------------------------------- // Predictions //---------------------------------------------------------- size_t predictionCount = 0; + size_t N_row = S.embedding.NRows(); + + //---------------------------------------------------------- // Loop for library sizes - for ( size_t lib_size_i = 0; - lib_size_i < paramCCM.librarySizes.size(); lib_size_i++ ) { + //---------------------------------------------------------- + for ( size_t libSize_i = 0; + libSize_i < S.parameters.librarySizes.size(); libSize_i++ ) { + + size_t libSize = S.parameters.librarySizes[ libSize_i ]; - size_t lib_size = paramCCM.librarySizes[ lib_size_i ]; + // Create random RNG sampler for this libSize out of N_row + std::uniform_int_distribution< size_t > distribution( 0, N_row - 1 ); - // Create random RNG sampler for this lib_size - std::uniform_int_distribution distribution( 0, N_row - 1 ); - #ifdef DEBUG_ALL { - std::lock_guard lck( EDM_CCM::mtx ); - std::cout << "lib_size: " << lib_size + std::lock_guard lck( EDM_CCM_Lock::mtx ); + std::cout << "libSize: " << libSize << " ------------------------------------------\n"; } #endif - + // Output statistics vectors std::valarray< double > rho ( maxSamples ); std::valarray< double > RMSE( maxSamples ); std::valarray< double > MAE ( maxSamples ); + //---------------------------------------------------------- // Loop for subsamples + //---------------------------------------------------------- for ( size_t n = 0; n < maxSamples; n++ ) { - - // Vector of row indices to include in this lib_size evaluation - std::vector< size_t > lib_i( lib_size ); - if ( paramCCM.randomLib ) { + //------------------------------------------------------ + // Generate library row indices for this subsample + //------------------------------------------------------ + std::vector< size_t > lib_i( libSize ); + + if ( S.parameters.randomLib ) { // Uniform random sample of rows - if ( paramCCM.replacement ) { + if ( S.parameters.replacement ) { // With replacement - for ( size_t i = 0; i < lib_size; i++ ) { + for ( size_t i = 0; i < libSize; i++ ) { lib_i[ i ] = distribution( DefaultRandomEngine ); } } else { - // Without replacement lib_size elements from [0, N_row-1] - // Robert W. Floyd's algorithm + // Without replacement libSize elements from [0, N_row-1] // NOTE: c++17 has the sample() function in - if ( lib_size >= N_row ) { + if ( libSize >= N_row ) { std::stringstream errMsg; - errMsg << "CrossMap(): lib_size=" << lib_size + errMsg << "CrossMap(): libSize=" << libSize << " must be less than N_row=" << N_row << " for random sample without replacement."; throw std::runtime_error( errMsg.str() ); } - + // unordered set to store samples - std::unordered_set samples; - - // Sample and insert values into samples - size_t r = 0; - while( samples.size() < lib_size ) { + std::unordered_set< size_t > samples; + + // Sample and insert unique values into samples + while( samples.size() < libSize ) { size_t v = distribution( DefaultRandomEngine ); - if ( not samples.insert( v ).second ) { - samples.insert( r ); - r++; - } + samples.insert( v ); } - + // Copy samples into result - std::vector result(samples.begin(), samples.end()); + std::vector result( samples.begin(), samples.end() ); - // Shuffle result - std::shuffle( result.begin(), result.end(), - DefaultRandomEngine ); - lib_i = result; // Copy result to lib_i } } else { // Not random samples, contiguous samples increasing size - if ( lib_size >= N_row ) { + if ( libSize >= N_row ) { // library size exceeded, back down lib_i.resize( N_row ); std::iota( lib_i.begin(), lib_i.end(), 0 ); - lib_size = N_row; + libSize = N_row; - if ( paramCCM.verbose ) { + if ( S.parameters.verbose ) { std::stringstream msg; msg << "CCM(): Sequential library samples," - << " max lib_size is " << N_row - << ", lib_size has been limited.\n"; + << " max libSize is " << N_row + << ", libSize has been limited.\n"; std::cout << msg.str(); } } else { // Contiguous blocks up to N_rows = maxSamples - if ( n + lib_size < N_row ) { + if ( n + libSize < N_row ) { std::iota( lib_i.begin(), lib_i.end(), n ); } else { - // n + lib_size > N_row, wrap around to data origin + // n + libSize > N_row, wrap around to data origin std::vector< size_t > lib_start( N_row - n ); std::iota( lib_start.begin(), lib_start.end(), n ); - size_t max_i = std::min( lib_size-(N_row - n), N_row ); + size_t max_i = std::min( libSize-(N_row - n), N_row ); std::vector< size_t > lib_wrap( max_i ); std::iota( lib_wrap.begin(), lib_wrap.end(), 0 ); @@ -527,14 +245,14 @@ void CrossMap( Parameters paramCCM, lib_i.insert( lib_i.end(), lib_wrap.begin(), lib_wrap.end() ); - lib_size = lib_i.size(); + libSize = lib_i.size(); } } } - + #ifdef DEBUG_ALL { - std::lock_guard lck( EDM_CCM::mtx ); + std::lock_guard lck( EDM_CCM_Lock::mtx ); std::cout << "lib_i: (" << lib_i.size() << ") "; for ( size_t i = 0; i < lib_i.size(); i++ ) { std::cout << lib_i[i] << " "; @@ -543,266 +261,204 @@ void CrossMap( Parameters paramCCM, #endif //---------------------------------------------------------- - // Nearest neighbors : Local CCMNeighbors() function + // Set library and predict indices to lib_i //---------------------------------------------------------- - Neighbors neighbors = CCMNeighbors( Distances, lib_i, paramCCM ); + S.parameters.library.resize( lib_i.size() ); + std::iota( S.parameters.library.begin(), + S.parameters.library.end(), 0 ); + S.parameters.prediction.resize( lib_i.size() ); + std::iota( S.parameters.prediction.begin(), + S.parameters.prediction.end(), 0 ); - //---------------------------------------------------------- - // Subset dataFrameIn to lib_i - //---------------------------------------------------------- - DataFrame< double > dataFrameLib_i( lib_i.size(), - dataFrameIn.NColumns(), - dataFrameIn.ColumnNames() ); - - for ( size_t i = 0; i < lib_i.size(); i++ ) { - dataFrameLib_i.WriteRow( i, dataFrameIn.Row( lib_i[ i ] ) ) ; - } + S.CopyData(); // Reset to input data for subsetting - std::valarray targetVec = - dataFrameLib_i.VectorColumnName( paramCCM.targetName ); - -#ifdef DEBUG_ALL - { - std::lock_guard lck( EDM_CCM::mtx ); - std::cout << "dataFrameLib_i -------------------------------\n"; - std::cout << dataFrameLib_i; - } -#endif - - //---------------------------------------------------------- - // Pack embedding, target, neighbors for SimplexProjection - //---------------------------------------------------------- - DataEmbedNN embedNN = DataEmbedNN( &dataFrameLib_i, dataBlock, - targetVec, neighbors ); + // Subset data to lib_i rows + S.data = S.dataCCM.DataFrameFromRowIndex( lib_i ); - //---------------------------------------------------------- - // Simplex Projection: lib_str & pred_str set from N_row - //---------------------------------------------------------- - DataFrame S = SimplexProjection( paramCCM, embedNN, false ); + S.PrepareEmbedding( false ); // checkDataRows = false - VectorError ve = ComputeError( - S.VectorColumnName( "Observations" ), - S.VectorColumnName( "Predictions" ) ); + S.Distances(); // Write EDM: allDistances, allLibRows + + S.FindNeighbors(); // On allDistances allLibRows + + S.Simplex(); + S.FormatOutput(); + + VectorError ve = ComputeError( + S.projection.VectorColumnName( "Observations" ), + S.projection.VectorColumnName( "Predictions" ) ); + #ifdef DEBUG_ALL { - std::lock_guard lck( EDM_CCM::mtx ); + std::lock_guard lck( EDM_CCM_Lock::mtx ); std::cout << "CCM Simplex ---------------------------------\n"; - S.MaxRowPrint() = S.NRows(); - std::cout << S; + S.projection.MaxRowPrint() = S.projection.NRows(); + std::cout << S.projection; std::cout << "rho " << ve.rho << " RMSE " << ve.RMSE << " MAE " << ve.MAE << std::endl; } #endif - // Record values for these samples + // Record values for these lib_i samples rho [ n ] = ve.rho; RMSE[ n ] = ve.RMSE; MAE [ n ] = ve.MAE; - if ( includeData ) { + if ( S.parameters.includeData ) { // Save stats for this prediction std::valarray< double > predOutVec( 8 ); predOutVec[ 0 ] = predictionCount + 1; // N - predOutVec[ 1 ] = paramCCM.E; // E - predOutVec[ 2 ] = paramCCM.knn; // nn - predOutVec[ 3 ] = paramCCM.tau; // tau - predOutVec[ 4 ] = lib_size; // LibSize + predOutVec[ 1 ] = S.parameters.E; // E + predOutVec[ 2 ] = S.parameters.knn; // nn + predOutVec[ 3 ] = S.parameters.tau; // tau + predOutVec[ 4 ] = libSize; // LibSize predOutVec[ 5 ] = ve.rho; // rho predOutVec[ 6 ] = ve.RMSE; // RMSE predOutVec[ 7 ] = ve.MAE; // MAE - - crossMapValues.PredictStats.WriteRow(predictionCount,predOutVec); - - // Save predictions - crossMapValues.Predictions.push_front( S ); + + if ( S.parameters.colToTargetFlag ) { + // S object is SimplexClass colToTargetCCM + // Write to EDM object CrossMapValues colToTarget + S.colToTarget.PredictStats.WriteRow( predictionCount, + predOutVec ); + // Save predictions + S.colToTarget.Predictions.push_front( S.projection ); + } + else { + // S object is SimplexClass targetToColCCM + // Write to EDM object CrossMapValues targetToCol + S.targetToCol.PredictStats.WriteRow( predictionCount, + predOutVec ); + S.targetToCol.Predictions.push_front( S.projection ); + } } - predictionCount++; } // for ( n = 0; n < maxSamples; n++ ) std::valarray< double > statVec( 4 ); - statVec[ 0 ] = lib_size; + statVec[ 0 ] = libSize; statVec[ 1 ] = rho.sum() / maxSamples; statVec[ 2 ] = RMSE.sum() / maxSamples; statVec[ 3 ] = MAE.sum() / maxSamples; - crossMapValues.LibStats.WriteRow( lib_size_i, statVec ); - } // for ( lib_size : param.librarySizes ) - + if ( S.parameters.colToTargetFlag ) { + S.colToTarget.LibStats.WriteRow( libSize_i, statVec ); + } + else { + S.targetToCol.LibStats.WriteRow( libSize_i, statVec ); + } + } // for ( libSize_i < parameters.librarySizes ) } // try catch(...) { // push exception pointer onto queue for main thread to catch - std::lock_guard lck( EDM_CCM::q_mtx ); - EDM_CCM::exceptionQ.push( std::current_exception() ); + std::lock_guard lck( EDM_CCM_Lock::q_mtx ); + EDM_CCM_Lock::exceptionQ.push( std::current_exception() ); } } -//--------------------------------------------------------------------- -// Note that for CCM the library and prediction rows are the same. -// Note that dataBlock does NOT have the time in column 0. -// -// Return Distances: a square matrix with distances. -// Matrix elements D[i,j] hold the distance between the E-dimensional -// phase space point (vector) between rows (observations) i and j. -//--------------------------------------------------------------------- -DataFrame< double > CCMDistances( const DataFrame< double > &dataBlock, - Parameters param ) { - - size_t N_row = dataBlock.NRows(); +//----------------------------------------------------------------- +// Populate EDM::SimplexClass Parameters objects for CrossMap calls +//----------------------------------------------------------------- +void CCMClass::SetupParameters() { - size_t E = param.E; + // Each thread has it's own copy of input data & parameters + colToTargetCCM.dataCCM = data; // Copy + targetToColCCM.dataCCM = data; // Copy - DataFrame< double > D = DataFrame< double >( N_row, N_row ); + colToTargetCCM.parameters = parameters; // Copy + targetToColCCM.parameters = parameters; // Copy - // Initialise D to DistanceMax to avoid sort() : Add init constructor? - std::valarray< double > row_init( EDM_CCM::DistanceMax, N_row ); - for ( size_t row = 0; row < N_row; row++ ) { - D.WriteRow( row, row_init ); - } + // Swap column : target in targetToColCCM + targetToColCCM.parameters.columnNames = + std::vector< std::string >( 1, parameters.targetName ); + targetToColCCM.parameters.targetName = parameters.columnNames[0]; - for ( size_t row = 0; row < N_row; row++ ) { - // Get E-dimensional vector from this library row - std::valarray< double > v1_ = dataBlock.Row( row ); - // The first column (i=0) is NOT time, use it - std::valarray< double > v1 = v1_[ std::slice( 0, E, 1 ) ]; - - // Only compute upper triangular D, the diagonal and - // lower left are redundant: (col < N_row); row >= col - for ( size_t col = 0; col < N_row; col++ ) { - // Avoid redundant computations - if ( row >= col ) { - continue; // Computed in upper triangle, copied below - } - - // Find distance between vector (v) and other library vector - std::valarray< double > v2_ = dataBlock.Row( col ); - // The first column (i=0) is NOT time, use it - std::valarray< double > v2 = v2_[ std::slice( 0, E, 1 ) ]; - - D( row, col ) = Distance( v1, v2, DistanceMetric::Euclidean ); - - // Insert degenerate values since D[i,j] = D[j,i] - D( col, row ) = D( row, col ); - } - } - return D; -} + // Set flags to track direction of mapping for output data routing + colToTargetCCM.parameters.colToTargetFlag = true; + targetToColCCM.parameters.colToTargetFlag = false; -//--------------------------------------------------------------------- -// Return Neighbors { neighbors, distances }. neighbors is a matrix of -// row indices in the library matrix. Each neighbors row represents one -// prediction vector. Columns are the indices of knn nearest neighbors -// for the prediction vector (phase-space point) in the library matrix. -// distances is a matrix with the same shape as neighbors holding the -// corresponding distance values in each row. -// -// Note that the indices in neighbors are not the original indices in -// the libraryMatrix rows (observations), but are with respect to the -// distances subset defined by the list of rows lib_i, and so have values -// from 0 to len(lib_i)-1. -// -//--------------------------------------------------------------------- -Neighbors CCMNeighbors( const DataFrame< double > &DistancesIn, - std::vector< size_t > lib_i, - Parameters param ) { - - size_t N_row = lib_i.size(); - size_t knn = param.knn; + // JP: No need for embedding, lib/pred adjust. Just to get target. + colToTargetCCM.PrepareEmbedding( false ); // embedding, target, lib, pred + targetToColCCM.PrepareEmbedding( false ); // embedding, target, lib, pred -#ifdef DEBUG_ALL - { - std::lock_guard lck( EDM_CCM::mtx ); - std::cout << "CCMNeighbors Distances\n"; - for ( size_t r = 0; r < 5; r++ ) { - for ( int c = 0; c < 5; c++ ) { - std::cout << DistancesIn(r,c) << " "; - } std::cout << std::endl; + // Each thread has it's own copy of input target + colToTargetCCM.targetCCM = colToTargetCCM.target; // Copy + targetToColCCM.targetCCM = targetToColCCM.target; // Copy + + //------------------------------------------------------------------ + // DataFrames for output CrossMapValues structs in EDM object + //------------------------------------------------------------------ + size_t maxSamples; + if ( parameters.randomLib ) { + // Random samples from library + maxSamples = parameters.librarySizes.size() * parameters.subSamples; } - std::cout << "lib_i N_row: " << N_row - << " DistancesIn NRow: " << DistancesIn.NRows() << std::endl; + else { + // Contiguous samples up to the size of the library + maxSamples = parameters.librarySizes.size(); } -#endif - // Matrix to hold libraryMatrix row indices - // One row for each prediction vector, knn columns for each index - DataFrame< size_t > neighbors( N_row, knn ); + DataFrame< double > PredictionStats1( maxSamples, 8, + "N E nn tau LibSize rho RMSE MAE" ); + DataFrame< double > PredictionStats2( maxSamples, 8, + "N E nn tau LibSize rho RMSE MAE" ); - // Matrix to hold libraryMatrix knn distance values - // One row for each prediction vector, k_NN columns for each index - DataFrame< double > distances( N_row, knn ); + DataFrame< double > LibStats1( parameters.librarySizes.size(), 4, + "LibSize rho RMSE MAE" ); + DataFrame< double > LibStats2( parameters.librarySizes.size(), 4, + "LibSize rho RMSE MAE" ); - // For each prediction vector (row in predictionMatrix) find the list - // of library indices that are the closest knn points - size_t row = 0; - std::valarray< double > knn_distances( knn ); - std::valarray< size_t > knn_neighbors( knn ); + // Instantiate EDM CrossMapValues output structs and insert DataFrames + colToTargetCCM.colToTarget = CrossMapValues(); + targetToColCCM.targetToCol = CrossMapValues(); -#ifdef DEBUG_ALL - { - std::lock_guard lck( EDM_CCM::mtx ); - std::cout << "CCMNeighbors lib_i: "; - for ( size_t i = 0; i < lib_i.size(); i++ ) { - std::cout << lib_i[i] << " "; - } std::cout << std::endl << std::flush; - } -#endif - - size_t shift0 = abs( param.tau ) * param.E; - - for ( auto row_i : lib_i ) { - - // Take Distances( row, col ) a row at a time - // col represent the other row distance - std::valarray< double > dist_row = DistancesIn.Row( row_i ); - - // These new column indices are with respect to the lib_i vector - // not the original Distances with all other columns - - // Reset the neighbor and distance vectors for this pred row - for ( size_t i = 0; i < knn; i++ ) { - knn_neighbors[ i ] = 0; - // This avoids the need to sort the distances of this row - knn_distances[ i ] = EDM_CCM::DistanceMax; - } + colToTargetCCM.colToTarget.LibStats = LibStats1; + targetToColCCM.targetToCol.LibStats = LibStats2; - for ( size_t col_i = 0; col_i < N_row; col_i++ ) { + if ( parameters.includeData ) { + colToTargetCCM.colToTarget.PredictStats = PredictionStats1; + targetToColCCM.targetToCol.PredictStats = PredictionStats2; + } +} - if ( col_i > N_row - shift0 ) { - continue; - } - - double d_i = dist_row[ lib_i[col_i] ]; - // If d_i is less than values in knn_distances, add to list - auto max_it = std::max_element( begin( knn_distances ), - end ( knn_distances ) ); - if ( d_i < *max_it ) { - size_t max_i = std::distance( begin(knn_distances), max_it ); - knn_neighbors[ max_i ] = col_i; // Save the index - knn_distances[ max_i ] = d_i; // Save the value - } - } - - neighbors.WriteRow( row, knn_neighbors ); - distances.WriteRow( row, knn_distances ); +//---------------------------------------------------------------- +// Copy full library input data to EDM::Simplex objects for threads +//---------------------------------------------------------------- +void CCMClass::CopyData () { + + colToTargetCCM.data = colToTargetCCM.dataCCM; + targetToColCCM.data = targetToColCCM.dataCCM; + + colToTargetCCM.target = colToTargetCCM.targetCCM; + targetToColCCM.target = targetToColCCM.targetCCM; +} - row = row + 1; - } +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +void CCMClass::FormatOutput () { + // Create unified column names of output DataFrame + std::stringstream libRhoNames; + libRhoNames << "LibSize " + << parameters.columnNames[0] <<":"<< parameters.targetName << " " + << parameters.targetName <<":"<< parameters.columnNames[0]; + + // Allocate unified LibStats output DataFrame in EDM object + allLibStats = DataFrame< double >( parameters.librarySizes.size(), 3, + libRhoNames.str() ); - Neighbors ccmNeighbors = Neighbors(); - ccmNeighbors.neighbors = neighbors; - ccmNeighbors.distances = distances; + allLibStats.WriteColumn(0, colToTargetCCM.colToTarget.LibStats.Column( 0 )); + allLibStats.WriteColumn(1, colToTargetCCM.colToTarget.LibStats.Column( 1 )); + allLibStats.WriteColumn(2, targetToColCCM.targetToCol.LibStats.Column( 1 )); +} -#ifdef DEBUG_ALL - { - std::lock_guard lck( EDM_CCM::mtx ); - std::cout << "CCMNeighbors knn_neighbors\n"; - for ( size_t r = 0; r < 5; r++ ) { - for ( int c = 0; c < ccmNeighbors.neighbors.NColumns(); c++ ) { - std::cout << ccmNeighbors.neighbors(r,c) << " "; - } std::cout << std::endl; - } +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +void CCMClass::WriteOutput () { + if ( parameters.predictOutputFile.size() ) { + // Write to disk + allLibStats.WriteData( parameters.pathOut, + parameters.predictOutputFile ); } -#endif - - return ccmNeighbors; } diff --git a/src/CCM.h b/src/CCM.h new file mode 100644 index 0000000..eff4ca7 --- /dev/null +++ b/src/CCM.h @@ -0,0 +1,36 @@ + +#ifndef EDM_CCM_H +#define EDM_CCM_H + +#include +#include +#include +#include +#include +#include + +#include "EDM.h" +#include "Simplex.h" + +//---------------------------------------------------------------- +// CCM class inherits from Simplex class and defines +// CCM-specific projection methods +//---------------------------------------------------------------- +class CCMClass : public SimplexClass { +public: + SimplexClass colToTargetCCM; // object for column to target mapping + SimplexClass targetToColCCM; // object for target to column mapping + + // Constructor + CCMClass ( DataFrame< double > & data, + Parameters & parameters ); + + // Method declarations + void Project(); + void SetupParameters(); + void CopyData(); + void CCM(); + void FormatOutput(); + void WriteOutput(); +}; +#endif diff --git a/src/Common.cc b/src/Common.cc index c3991a8..b44773a 100644 --- a/src/Common.cc +++ b/src/Common.cc @@ -7,8 +7,8 @@ //--------------------------------------------------------------- // Binary sort function for FindNeighbors() & CCMNeighbors() //--------------------------------------------------------------- -bool DistanceCompare( const std::pair &x, - const std::pair &y ) { +bool DistanceCompare( const std::pair & x, + const std::pair & y ) { return x.first < y.first; } @@ -20,7 +20,7 @@ std::string ToLower( std::string str ) { std::string lowerStr( str ); std::transform( lowerStr.begin(), lowerStr.end(), lowerStr.begin(), ::tolower ); - + return lowerStr; } @@ -52,7 +52,7 @@ bool OnlyDigits( std::string str, bool integer ) { else { digits = "-.0123456789"; } - + // Is str_ purely numeric characters? bool onlyDigits = strspn(str_.c_str(), digits.c_str()) == str_.size(); @@ -83,7 +83,7 @@ std::vector SplitString( std::string inString, bool foundEnd = false; std::vector splitString; - + std::string word; eos = inString.length(); @@ -110,13 +110,13 @@ std::vector SplitString( std::string inString, if ( foundStart and foundEnd ) { foundStart = false; foundEnd = false; - + word = inString.substr( wordStart, wordEnd - wordStart ); - + // remove whitespace word.erase( std::remove_if( word.begin(), word.end(), ::isspace ), word.end() ); - + splitString.push_back( word ); } if ( pos == eos ) { @@ -133,7 +133,15 @@ std::vector SplitString( std::string inString, //---------------------------------------------------------------- VectorError ComputeError( std::valarray< double > obsIn, std::valarray< double > predIn ) { - + + if ( obsIn.size() != predIn.size() ) { + std::stringstream errMsg; + errMsg << "ComputeError(): Observation size " + << obsIn.size() << " is not equal to prediction size " + << predIn.size(); + throw std::runtime_error( errMsg.str() ); + } + // JP does find work on nan? Since nan != nan, probably not... // Use a slice to extract the overlapping subset of obsIn, PredIn // We need to find the appropriate slice parameters @@ -145,12 +153,12 @@ VectorError ComputeError( std::valarray< double > obsIn, for ( auto o : obsIn ) { if ( std::isnan( o ) ) { nanObs = true; break; } } for ( auto p : predIn ) { if ( std::isnan( p ) ) { nanPred= true; break; } } - - // vectors to hold data with no nans: reassigned below - std::valarray obs; - std::valarray pred; - size_t Nin = obsIn.size(); + // vectors to hold data with no nans: reassigned below + std::valarray< double > obs; + std::valarray< double > pred; + size_t Nin = obsIn.size(); + if ( not nanObs and not nanPred ) { obs = std::valarray< double >( obsIn ); pred = std::valarray< double >( predIn ); @@ -186,26 +194,31 @@ VectorError ComputeError( std::valarray< double > obsIn, lastValid ); lastValidIndex = Nin - lastValidIndex; // reverse iterator used... + + int Nout = (int) lastValidIndex - (int) firstValidIndex; - if ( lastValidIndex < firstValidIndex ) { - std::stringstream errMsg; - errMsg << "ComputeError(): Invalid lastIndex in nan detection " - << "lastValidIndex = " << lastValidIndex - << " firstValidIndex = " << firstValidIndex; - throw std::runtime_error( errMsg.str() ); + if ( Nout < 0 ) { + std::stringstream msg; + msg << "WARNING: ComputeError(): nan predictions found" + << " error not computed." << std::endl; + std::cout << msg.str(); + + Nout = 0; + obs = std::valarray< double >( 0., 1 ); // vector [0.] N = 1 + pred = std::valarray< double >( 0., 1 ); // vector [0.] N = 1 + } + else { + // Allocate the output arrays and fill with slices + obs = std::valarray< double >( Nout ); + pred = std::valarray< double >( Nout ); + + std::slice nonNan = std::slice( firstValidIndex, Nout, 1 ); + obs [ std::slice( 0, Nout, 1 ) ] = obsIn [ nonNan ]; + pred[ std::slice( 0, Nout, 1 ) ] = predIn[ nonNan ]; } - - // Allocate the output arrays and fill with slices - size_t Nout = lastValidIndex - firstValidIndex; - obs = std::valarray< double >( Nout ); - pred = std::valarray< double >( Nout ); - - std::slice nonNan = std::slice( firstValidIndex, Nout, 1 ); - obs [ std::slice( 0, Nout, 1 ) ] = obsIn [ nonNan ]; - pred[ std::slice( 0, Nout, 1 ) ] = predIn[ nonNan ]; } - - size_t N = pred.size(); + + size_t N = std::max( 1, (int) pred.size() ); std::valarray< double > two( 2, N ); // Vector of 2's for squaring double sumPred = pred.sum(); @@ -217,25 +230,24 @@ VectorError ComputeError( std::valarray< double > obsIn, double sumErr = abs( obs - pred ).sum(); double sumSqrErr = pow( obs - pred, two ).sum(); double sumProd = ( obs * pred ).sum(); - + double rho; // Pearson correlation coefficient - if ( sumSqrPred * N == sumSqrPred ) { + double denom = ( std::sqrt( ( sumSqrObs - N * pow( meanObs, 2 ) ) ) * + std::sqrt( ( sumSqrPred - N * pow( meanPred, 2 ) ) ) ); + + if ( denom == 0 or std::isnan( denom ) ) { rho = 0; } else { - rho = ( sumProd - N * meanObs * meanPred ) / - ( std::sqrt( ( sumSqrObs - N * pow( meanObs, 2 ) ) ) * - std::sqrt( ( sumSqrPred - N * pow( meanPred, 2 ) ) ) ); + rho = ( sumProd - N * meanObs * meanPred ) / denom; } VectorError vectorError = VectorError(); - - vectorError.RMSE = sqrt( sumSqrErr / N ); - vectorError.MAE = sumErr / N; + vectorError.RMSE = sqrt( sumSqrErr / N ); + vectorError.MAE = sumErr / N; + vectorError.rho = rho; - vectorError.rho = rho; - return vectorError; } diff --git a/src/Common.h b/src/Common.h index d728af0..ba88862 100644 --- a/src/Common.h +++ b/src/Common.h @@ -1,5 +1,5 @@ -#ifndef COMMON_H -#define COMMON_H +#ifndef EDM_COMMON_H +#define EDM_COMMON_H #include #include @@ -9,21 +9,18 @@ #include #include #include +#include // std::ref #ifdef _MSC_VER #include // macro constants for MSVC C++ operators not in ISO646 #endif -#include "DataFrame.h" // has #include Common.h - -// forward declaration for SMap solver -std::valarray < double > SVD( DataFrame < double > A, - std::valarray< double > B ); - // Enumerations enum class Method { None, Embed, Simplex, SMap, CCM }; enum class DistanceMetric { Euclidean, Manhattan }; +#include "DataFrame.h" + //--------------------------------------------------------- // Data structs //--------------------------------------------------------- @@ -53,37 +50,19 @@ struct CCMValues { }; struct MultiviewValues { - DataFrame< double > Combo_rho; // col_i..., rho, MAE, RMSE + DataFrame< double > ComboRho; // col_i..., rho, MAE, RMSE DataFrame< double > Predictions; - std::vector< std::string > Combo_rho_table; // includes column names - -#ifdef MULTIVIEW_VALUES_OVERLOAD - // Don't define constructors for the setuptools module build on Windows - // The MSVC compiler with pybind11 does not handle overloads easily... - // https://pybind11.readthedocs.io/en/stable/classes.html - - // Constructors - MultiviewValues(); - - MultiviewValues( DataFrame< double > combo_rho, - DataFrame< double > predictions, - std::vector< std::string > combo_rho_table ): - Combo_rho( combo_rho ), Predictions( predictions ), - Combo_rho_table( combo_rho_table ) {} -#endif + std::vector< std::string > ComboRhoTable; // includes column names }; //------------------------------------------------------------- // Prototypes -// Primary API functions generally have two call-signatures. -// The first takes a (path, file name) pair specifying the data -// file image on disk to be loaded and converted to a data frame. -// The second replaces these two arguments with a DataFrame object. -// -// NOTE: These are the first declarations seen by the compiler -// for the API and provide default argument values //------------------------------------------------------------- -std::string ToLower( std::string str ); +std::string ToLower ( std::string str ); +bool OnlyDigits( std::string str, bool integerOnly ); + +std::vector SplitString( std::string inString, + std::string delimeters ); VectorError ComputeError( std::valarray< double > obs, std::valarray< double > pred ); @@ -91,294 +70,4 @@ VectorError ComputeError( std::valarray< double > obs, std::string increment_datetime_str( std::string datetime1, std::string datetime2, int tp ); - -bool DistanceCompare( const std::pair &x, - const std::pair &y ); - -// API functions Embed() and MakeBlock() are in Embed.h Embed.cc - -DataFrame Simplex( std::string pathIn = "./data/", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int E = 0, - int Tp = 1, - int knn = 0, - int tau = -1, - int exclusionRadius = 0, - std::string colNames = "", - std::string targetName = "", - bool embedded = false, - bool const_predict = false, - bool verbose = true ); - -DataFrame Simplex( DataFrame< double > &dataFrameIn, - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int E = 0, - int Tp = 1, - int knn = 0, - int tau = -1, - int exclusionRadius = 0, - std::string colNames = "", - std::string targetName = "", - bool embedded = false, - bool const_predict = false, - bool verbose = true ); - -// SMap is a special case since it can be called with a function pointer -// to the SVD solver. This is done so that interfaces such as pybind11 -// can provide their own object for the solver. -// 1) Data path/file with default SVD (LAPACK) assigned in Smap.cc 2) -SMapValues SMap( std::string pathIn = "./data/", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int E = 0, - int Tp = 1, - int knn = 0, - int tau = -1, - double theta = 0, - int exclusionRadius = 0, - std::string columns = "", - std::string target = "", - std::string smapFile = "", - std::string derivatives = "", - bool embedded = false, - bool const_predict = false, - bool verbose = true ); - -// 2) DataFrame with default SVD (LAPACK) assigned in Smap.cc 2) -SMapValues SMap( DataFrame< double > &dataFrameIn, - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int E = 0, - int Tp = 1, - int knn = 0, - int tau = -1, - double theta = 0, - int exclusionRadius = 0, - std::string columns = "", - std::string target = "", - std::string smapFile = "", - std::string derivatives = "", - bool embedded = false, - bool const_predict = false, - bool verbose = true ); - -// 3) Data path/file with external solver object, init to default SVD -SMapValues SMap( std::string pathIn = "./data/", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int E = 0, - int Tp = 1, - int knn = 0, - int tau = -1, - double theta = 0, - int exclusionRadius = 0, - std::string columns = "", - std::string target = "", - std::string smapFile = "", - std::string derivatives = "", - std::valarray (*solver)(DataFrame < double >, - std::valarray < double >) = &SVD, - bool embedded = false, - bool const_predict = false, - bool verbose = true ); - -// 4) DataFrame with external solver object, init to default SVD -SMapValues SMap( DataFrame< double > &dataFrameIn, - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int E = 0, - int Tp = 1, - int knn = 0, - int tau = -1, - double theta = 0, - int exclusionRadius = 0, - std::string columns = "", - std::string target = "", - std::string smapFile = "", - std::string derivatives = "", - std::valarray (*solver)(DataFrame < double >, - std::valarray < double >) = &SVD, - bool embedded = false, - bool const_predict = false, - bool verbose = true ); - -CCMValues CCM( std::string pathIn = "./data/", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - int E = 0, - int Tp = 0, - int knn = 0, - int tau = -1, - std::string colNames = "", - std::string targetName = "", - std::string libSizes_str = "", - int sample = 0, - bool random = true, - bool replacement = false, - unsigned seed = 0, // seed=0: use RNG - bool includeData = false, - bool verbose = true ); - -CCMValues CCM( DataFrame< double > dataFrameIn, - std::string pathOut = "./", - std::string predictFile = "", - int E = 0, - int Tp = 0, - int knn = 0, - int tau = -1, - std::string colNames = "", - std::string targetName = "", - std::string libSizes_str = "", - int sample = 0, - bool random = true, - bool replacement = false, - unsigned seed = 0, // seed=0: use RNG - bool includeData = false, - bool verbose = true ); - -MultiviewValues Multiview( std::string pathIn = "./", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int D = 0, - int E = 1, - int Tp = 1, - int knn = 0, - int tau = -1, - std::string columns = "", - std::string target = "", - int multiview = 0, - int exclusionRadius = 0, - bool trainLib = true, - bool verbose = false, - unsigned nThreads = 4 ); - -MultiviewValues Multiview( DataFrame< double >, - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int D = 0, - int E = 1, - int Tp = 1, - int knn = 0, - int tau = -1, - std::string columns = "", - std::string target = "", - int multiview = 0, - int exclusionRadius = 0, - bool trainLib = true, - bool verbose = false, - unsigned nThreads = 4 ); - -DataFrame EmbedDimension( std::string pathIn = "./data/", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int maxE = 10, - int Tp = 1, - int tau = -1, - std::string colNames = "", - std::string targetName = "", - bool embedded = false, - bool verbose = true, - unsigned nThreads = 4 ); - -DataFrame EmbedDimension( DataFrame< double > &dataFrameIn, - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int maxE = 10, - int Tp = 1, - int tau = -1, - std::string colNames = "", - std::string targetName = "", - bool embedded = false, - bool verbose = true, - unsigned nThreads = 4 ); - -DataFrame PredictInterval( std::string pathIn = "./data/", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int maxTp = 10, - int E = 0, - int tau = -1, - std::string colNames = "", - std::string targetName = "", - bool embedded = false, - bool verbose = true, - unsigned nThreads = 4 ); - -DataFrame PredictInterval( DataFrame< double > &dataFrameIn, - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - int maxTp = 10, - int E = 0, - int tau = -1, - std::string colNames = "", - std::string targetName = "", - bool embedded = false, - bool verbose = true, - unsigned nThreads = 4 ); - -DataFrame PredictNonlinear( std::string pathIn = "./data/", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - std::string theta = "", - int E = 0, - int Tp = 1, - int knn = 0, - int tau = -1, - std::string colNames = "", - std::string targetName = "", - bool embedded = false, - bool verbose = true, - unsigned nThreads = 4 ); - -DataFrame PredictNonlinear( DataFrame< double > &dataFrameIn, - std::string pathOut = "./", - std::string predictFile = "", - std::string lib = "", - std::string pred = "", - std::string theta = "", - int E = 0, - int Tp = 1, - int knn = 0, - int tau = -1, - std::string colNames = "", - std::string targetName = "", - bool embedded = false, - bool verbose = true, - unsigned nThreads = 4 ); #endif diff --git a/src/DataFrame.h b/src/DataFrame.h index cb06c63..210a4a1 100644 --- a/src/DataFrame.h +++ b/src/DataFrame.h @@ -2,8 +2,8 @@ #define DATAFRAME_H // NOTE: This header deviates from the desired class implementation -// where *.h provides declarations, *.cc methods. This is solely to -// accomodate the OSX XCode environment which seems unable to deal +// where *.h provides declarations, *.cc implementation. This is solely +// to accomodate the OSX XCode environment which seems unable to deal // with c++11 standard template implemenations. // A possible solution is to link against libc++ on OSX. // See ../etc/Notes, ../etc/libstdc++_Notes.txt. @@ -12,24 +12,14 @@ #include #include -#include "Common.h" - -// Since #include DataFrame.h is in Common.h, need forward declarations -bool OnlyDigits( std::string str, bool integerOnly ); - -std::vector SplitString( std::string inString, - std::string delimeters = "," ); +// Common.cc +extern std::vector SplitString( std::string inString, + std::string delimeters = "," ); +extern bool OnlyDigits( std::string str, bool integerOnly ); // Type definition for CSV NamedData to pair column names & column data typedef std::vector>> NamedData; -// Container for parsed data file returned by ReadData() -struct ParsedData { - std::vector< std::string > time; - std::string timeName; - NamedData namedData; -}; - //---------------------------------------------------------------- // DataFrame class // Data container is a single, contiguous valarray: elements. @@ -39,49 +29,51 @@ struct ParsedData { //---------------------------------------------------------------- template class DataFrame { - + size_t n_rows; size_t n_columns; std::valarray elements; - + std::vector< std::string > columnNames; std::map< std::string, size_t > columnNameToIndex; std::vector< std::string > time; std::string timeName; - + NamedData namedData; + size_t maxRowPrint; bool noTime; bool partialDataRowsDeleted; - + public: //----------------------------------------------------------------- // Destructor //----------------------------------------------------------------- ~DataFrame() {} - + //----------------------------------------------------------------- // Constructors //----------------------------------------------------------------- - DataFrame() {} - + DataFrame() : n_rows(0), n_columns(0), noTime( false ), + partialDataRowsDeleted( false ) {} + //----------------------------------------------------------------- // Load data from CSV file path/fileName, populate DataFrame //----------------------------------------------------------------- DataFrame( std::string path, std::string fileName, bool noTime = false ): maxRowPrint( 10 ), noTime( noTime ), partialDataRowsDeleted( false ) { - ParsedData parsedData = ReadData( path, fileName ); - SetupDataFrame( parsedData ); // Process parsedData into a DataFrame + ReadData( path, fileName ); + SetupDataFrame(); // Process parsedData into a DataFrame } - + //----------------------------------------------------------------- // Empty DataFrame of size (row, columns), no column names //----------------------------------------------------------------- DataFrame( size_t rows, size_t columns ): n_rows( rows ), n_columns( columns ), elements( columns * rows ), maxRowPrint( 10 ), partialDataRowsDeleted( false ) {} - + //----------------------------------------------------------------- // Empty DataFrame of size (rows, columns) with column names in a // single whitespace delimited string. @@ -93,7 +85,7 @@ class DataFrame { { BuildColumnNameIndex( colNames ); } - + //----------------------------------------------------------------- // Empty DataFrame of size (rows, columns) with column names in a // string vector. @@ -106,7 +98,7 @@ class DataFrame { { BuildColumnNameIndex(); } - + //----------------------------------------------------------------- // Fortran style element access operators M(row,col) //----------------------------------------------------------------- @@ -123,19 +115,19 @@ class DataFrame { size_t NColumns() const { return n_columns; } size_t NRows() const { return n_rows; } size_t size() const { return n_rows * n_columns; } - + std::valarray Elements() const { return elements; } std::valarray &Elements() { return elements; } - + std::vector< std::string > Time() const { return time; } std::vector< std::string > &Time() { return time; } - + std::string TimeName() const { return timeName; } std::string &TimeName() { return timeName; } - + std::vector< std::string > ColumnNames() const { return columnNames; } std::vector< std::string > &ColumnNames() { return columnNames; } - + std::map< std::string, size_t > ColumnNameToIndex() const { return columnNameToIndex; } @@ -145,10 +137,10 @@ class DataFrame { size_t MaxRowPrint() const { return maxRowPrint; } size_t &MaxRowPrint() { return maxRowPrint; } - + bool PartialDataRowsDeleted() const { return partialDataRowsDeleted; } bool &PartialDataRowsDeleted() { return partialDataRowsDeleted; } - + //----------------------------------------------------------------- // Return column from index col //----------------------------------------------------------------- @@ -184,11 +176,11 @@ class DataFrame { } errMsg << "]" << std::endl; throw std::runtime_error( errMsg.str() ); } - + size_t col_i = std::distance( columnNames.begin(), ci ); - - std::valarray vec = Column( col_i ); - + + std::valarray< double > vec = Column( col_i ); + return vec; } @@ -196,11 +188,11 @@ class DataFrame { // Return (sub)DataFrame of specified column indices //----------------------------------------------------------------- DataFrame DataFrameFromColumnIndex( std::vector column_i ) { - + DataFrame M = DataFrame( n_rows, column_i.size() ); - size_t col_j = 0; - + // Can't use slice since column_i are not structured + size_t col_j = 0; for ( size_t i = 0; i < column_i.size(); i++ ) { size_t col_i = column_i[ i ]; @@ -211,13 +203,13 @@ class DataFrame { << col_i << ") exceeds the data frame domain.\n"; throw std::runtime_error( errMsg.str() ); } - - std::valarray column_vec_i = Column( col_i ); + + std::valarray< double > column_vec_i = Column( col_i ); M.WriteColumn( col_j, column_vec_i ); col_j++; } - + // Add time vector if present if ( time.size() ) { M.Time() = time; @@ -232,7 +224,7 @@ class DataFrame { M.ColumnNames() = colNames; M.BuildColumnNameIndex(); } - + return M; } @@ -244,18 +236,18 @@ class DataFrame { std::vector colNames ) { // vector of column indices for DataFrameFromColumnIndex() - std::vector col_i_vec; - + std::vector< size_t > col_i_vec; + // Map column names to indices - std::vector::iterator si; + std::vector< std::string >::iterator si; for ( auto ci = colNames.begin(); ci != colNames.end(); ++ci ) { auto si = find( columnNames.begin(), columnNames.end(), *ci ); - + if ( si != columnNames.end() ) { col_i_vec.push_back( std::distance( columnNames.begin(), si ) ); } } - + // Validation if ( col_i_vec.size() != colNames.size() ) { std::stringstream errMsg; @@ -270,25 +262,67 @@ class DataFrame { } errMsg << "]" << std::endl; throw std::runtime_error( errMsg.str() ); } - - DataFrame M_col = DataFrameFromColumnIndex( col_i_vec ); - + + DataFrame< double > M_col = DataFrameFromColumnIndex( col_i_vec ); + // Insert columnNames if not already present if ( not M_col.ColumnNames().size() ) { M_col.ColumnNames() = colNames; M_col.BuildColumnNameIndex(); } - + return M_col; } + //----------------------------------------------------------------- + // Return (sub)DataFrame of specified row indices + //----------------------------------------------------------------- + DataFrame DataFrameFromRowIndex( std::vector row_index ) { + + DataFrame< double > M = DataFrame( row_index.size(), n_columns ); + + // Can't use slice since row_index_i are not structured + size_t row_j = 0; + for ( size_t row_i : row_index ) { + if ( row_i >= n_rows ) { + std::stringstream errMsg; + errMsg << "DataFrame::DataFrameFromRowIndex(): " + << "A row index (" + << row_i << ") exceeds the data frame domain.\n"; + throw std::runtime_error( errMsg.str() ); + } + + std::valarray row_vec_i = Row( row_i ); + + M.WriteRow( row_j, row_vec_i ); + row_j++; + } + + // Add time vector if present + if ( time.size() ) { + std::vector< std::string > timeRow( row_index.size() ); + for ( size_t i = 0; i < row_index.size(); i++ ) { + timeRow[ i ] = time[ row_index[ i ] ]; + } + M.Time() = timeRow; + M.TimeName() = timeName; + } + // Add columnNames if present + if ( columnNames.size() ) { + M.ColumnNames() = columnNames; + M.BuildColumnNameIndex(); + } + + return M; + } + //----------------------------------------------------------------- // Return Elements in Column Major order (Fortran) //----------------------------------------------------------------- std::valarray ColumnMajorData() const { std::valarray colMajorElements( elements.size() ); - + for ( size_t col = 0; col < n_columns; col++ ) { // slice( size_t start, size_t length, size_t stride ) colMajorElements[ std::slice( col * n_rows, n_rows, 1 ) ] = @@ -303,7 +337,7 @@ class DataFrame { //----------------------------------------------------------------- void WriteRow( size_t row, std::valarray array ) { size_t N = array.size(); - + if ( N != n_columns ) { std::stringstream errMsg; errMsg << "DataFrame::WriteRow(): array must have " @@ -357,7 +391,7 @@ class DataFrame { "already been deleted." << std::endl; return; } - + partialDataRowsDeleted = true; if ( nrows > n_rows ) { @@ -367,7 +401,7 @@ class DataFrame { << "NRows (" << n_rows << ")" << std::endl; throw( std::runtime_error( errMsg.str() ) ); } - + // Update n_rows n_rows = n_rows - nrows; @@ -381,7 +415,7 @@ class DataFrame { // Copy elements into data std::valarray< double > data( elements ); - + // Resize elements size_t n_elements = elements.size() - nrows * n_columns; elements.resize( n_elements ); @@ -399,7 +433,7 @@ class DataFrame { elements[ std::slice( 0, n_elements, 1 ) ] = ( std::valarray< double > ) data[ elements_i ]; } - + //----------------------------------------------------------------- // Build Column Name Index( std::string colNames ) //----------------------------------------------------------------- @@ -440,7 +474,7 @@ class DataFrame { columnNameToIndex[ columnNames[i] ] = i; } } - + //------------------------------------------------------------------ // Stream DataFrame to ostream //------------------------------------------------------------------ @@ -449,23 +483,23 @@ class DataFrame { os.precision( 4 ); os.fill( ' ' ); os.setf( std::ios::fixed, std::ios::floatfield ); - + os << "DataFrame: -----------------------------------\n"; os << D.NRows() << " rows, " << D.NColumns() << " columns.\n"; os << "---------------- First " << D.MaxRowPrint() << " rows ---------------\n"; - + // print names of columns if ( D.timeName.size() ) { os << std::setw(10) << D.timeName; } - + for ( size_t i = 0; i < D.ColumnNames().size(); i++ ) { os << std::setw(13) << D.ColumnNames()[i]; } os << std::endl; - + os << "----------------------------------------------\n"; - + // print vec data up to maxRowPrint points for ( size_t row = 0; row < D.NRows() and row < D.MaxRowPrint(); row++ ) { @@ -474,7 +508,7 @@ class DataFrame { if ( D.time.size() ) { os << std::setw(10) << D.time[ row ]; } - + // print data points from each col for ( size_t col = 0; col < D.NColumns(); col++ ) { os << std::setw(13) << D( row, col ); @@ -490,7 +524,7 @@ class DataFrame { // Write contents to file //------------------------------------------------------------------ void WriteData( std::string outputFilePath, std::string outputFileName ) { - + // Vector of strings to hold image of DataFrame for file output std::vector< std::string > fileLines; @@ -520,7 +554,7 @@ class DataFrame { if ( TimeName().size() ) { lineStr << TimeName() << ","; } - + // Push column name from each column into the string stream for ( size_t colIdx = 0; colIdx < n_columns; colIdx++ ) { lineStr << ColumnNames()[ colIdx ]; @@ -559,9 +593,9 @@ class DataFrame { // Write contents to file std::ofstream outputFile( outputFilePath + outputFileName ); - + if ( outputFile.is_open() ) { - + std::copy( fileLines.begin(), fileLines.end(), std::ostream_iterator(outputFile,"\n") ); @@ -580,9 +614,7 @@ class DataFrame { //------------------------------------------------------------------ // Process parsedData from ReadData() to populate DataFrame //------------------------------------------------------------------ - void SetupDataFrame( ParsedData parsedData ) { - - NamedData namedData = parsedData.namedData; + void SetupDataFrame() { // Setup column names in same order as dataFrame std::vector< std::string > colNames; @@ -590,14 +622,12 @@ class DataFrame { iterate != namedData.end(); iterate++ ) { colNames.push_back( iterate->first ); } - + // Initialize DataFrame members and storage n_rows = namedData.begin()->second.size(); n_columns = namedData.size(); elements = std::valarray ( n_rows * n_columns ); columnNames = colNames; - time = parsedData.time; - timeName = parsedData.timeName; BuildColumnNameIndex(); @@ -606,7 +636,7 @@ class DataFrame { // NamedData is : pair< string, vector > for ( NamedData::iterator iterate = namedData.begin(); iterate != namedData.end(); iterate++ ) { - + size_t colIdx = std::distance( namedData.begin(), iterate ); for ( size_t rowIdx = 0; rowIdx < n_rows; rowIdx++ ) { @@ -614,15 +644,15 @@ class DataFrame { } } } - + //------------------------------------------------------------------ // Read disk file. Parse into a NamedData container and time vector. //------------------------------------------------------------------ - ParsedData ReadData( std::string path, std::string fileName ) { - + void ReadData( std::string path, std::string fileName ) { + // Create input file stream and open file for input std::ifstream dataStrm( path + fileName ); - + // Ensure file access is good before reading if ( not dataStrm.is_open() ) { std::stringstream errMsg; @@ -636,16 +666,16 @@ class DataFrame { << " is not ready for reading." << std::endl; throw std::runtime_error( errMsg.str() ); } - + // Read into a vector of strings, one line per string std::vector< std::string > dataLines; std::string tmp; - + while( getline( dataStrm, tmp ) ) { dataLines.push_back( tmp ); } dataStrm.close(); - + #ifdef DEBUG_ALL std::cout << "------- ReadData() Contents of file " << fileName << " -------" << std::endl; @@ -655,20 +685,13 @@ class DataFrame { } #endif - // Vector of times - std::vector< std::string > time; - std::string timeName; - - // Container of data name : vector pairs - NamedData namedData; // vector< pair< string, vector< double >> >; - // Container of column names in the same order as in csv file std::vector< std::string > colNames; // Check first line to see if it's only numeric digits, or a header bool onlyDigits = true; std::vector firstLineWords = SplitString( dataLines[0] ); - + for ( auto si = firstLineWords.begin(); si != firstLineWords.end(); ++si ){ @@ -694,7 +717,7 @@ class DataFrame { if ( not noTime ) { timeName = colNames[ 0 ]; } - + // Setup each col in namedData with new vec to insert numerical data // If noTime true then first column is data size_t startCol_i = noTime ? 0 : 1; @@ -703,7 +726,7 @@ class DataFrame { std::vector() ); namedData.push_back( colPair ); } - + // Process each line in dataLines to fill in data vectors and time for ( size_t lineIdx = 0; lineIdx < dataLines.size(); lineIdx++ ) { @@ -722,7 +745,7 @@ class DataFrame { if ( not noTime ) { time.push_back( words[ 0 ] ); } - + try { // Convert data columns to double, add to namedData for ( size_t colIdx = startCol_i; @@ -760,13 +783,6 @@ class DataFrame { } std::cout << std::endl; } #endif - - ParsedData parsedData; - parsedData.time = time; - parsedData.timeName = timeName; - parsedData.namedData = namedData; - - return parsedData; } }; #endif diff --git a/src/DateTimeUtil.cc b/src/DateTime.cc similarity index 86% rename from src/DateTimeUtil.cc rename to src/DateTime.cc index 6dfc69b..0a3cae0 100644 --- a/src/DateTimeUtil.cc +++ b/src/DateTime.cc @@ -1,3 +1,4 @@ + #include "DateTime.h" // Provide some utility for parsing datetime std::strings @@ -30,9 +31,9 @@ std::string fmt_hhmmsssss ("%H:%M:%S"); // @param date_fmt : true if this is a date object // @return : none, just populates the tm obj //---------------------------------------------------------------------- -void parse_datetime_str ( struct tm & time_obj, - std::string datetime_str, - bool date_fmt ) { +void ParseDatetimeString ( struct tm & time_obj, + std::string datetime_str, + bool date_fmt ) { // parsing delim is different for date or time char parse_delim = date_fmt ? '-' : ':'; @@ -72,18 +73,18 @@ void parse_datetime_str ( struct tm & time_obj, // @param datetime : the datetime to parse // @return datetime : the datetime to parse //---------------------------------------------------------------------- -datetime_info parse_datetime ( std::string datetime ) { +datetime_info ParseDatetime ( std::string datetime ) { datetime_info output; // check which time format we have. populate output with each case if ( std::regex_match( datetime, regEx_yyyymmdd ) ) { output.datetime_fmt = fmt_yyyymmdd; - parse_datetime_str( output.time, datetime, true ); + ParseDatetimeString( output.time, datetime, true ); } else if ( std::regex_match( datetime, regEx_hhmmss )) { output.datetime_fmt = fmt_hhmmss; - parse_datetime_str( output.time, datetime, false ); + ParseDatetimeString( output.time, datetime, false ); } else if ( std::regex_match( datetime, regEx_yymmddhhmmss )) { output.datetime_fmt = fmt_yymmddhhmmss; @@ -91,8 +92,8 @@ datetime_info parse_datetime ( std::string datetime ) { int delim_pos = datetime.find(' '); std::string date = datetime.substr(0, delim_pos); std::string time = datetime.substr(delim_pos+1, datetime.size()); - parse_datetime_str( output.time, date, true ); - parse_datetime_str( output.time, time, false ); + ParseDatetimeString( output.time, date, true ); + ParseDatetimeString( output.time, time, false ); } else if ( std::regex_match( datetime, regEx_yymmddthhmmss )) { output.datetime_fmt = fmt_yymmddthhmmss; @@ -100,14 +101,14 @@ datetime_info parse_datetime ( std::string datetime ) { int delim_pos = datetime.find('T'); std::string date = datetime.substr(0, delim_pos); std::string time = datetime.substr(delim_pos+1, datetime.size()); - parse_datetime_str( output.time, date, true ); - parse_datetime_str( output.time, time, false ); + ParseDatetimeString( output.time, date, true ); + ParseDatetimeString( output.time, time, false ); } else if ( std::regex_match( datetime, regEx_hhmmsssss )) { output.datetime_fmt = fmt_hhmmsssss; // trim the milliseconds off and parse datetime = datetime.substr(0,datetime.size()-4); - parse_datetime_str( output.time, datetime, false ); + ParseDatetimeString( output.time, datetime, false ); } else { output.unrecognized_fmt = true; @@ -125,11 +126,11 @@ datetime_info parse_datetime ( std::string datetime ) { // @param tp : the amount to increment the time diff by // @return : the new incremented timestd::string //---------------------------------------------------------------------- -std::string increment_datetime_str ( std::string datetime1, - std::string datetime2, int tp ) { - //parse datetimes - datetime_info dtinfo1 = parse_datetime( datetime1 ); - datetime_info dtinfo2 = parse_datetime( datetime2 ); +std::string IncrementDatetime ( std::string datetime1, + std::string datetime2, int tp ) { + // parse datetimes + datetime_info dtinfo1 = ParseDatetime( datetime1 ); + datetime_info dtinfo2 = ParseDatetime( datetime2 ); if ( dtinfo1.unrecognized_fmt or dtinfo2.unrecognized_fmt ) { // return empty string diff --git a/src/DateTime.h b/src/DateTime.h index 9084864..77f1742 100644 --- a/src/DateTime.h +++ b/src/DateTime.h @@ -19,14 +19,13 @@ struct datetime_info { }; // Prototypes -void parse_datetime_str ( struct tm & time_obj, - std::string datetime_str, - bool date_fmt ); +void ParseDatetimeString ( struct tm & time_obj, + std::string datetime_str, + bool date_fmt ); -datetime_info parse_datetime ( std::string datetime ); - -std::string increment_datetime_str ( std::string datetime1, - std::string datetime2, - int tp ); +datetime_info ParseDatetime ( std::string datetime ); +std::string IncrementDatetime ( std::string datetime1, + std::string datetime2, + int tp ); #endif diff --git a/src/EDM.cc b/src/EDM.cc new file mode 100644 index 0000000..84d773c --- /dev/null +++ b/src/EDM.cc @@ -0,0 +1,88 @@ + +#include "EDM.h" + +// Declared in API.h +extern DataFrame< double > MakeBlock( DataFrame< double > &, int, int, + std::vector< std::string > ); + +//---------------------------------------------------------------- +// Constructors +//---------------------------------------------------------------- +EDM::EDM ( DataFrame< double > & data, + Parameters & parameters ) : + data( data ), anyTies( false ), parameters( parameters ) {} + +//---------------------------------------------------------------- +// FindNeighbors : See EDM_Neighbors.cc +//---------------------------------------------------------------- + +//---------------------------------------------------------------- +// Project : Implemented in sub-class +//---------------------------------------------------------------- +void EDM::Project () {} + +//---------------------------------------------------------------- +// Implemented as a wrapper for API MakeBlock() +// Note: dataFrame must have the columnNameToIndex map +// +// NOTE: Truncates data by tau * (E-1) rows to remove +// nan values (partial data rows) +// NOTE: The returned data block does NOT have the time column +//---------------------------------------------------------------- +void EDM::EmbedData() { + + if ( not parameters.columnIndex.size() and + data.ColumnNameToIndex().empty() ) { + throw std::runtime_error("EDM::Embed(): columnNameIndex empty.\n"); + } + + // If columns provided, validate they are in dataFrameIn + for ( auto colName : parameters.columnNames ) { + auto ci = find( data.ColumnNames().begin(), + data.ColumnNames().end(), colName ); + + if ( ci == data.ColumnNames().end() ) { + std::stringstream errMsg; + errMsg << "EDM::Embed(): Failed to find column " + << colName << " in dataFrame with columns: [ "; + for ( auto col : data.ColumnNames() ) { + errMsg << col << " "; + } errMsg << " ]\n"; + throw std::runtime_error( errMsg.str() ); + } + } + + // Get column names for MakeBlock + std::vector< std::string > colNames; + if ( parameters.columnNames.size() ) { + // column names are strings use as-is + colNames = parameters.columnNames; + } + else if ( parameters.columnIndex.size() ) { + // columns are indices : Create column names for MakeBlock + for ( size_t i = 0; i < parameters.columnIndex.size(); i++ ) { + std::stringstream ss; + ss << "V" << parameters.columnIndex[i]; + colNames.push_back( ss.str() ); + } + } + else { + throw std::runtime_error( "EDM::Embed(): columnNames and " + " columnIndex are empty.\n" ); + } + + // Extract the specified columns (sub)DataFrame from dataFrameIn + DataFrame< double > dataFrame; + + if ( parameters.columnNames.size() ) { + dataFrame = data.DataFrameFromColumnNames( parameters.columnNames ); + } + else if ( parameters.columnIndex.size() ) { + // already have column indices + // Note there will be no column names transferred + dataFrame = data.DataFrameFromColumnIndex( parameters.columnIndex ); + } + + embedding = MakeBlock( std::ref( dataFrame ), parameters.E, + parameters.tau, colNames ); +} diff --git a/src/EDM.h b/src/EDM.h new file mode 100644 index 0000000..4f14acb --- /dev/null +++ b/src/EDM.h @@ -0,0 +1,67 @@ + +#ifndef EDM_H +#define EDM_H + +#include +#include "Common.h" +#include "Parameter.h" + +//--------------------------------------------------------------------- +// EDM Class +// Central data object and base class for EDM algorithms. +// Specific algorithm projection methods defined in sub-classes. +// +// NOTE JP: Tony recommends to explicitly define special members: +// http://www.cplusplus.com/doc/tutorial/classes2/ +//--------------------------------------------------------------------- +class EDM { + +public: // No need for private or protected + DataFrame< double > data; + DataFrame< double > embedding; + + DataFrame< size_t > knn_neighbors; // N pred rows, knn columns; sorted + DataFrame< double > knn_distances; // N pred rows, knn columns; sorted + + DataFrame< size_t > allLibRows; // 1 row, N lib columns + DataFrame< double > allDistances; // N pred rows N lib columns + + DataFrame< double > projection; // Simplex & SMap Output + DataFrame< double > coefficients; // SMap Output + + DataFrame< double > allLibStats; // CCM unified libsize, rho, RMSE, MAE + CrossMapValues colToTarget; // CCM CrossMap() thread results + CrossMapValues targetToCol; // CCM CrossMap() thread results + + // Project() vectors to populate projection DataFrame in FormatData() + // JP Can we do away with these and write directly to projection (+Tp)? + std::valarray< double > predictions; + std::valarray< double > const_predictions; + std::valarray< double > variance; + + // Prediction row accounting of library neighbor ties + bool anyTies; + std::vector< bool > ties; // true/false each prediction row + std::vector< std::vector< std::pair< double, size_t > > > tiePairs; + + std::valarray< double > target; // JP Can we do away with this? + + Parameters parameters; + + // Constructor declaration + EDM ( DataFrame< double > & data, Parameters & parameters ); + + // Method declarations + void CheckDataRows( std::string call ); + void PrepareEmbedding( bool checkDataRows = true ); + void Distances(); + void EmbedData(); + void FindNeighbors(); + void Project(); + void FormatOutput(); + void FillTimes( std::vector< std::string > & timeOut ); + + void PrintLibPred(); // ifdef DEBUG_ALL + void PrintNeighbors(); // ifdef DEBUG_ALL +}; +#endif diff --git a/src/EDM_Formatting.cc b/src/EDM_Formatting.cc new file mode 100644 index 0000000..b2404de --- /dev/null +++ b/src/EDM_Formatting.cc @@ -0,0 +1,347 @@ + +#include "EDM.h" +#include "DateTime.h" + +//---------------------------------------------------------- +// Validate dataFrameIn rows against lib and pred indices +//---------------------------------------------------------- +void EDM::CheckDataRows( std::string call ) +{ + // parameters.prediction & library have been zero-offset in Validate() + // to convert from user specified data row to array indicies + size_t prediction_max_i = + parameters.prediction[ parameters.prediction.size() - 1 ]; + + size_t library_max_i = + parameters.library[ parameters.library.size() - 1 ]; + + size_t shift; + if ( parameters.embedded ) { + shift = 0; + } + else { + if ( parameters.E < 1 ) { + std::stringstream errMsg; + errMsg << "CheckDataRows(): E = " << parameters.E + << " is invalid.\n" ; + throw std::runtime_error( errMsg.str() ); + } + + shift = abs( parameters.tau ) * ( parameters.E - 1 ); + } + + if ( data.NRows() <= prediction_max_i ) { + std::stringstream errMsg; + errMsg << "CheckDataRows(): " << call + << ": The prediction index " + << prediction_max_i + 1 + << " exceeds the number of data rows " + << data.NRows(); + throw std::runtime_error( errMsg.str() ); + } + + if ( data.NRows() <= library_max_i + shift ) { + std::stringstream errMsg; + errMsg << "CheckDataRows(): " << call + << ": The library index " << library_max_i + 1 + << " + tau(E-1) " << shift << " = " + << library_max_i + 1 + shift + << " exceeds the number of data rows " + << data.NRows(); + throw std::runtime_error( errMsg.str() ); + } +} + +//---------------------------------------------------------- +// Common code for Simplex and Smap output generation +//---------------------------------------------------------- +void EDM::FormatOutput() { + //---------------------------------------------------- + // TimeOut vector with additional Tp points + //---------------------------------------------------- + size_t N_time = data.Time().size(); + size_t N_row = parameters.prediction.size(); + size_t Tp_magnitude = abs( parameters.Tp ); + + std::vector< std::string > timeOut( N_row + Tp_magnitude ); + + // Populate timeOut vector with strings for output + if ( N_time ) { + FillTimes( std::ref( timeOut ) ); + } + + //---------------------------------------------------- + // Observations: Insert data; add Tp nan at end/start + //---------------------------------------------------- + std::valarray< double > observations( N_row + Tp_magnitude ); + + if ( parameters.Tp > -1 ) { // Positive Tp --------------------------- + std::slice pred_i = std::slice( parameters.prediction[0], N_row, 1 ); + + observations[ std::slice( 0, N_row, 1 ) ] = + ( std::valarray< double > ) target[ pred_i ]; + + for ( size_t i = N_row; i < N_row + parameters.Tp; i++ ) { + observations[ i ] = NAN; // assign nan at end + } + } + else { // Negative Tp ------------------------------------------- + std::slice pred_i; + + if ( parameters.prediction[0] >= Tp_magnitude ) { + pred_i = std::slice( parameters.prediction[ 0 ] - Tp_magnitude, + N_row + Tp_magnitude, 1 ); + + observations[ std::slice( 0, N_row + Tp_magnitude, 1 ) ] = + ( std::valarray< double > ) target[ pred_i ]; + } + else { + // Edge case where -Tp preceeds available record pred + pred_i = std::slice( 0, N_row + Tp_magnitude, 1 ); + + observations[std::slice( Tp_magnitude, N_row, 1 )] = + ( std::valarray< double > ) target[ pred_i ]; + + for ( size_t i = 0; i < Tp_magnitude; i++ ) { + observations[ i ] = NAN; // assign nan at start + } + } + } + + //------------------------------------------------------------------ + // Predictions & variance: Assign values; insert Tp nan at start/end + //------------------------------------------------------------------ + std::valarray< double > predictionsOut ( N_row + Tp_magnitude ); + std::valarray< double > constPredictionsOut( N_row + Tp_magnitude ); + std::valarray< double > varianceOut ( N_row + Tp_magnitude ); + + if ( parameters.Tp > -1 ) { // Positive Tp --------------------------- + std::slice predOut_i = std::slice( parameters.Tp, N_row, 1 ); + + for ( int i = 0; i < parameters.Tp; i++ ) { + predictionsOut[ i ] = NAN; // assign nan at start + varianceOut [ i ] = NAN; // assign nan at start + } + predictionsOut[ predOut_i ] = predictions; + varianceOut [ predOut_i ] = variance; + + if ( parameters.const_predict ) { + for ( int i = 0; i < parameters.Tp; i++ ) { + constPredictionsOut[ i ] = NAN; // assign nan at start + } + constPredictionsOut[ predOut_i ] = const_predictions; + } + } + else { // Negative Tp -------------------------------------------- + std::slice predOut_i = std::slice( 0, N_row - Tp_magnitude, 1 ); + std::slice predIn_i = std::slice( 0, N_row, 1 ); + + predictionsOut[ predOut_i ] = predictions[ predIn_i ]; + varianceOut [ predOut_i ] = variance [ predIn_i ]; + + for ( size_t i = N_row; i < N_row + Tp_magnitude; i++ ) { + predictionsOut[ i ] = NAN; // assign nan at end + varianceOut [ i ] = NAN; // assign nan at end + } + + if ( parameters.const_predict ) { + constPredictionsOut[ predOut_i ] = const_predictions[ predIn_i ]; + + for ( size_t i = N_row; i < N_row + Tp_magnitude; i++ ) { + constPredictionsOut[ i ] = NAN; // assign nan at end + } + } + } + + //---------------------------------------------------- + // Output DataFrame + //---------------------------------------------------- + size_t dataFrameColumms = parameters.const_predict ? 4 : 3; + + projection = DataFrame< double >( N_row + Tp_magnitude, dataFrameColumms ); + + if ( parameters.const_predict ) { + projection.ColumnNames() = { "Observations", "Predictions", + "Pred_Variance", "Const_Predictions" }; + } + else { + projection.ColumnNames()={"Observations","Predictions","Pred_Variance"}; + } + + if ( N_time ) { + projection.TimeName() = data.TimeName(); + projection.Time() = timeOut; + } + + projection.WriteColumn( 0, observations ); + projection.WriteColumn( 1, predictionsOut ); + projection.WriteColumn( 2, varianceOut ); + + if ( parameters.const_predict ) { + projection.WriteColumn( 3, constPredictionsOut ); + } + +#ifdef DEBUG_ALL + std::cout << "EDM::FormatOutput() time " << timeOut.size() + << " pred " << predictionsOut.size() + << " obs " << observations.size() << std::endl; + std::cout << "FormatOutput() projection -------------------" << std::endl; + std::cout << projection; +#endif +} + +//---------------------------------------------------------- +// Copy strings of time values into timeOut. +// If prediction times exceed times from the data, +// create new entries for the additional times. +//---------------------------------------------------------- +void EDM::FillTimes( std::vector< std::string > & timeOut ) +{ + size_t N_time = data.Time().size(); + size_t N_row = parameters.prediction.size(); + size_t max_pred_i = parameters.prediction[ N_row - 1 ]; + size_t min_pred_i = parameters.prediction[ 0 ]; + size_t Tp_magnitude = abs( parameters.Tp ); + + if ( max_pred_i >= N_time ) { + // If tau > 0 end rows were deleted. max_pred_i might exceed time bounds + max_pred_i = N_time - 1; + } + + if ( timeOut.size() != N_row + Tp_magnitude ) { + std::stringstream errMsg; + errMsg << "FillTimes(): timeOut vector length " << timeOut.size() + << " is not equal to the number of predictions + Tp " + << N_row + Tp_magnitude << std::endl; + throw std::runtime_error( errMsg.str() ); + } + + // Positive Tp ----------------------------------------------------- + if ( parameters.Tp > -1 ) { + // Fill in times guaranteed to be in parameters.prediction indices + for ( size_t i = 0; i < N_row; i++ ) { + size_t pred_i = parameters.prediction[ i ]; + if ( pred_i < N_time ) { + timeOut[ i ] = data.Time()[ pred_i ]; + } + } + + // Now fill in times beyond parameters.prediction indices + if ( max_pred_i + parameters.Tp < N_time ) { + // All prediction times are available in time, get the rest + for ( int i = 0; i < parameters.Tp; i++ ) { + timeOut[ N_row + i ] = data.Time()[ max_pred_i + i + 1 ]; + } + } + else { + // Tp introduces time values beyond the range of time + bool timeFormatWarningPrinted = false; + + // Try to parse the last time vector string as a date or datetime + // if dtinfo.unrecognized_fmt = true; it is not a date or datetime + datetime_info dtinfo = ParseDatetime( data.Time()[ max_pred_i ] ); + + for ( int i = 0; i < parameters.Tp; i++ ) { + std::stringstream tss; + + if ( dtinfo.unrecognized_fmt ) { + // Numeric so add Tp + tss << std::stod( data.Time()[ max_pred_i ] ) + i + 1; + } + else { + int time_delta = i + 1; + // Last two datetimes to compute time diff to add time delta + std::string time_new( data.Time()[ max_pred_i ] ); + std::string time_old( data.Time()[ max_pred_i - 1 ] ); + std::string new_time = + IncrementDatetime( time_old, time_new, time_delta ); + + // Add +ti if not recognized format(datetime util returns "") + if ( new_time.size() ) { + tss << new_time; + } + else { + tss << data.Time()[ max_pred_i ] << " +" << i + 1; + + if ( not timeFormatWarningPrinted ) { + std::cout << "FillTimes(): " + << "time column unrecognized time format." + << "\n\tManually adding + tp to the last" + << " time column available." << std::endl; + timeFormatWarningPrinted = true; + } + } + } + + timeOut[ N_row + i ] = tss.str(); + } + } + } + // Negative Tp ----------------------------------------------------- + else { + // Fill in times guaranteed to be in parameters.prediction indices + for ( size_t i = 0; i < N_row; i++ ) { + size_t pred_i = parameters.prediction[ i ]; + if ( pred_i < N_time ) { + // parameters.Tp is negative, start at timeOut[0 - parameters.Tp] + // timeOut is shifted forward to accomodate the preceeding Tp + timeOut[ i + Tp_magnitude ] = data.Time()[ pred_i ]; + } + } + + // Now fill in times before parameters.prediction indices + if ( (int) min_pred_i + parameters.Tp >= 0 ) { + // All prediction times are available in time, get the rest + for ( size_t i = 0; i < Tp_magnitude; i++ ) { + timeOut[ i ] = data.Time()[ parameters.prediction[ i ] - + Tp_magnitude ]; + } + } + else { + // Tp introduces time values before the range of time + bool timeFormatWarningPrinted = false; + + // Try to parse the first time vector string as a date or datetime + // if dtinfo.unrecognized_fmt = true; it is not a date or datetime + datetime_info dtinfo = ParseDatetime( data.Time()[ 0 ] ); + + for ( size_t i = 0; i < Tp_magnitude; i++ ) { + std::stringstream tss; + + if ( dtinfo.unrecognized_fmt ) { + // Numeric so subtract i Tp + tss << std::stod( data.Time()[Tp_magnitude - 1] ) - (i + 1); + } + else { + int time_delta = i - 1; + // Get first two datetimes to compute time diff + // to add time delta + std::string time_new( data.Time()[ 1 ] ); + std::string time_old( data.Time()[ 0 ] ); + std::string new_time = + IncrementDatetime( time_old, time_new, time_delta ); + + // Subtract +ti if not a recognized format + // (datetime util returns "") + if ( new_time.size() ) { + tss << new_time; + } + else { + tss << data.Time()[ max_pred_i ] << " -" << i + 1; + + if ( not timeFormatWarningPrinted ) { + std::cout << "FillTimes(): " + << "time column unrecognized time format." + << "\n\tManually adding - tp to the first" + << " time column available." << std::endl; + timeFormatWarningPrinted = true; + } + } + } // else not dtinfo.unrecognized_fmt + + timeOut[ i ] = tss.str(); + + } // for ( size_t i = 0; i < Tp_magnitude; i++ ) + } // else Tp introduces time values before the range of time + } // else Negative Tp ------------------------------------------------ +} diff --git a/src/EDM_Neighbors.cc b/src/EDM_Neighbors.cc new file mode 100644 index 0000000..359bb8e --- /dev/null +++ b/src/EDM_Neighbors.cc @@ -0,0 +1,469 @@ + +#include "EDM_Neighbors.h" + +namespace EDM_Neighbors_Lock { + std::mutex mtx; +} + +//---------------------------------------------------------------- +// Common code for Simplex and Smap: +// 0) CheckDataRows() +// 1) Extract or Embed() data into embedding +// 2) Get target (library) vector +// 3) DeletePartialDataRows() +// 4) Adjust parameters.library and parameters.prediction indices +// +// NOTE: time column is not returned in the embedding dataBlock. +// +// NOTE: If data is embedded by Embed(), the returned dataBlock +// has tau * (E-1) fewer rows than data. Since data is +// included in the returned DataEmbedNN struct, the first +// (or last) tau * (E-1) data rows are deleted to match +// dataBlock. The target vector is also reduced. +// +// NOTE: If rows are deleted, then the library and prediction +// vectors in Parameters are updated to reflect this. +// +//---------------------------------------------------------------- +void EDM::PrepareEmbedding( bool checkDataRows ) { + + if ( checkDataRows ) { + CheckDataRows( "PrepareEmbedding" ); + } + + // Embed + if ( parameters.embedded ) { + // dataIn is a multivariable block, no embedding needed + // Select the specified columns into embedding + if ( parameters.columnNames.size() ) { + embedding = data.DataFrameFromColumnNames( parameters.columnNames ); + } + else if ( parameters.columnIndex.size() ) { + embedding = data.DataFrameFromColumnIndex( parameters.columnIndex ); + } + else { + throw std::runtime_error( "PrepareEmbedding(): colNames and " + " colIndex are empty.\n" ); + } + } + else { + // embedded = false: Create the embedding dataBlock via EmbedData() + // dataBlock will have tau * (E-1) fewer rows than dataIn + EmbedData(); + } + + // Get target (library) vector + if ( parameters.targetIndex ) { + target = data.Column( parameters.targetIndex ); + } + else if ( parameters.targetName.size() ) { + target = data.VectorColumnName( parameters.targetName ); + } + else { + // Default to first column + target = data.Column( 0 ); + } + + //------------------------------------------------------------ + // embedded = false: Embed() was called on dataIn + // Remove target, data rows as needed + // Adjust parameters.library and parameters.prediction indices + //------------------------------------------------------------ + if ( not parameters.embedded ) { + + if ( parameters.E < 1 ) { + std::stringstream errMsg; + errMsg << "EmbedNN(): E = " << parameters.E << " is invalid.\n" ; + throw std::runtime_error( errMsg.str() ); + } + + size_t shift = abs( parameters.tau ) * ( parameters.E - 1 ); + + // Copy targetIn excluding partial data into targetEmbed + std::valarray< double > targetEmbed( data.NRows() - shift ); + + // Bogus cast to ( std::valarray ) for MSVC + // as it doesn't export its own slice_array applied to [] + if ( parameters.tau < 0 ) { + targetEmbed = ( std::valarray< double > ) + target[ std::slice( shift, target.size() - shift, 1 ) ]; + } + else { + targetEmbed = ( std::valarray< double > ) + target[ std::slice( 0, target.size() - shift, 1 ) ]; + } + + // Resize target to ignore partial data rows + target.resize( targetEmbed.size() ); + + // Copy target without partial data into resized targetIn + std::slice targetEmbed_i = std::slice( 0, targetEmbed.size(), 1 ); + target[ targetEmbed_i ] = ( std::valarray< double > ) + targetEmbed[ targetEmbed_i ]; + + // Delete dataIn top or bottom rows of partial data + if ( not data.PartialDataRowsDeleted() ) { + // Not thread safe + std::lock_guard lck( EDM_Neighbors_Lock::mtx ); + + data.DeletePartialDataRows( shift, parameters.tau ); + } + + // Adjust parameters.library and parameters.prediction vectors of indices + if ( shift > 0 ) { + parameters.DeleteLibPred(); + } + + // Check boundaries again since rows were removed + if ( checkDataRows ) { + CheckDataRows( "PrepareEmbedding: Embedded data" ); + } + } +} + +//---------------------------------------------------------------- +// Assumed that EDM::Distances() has been called. +// +// Writes to EDM object: +// knn_distances : sorted knn distances +// knn_neighbors : library neighbor rows of knn_distances +// ties : pred row vector of bool, true if tie +// tiePairs : pred row vector of < distance, libRow > pairs +//---------------------------------------------------------------- +void EDM::FindNeighbors() { + +#ifdef DEBUG_ALL + PrintDataFrameIn(); +#endif + + if ( not parameters.validated ) { + std::string errMsg( "FindNeighbors(): Parameters not validated." ); + throw( std::runtime_error( errMsg ) ); + } + + if ( parameters.embedded and parameters.E > (int) embedding.NColumns() ) { + std::stringstream errMsg; + errMsg << "WARNING: FindNeighbors() Multivariate data " + << "(embedded = true): The number of embedding columns (" + << embedding.NColumns() << ") is less than the embedding " + << "dimension E (" << parameters.E << ")\n"; + std::cout << errMsg.str(); + } + + size_t N_library_rows = parameters.library.size(); + size_t N_prediction_rows = parameters.prediction.size(); + + auto max_lib_it = std::max_element( parameters.library.begin(), + parameters.library.end() ); + int max_lib_index = *max_lib_it; + + if ( parameters.verbose and parameters.method != Method::CCM ) { + // Identify degenerate library : prediction points by + // set_intersection() of lib & pred indices, needs a result vector + std::vector< double > result( N_library_rows + N_prediction_rows, 0 ); + + std::vector< double >::iterator ii = set_intersection ( + parameters.prediction.begin(), parameters.prediction.end(), + parameters.library.begin(), parameters.library.end(), + result.begin() ); + + if ( ii != result.begin() ) { + // Overlapping indices exist + std::stringstream msg; + msg << "WARNING: FindNeighbors(): Degenerate library and " + << " prediction data found. Overlap indices: "; + for ( auto ri = result.begin(); ri != ii; ++ri ) { + msg << *ri << " "; + } msg << std::endl; + std::cout << msg.str(); + } + } + + // allLibrows are the lib row indices, 1 row x lib columns + std::valarray< size_t > rowLib = allLibRows.Row( 0 ); + + // Pair the distances and library row indices for sort on distance + // Each predPairs element correponds to a prediction row and + // holds a vector of < distance, lib_row > pairs for each lib_row + std::vector< std::vector< std::pair< double, size_t > > > + predPairs( N_prediction_rows ); + + for ( size_t pred_row = 0; pred_row < N_prediction_rows; pred_row++ ) { + std::valarray< double > rowDist = allDistances.Row( pred_row ); + + std::vector< std::pair< double, size_t > > rowPairs( rowDist.size() ); + + for ( size_t i = 0; i < rowDist.size(); i++ ) { + rowPairs[ i ] = std::make_pair( rowDist[ i ], rowLib[ i ] ); + } + // insert into predPairs + predPairs[ pred_row ] = rowPairs; + } + +#ifdef DEBUG_ALL + std::cout << allLibRows; + std::cout << allDistances; + for ( size_t pred_row = 0; pred_row < predPairs.size(); pred_row++ ) { + std::vector< std::pair > rowPair = predPairs[ pred_row ]; + for ( size_t i = 0; i < rowPair.size(); i++ ) { + std::pair thisPair = rowPair[ i ]; + std::cout << "[" << thisPair.first << ", " + << thisPair.second << "] "; + } std::cout << std::endl; + } std::cout << std::endl; +#endif + + // Allocate in EDM class object + // JP Put on heap & destructor, or use smart pointers + knn_neighbors = DataFrame< size_t >( N_prediction_rows, parameters.knn ); + knn_distances = DataFrame< double >( N_prediction_rows, parameters.knn ); + ties = std::vector< bool >( N_prediction_rows, false ); + tiePairs = std::vector< std::vector< std::pair< double, size_t > > > + ( N_prediction_rows ); + + //------------------------------------------------------------------- + // For each prediction vector (row in prediction DataFrame) find the + // list of library indices that are within k_NN points + //------------------------------------------------------------------- + for ( size_t pred_row = 0; pred_row < predPairs.size(); pred_row++ ) { + + // rowPair is a vector of pairs of length library rows + // Get the rowPair for this prediction row + std::vector< std::pair > rowPair = predPairs[ pred_row ]; + + // sort < distance, lib_row > pairs for this pred_row + // distance must be .first + std::sort( rowPair.begin(), rowPair.end(), DistanceCompare ); + + // Insert knn distance / library row index into knn vectors + std::valarray< double > knnDistances( parameters.knn ); + std::valarray< size_t > knnLibRows ( parameters.knn ); + + int lib_row_i = 0; + int k = 0; + while ( k < parameters.knn ) { + double distance = rowPair[ lib_row_i ].first; + int lib_row = rowPair[ lib_row_i ].second; + + if ( not parameters.noNeighborLimit ) { + // Reach exceeding grasp : forecast point is outside library + if ( lib_row + parameters.Tp > max_lib_index or + lib_row + parameters.Tp < 0 ) { + lib_row_i++; + continue; // keep looking + } + } + + // Exclusion radius: units are data rows, not time + if ( parameters.exclusionRadius ) { + int xrad = (int) lib_row - (int) pred_row; + if ( std::abs( xrad ) <= parameters.exclusionRadius ) { + lib_row_i++; + continue; // skip this neighbor + } + } + + knnDistances[ k ] = distance; + knnLibRows [ k ] = lib_row; + lib_row_i++; + k++; + } + + knn_distances.WriteRow( pred_row, knnDistances ); + knn_neighbors.WriteRow( pred_row, knnLibRows ); + + // Check for ties. 1.18e−38 is float 32-bit min + if ( k < (int) rowPair.size() ) { + if ( rowPair[ k ].first <= rowPair[ k-1 ].first ) { + // At least one tie... + std::vector< std::pair< double, size_t > > rowTiePairs; + + while( k < (int) rowPair.size() and rowPair[ k ].first > 0 and + rowPair[ k ].first <= rowPair[ k-1 ].first ) { + + // Set flag in ties and store tie pairs in tiePairs + ties[ pred_row ] = true; + + rowTiePairs.push_back(std::make_pair( rowPair[ k ].first, + rowPair[ k ].second )); + k++; + } + + if ( find( ties.begin(), ties.end(), true ) != ties.end() ) { + anyTies = true; + tiePairs[ pred_row ] = rowTiePairs; + } + } + } + } // for ( pred_row = 0; pred_row < predPairs.size(); pred_row++ ) + +#ifdef DEBUG_ALL + for ( size_t i = 0; i < ties.size(); i++ ) { + if ( ties[ i ] ) { + std::vector< std::pair< double, size_t > > rowTiePairs = + tiePairs[ i ]; + std::cout << "Ties at pred_i " << i << ": "; + for ( size_t j = 0; j < rowTiePairs.size(); j++ ) { + double dist = rowTiePairs[ j ].first; + size_t prow = rowTiePairs[ j ].second; + std::cout << "[ " << dist << ", " << prow << "] "; + } std::cout << std::endl; + } + } + PrintNeighbors(); +#endif +} + +//--------------------------------------------------------------------- +// Compute all prediction row : library row distances. +// Note that embedding does NOT have the time in column 0. +// +// Writes EDM objects: +// allDistances: pred rows x lib columns matrix with distances. +// distance(i,j) is distance between the E-dimensional +// phase space point prediction row i and library row j. +// allLibRows : 1 row x lib cols matrix with lib rows +//--------------------------------------------------------------------- +void EDM::Distances () { + + // Validate library and prediction rows are in embedding + auto max_it = std::max_element( parameters.prediction.begin(), + parameters.prediction.end() ); + size_t maxPredIndex = (size_t) *max_it; + + max_it = std::max_element( parameters.library.begin(), + parameters.library.end() ); + size_t maxLibIndex = (size_t) *max_it; + + if ( maxPredIndex >= embedding.NRows() or + maxLibIndex >= embedding.NRows() ) { + std::stringstream errMsg; + errMsg << "Distances() library or prediction index exceeds embedding " + << "rows: " << embedding.NRows(); + throw std::runtime_error( errMsg.str() ); + } + + size_t Npred = parameters.prediction.size(); + size_t Nlib = parameters.library.size(); + + // Allocate output distance matrix and libRows list in EDM object + allDistances = DataFrame< double >( Npred, Nlib ); + allLibRows = DataFrame< size_t >( 1, Nlib ); + + // Initialise D to DistanceMax + std::valarray< double > row_init( EDM_Distance::DistanceMax, Nlib ); + for ( size_t row = 0; row < Npred; row++ ) { + allDistances.WriteRow( row, row_init ); + } + + // Set lib indices into allLibRows + for ( size_t col = 0; col < Nlib; col++ ) { + allLibRows( 0, col ) = parameters.library[ col ]; + } + + // Compute all prediction row : library row distances + for ( size_t predRow = 0; predRow < Npred; predRow++ ) { + + size_t predictionRow = parameters.prediction[ predRow ]; + + // Get E-dimensional vector from this prediction row + std::valarray< double > v1 = embedding.Row( predictionRow ); + + for ( size_t libRow = 0; libRow < Nlib; libRow++ ) { + + if ( predictionRow == parameters.library[ libRow ] ) { + continue; // degenerate pred & lib + } + + // Find distance between vector (v1) and library vector v2 + std::valarray< double > v2 = + embedding.Row( parameters.library[ libRow ] ); + + allDistances( predRow, libRow ) = + Distance( v1, v2, DistanceMetric::Euclidean ); + } + } +} + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +double Distance( const std::valarray< double > & v1, + const std::valarray< double > & v2, + DistanceMetric metric ) +{ + double distance = 0; + + // For efficiency sake, we forego the usual validation of v1 & v2. + + if ( metric == DistanceMetric::Euclidean ) { + double sum = 0; + double delta = 0; + for ( size_t i = 0; i < v1.size(); i++ ) { + delta = v2[i] - v1[i]; + sum += delta * delta; // avoid call to pow() + } + distance = sqrt( sum ); + + // Note: this implicit implementation is slower + // std::valarray delta = v2 - v1; + // distance = sqrt( (delta * delta).sum() ); + } + else if ( metric == DistanceMetric::Manhattan ) { + double sum = 0; + for ( size_t i = 0; i < v1.size(); i++ ) { + sum += abs( v2[i] - v1[i] ); + } + distance = sum; + } + else { + std::stringstream errMsg; + errMsg << "Distance() Invalid DistanceMetric: " + << static_cast( metric ); + throw std::runtime_error( errMsg.str() ); + } + + return distance; +} + +#ifdef DEBUG_ALL +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +void EDM::PrintDataFrameIn() +{ + std::cout << "FindNeighbors(): library:" << std::endl; + for ( size_t row = 0; row < parameters.library.size(); row++ ) { + size_t row_i = parameters.library[row]; + std::cout << "row " << row_i << " : "; + for ( size_t col = 0; col < data.NColumns(); col++ ) { + std::cout << data(row_i,col) << " "; + } std::cout << std::endl; + } + std::cout << "FindNeighbors(): prediction:" << std::endl; + for ( size_t row = 0; row < parameters.prediction.size(); row++ ) { + size_t row_i = parameters.prediction[row]; + std::cout << "row " << row_i << " : "; + for ( size_t col = 0; col < data.NColumns(); col++ ) { + std::cout << data(row_i,col) << " "; + } std::cout << std::endl; + } +} + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +void EDM::PrintNeighbors() +{ + std::cout << "EDM::FindNeighbors(): neighbors:distances" << std::endl; + for ( size_t i = 0; i < knn_neighbors.NRows(); i++ ) { + std::cout << "Row " << i << " | "; + for ( size_t j = 0; j < knn_neighbors.NColumns(); j++ ) { + std::cout << knn_neighbors( i, j ) << " "; + } std::cout << " : "; + for ( size_t j = 0; j < knn_neighbors.NColumns(); j++ ) { + std::cout << knn_distances( i, j ) << " "; + } std::cout << std::endl; + } +} +#endif diff --git a/src/EDM_Neighbors.h b/src/EDM_Neighbors.h new file mode 100644 index 0000000..5223a87 --- /dev/null +++ b/src/EDM_Neighbors.h @@ -0,0 +1,19 @@ +#ifndef EDM_NEIGHBORS_H +#define EDM_NEIGHBORS_H + +#include "EDM.h" + +namespace EDM_Distance { + // Define the initial maximum distance for neigbors + // DBL_MAX is a Macro equivalent to: std::numeric_limits::max() + double DistanceMax = std::numeric_limits::max(); +} + +// Prototypes +double Distance( const std::valarray &v1, + const std::valarray &v2, + DistanceMetric metric ); + +bool DistanceCompare( const std::pair &x, + const std::pair &y ); +#endif diff --git a/src/Eval.cc b/src/Eval.cc index d974463..5a6c9aa 100644 --- a/src/Eval.cc +++ b/src/Eval.cc @@ -4,7 +4,7 @@ #include #include -#include "Common.h" +#include "API.h" namespace EDM_Eval { // Thread Work Queue : Vector of int @@ -80,26 +80,25 @@ void SMapThread( EDM_Eval::WorkQueue &workQ, // API Overload 1: Explicit data file path/name // Implemented as a wrapper to API Overload 2: //---------------------------------------------------------------- -DataFrame EmbedDimension( std::string pathIn, - std::string dataFile, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int maxE, - int Tp, - int tau, - std::string colNames, - std::string targetName, - bool embedded, - bool verbose, - unsigned nThreads ) { +DataFrame< double > EmbedDimension( std::string pathIn, + std::string dataFile, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int maxE, + int Tp, + int tau, + std::string colNames, + std::string targetName, + bool embedded, + bool verbose, + unsigned nThreads ) { // Create DataFrame (constructor loads data) - DataFrame< double > *dataFrameIn = - new DataFrame< double > ( pathIn, dataFile ); - - DataFrame E_rho = EmbedDimension( std::ref( *dataFrameIn ), + DataFrame< double > dataFrameIn ( pathIn, dataFile ); + + DataFrame E_rho = EmbedDimension( std::ref( dataFrameIn ), pathOut, predictFile, lib, @@ -112,9 +111,6 @@ DataFrame EmbedDimension( std::string pathIn, embedded, verbose, nThreads ); - - delete dataFrameIn; - return E_rho; } @@ -122,22 +118,22 @@ DataFrame EmbedDimension( std::string pathIn, // EmbedDimension() : Evaluate Simplex rho vs. dimension E // API Overload 2: DataFrame provided //---------------------------------------------------------------- -DataFrame EmbedDimension( DataFrame< double > &data, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int maxE, - int Tp, - int tau, - std::string colNames, - std::string targetName, - bool embedded, - bool verbose, - unsigned nThreads ) { - +DataFrame< double > EmbedDimension( DataFrame< double > & data, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int maxE, + int Tp, + int tau, + std::string colNames, + std::string targetName, + bool embedded, + bool verbose, + unsigned nThreads ) { + // Container for results - DataFrame E_rho( maxE, 2, "E rho" ); + DataFrame< double > E_rho( maxE, 2, "E rho" ); // Build work queue EDM_Eval::WorkQueue workQ( maxE ); @@ -149,12 +145,12 @@ DataFrame EmbedDimension( DataFrame< double > &data, unsigned maxThreads = std::thread::hardware_concurrency(); if ( maxThreads < nThreads ) { nThreads = maxThreads; } - if ( nThreads > maxE ) { nThreads = maxE; } - + if ( (int) nThreads > maxE ) { nThreads = maxE; } + // thread container std::vector< std::thread > threads; for ( unsigned i = 0; i < nThreads; i++ ) { - + threads.push_back( std::thread( EmbedThread, std::ref( workQ ), std::ref( data ), @@ -168,12 +164,12 @@ DataFrame EmbedDimension( DataFrame< double > &data, embedded, verbose ) ); } - + // join threads for ( auto &thrd : threads ) { thrd.join(); } - + // If thread threw exception, get from queue and rethrow if ( not EDM_Eval::embedDimExceptQ.empty() ) { std::lock_guard lck( EDM_Eval::q_mtx ); @@ -183,44 +179,42 @@ DataFrame EmbedDimension( DataFrame< double > &data, // Unroll all other exception from the thread/loops while( not EDM_Eval::embedDimExceptQ.empty() ) { - // JP When do these exception_ptr get deleted? Is it a leak? EDM_Eval::embedDimExceptQ.pop(); } std::rethrow_exception( exceptionPtr ); } - + if ( predictFile.size() ) { E_rho.WriteData( pathOut, predictFile ); } - + return E_rho; } //---------------------------------------------------------------- // Worker thread for EmbedDimension() //---------------------------------------------------------------- -void EmbedThread( EDM_Eval::WorkQueue &workQ, - DataFrame< double > &data, - DataFrame< double > &E_rho, - std::string lib, - std::string pred, - int Tp, - int tau, - std::string colNames, - std::string targetName, - bool embedded, - bool verbose ) +void EmbedThread( EDM_Eval::WorkQueue & workQ, + DataFrame< double > & data, + DataFrame< double > & E_rho, + std::string lib, + std::string pred, + int Tp, + int tau, + std::string colNames, + std::string targetName, + bool embedded, + bool verbose ) { - std::size_t i = std::atomic_fetch_add( &EDM_Eval::embed_count_i, std::size_t(1) ); - + while( i < workQ.size() ) { - + // WorkQueue stores E int E = workQ[ i ]; - // Simplex() -> EmbedNN() -> Embed() -> DeletePartialDataRows() + // Simplex() -> Embed() -> DeletePartialDataRows() // In a multthreaded application we need to pass a unique copy of // the data so that DeletePartialDataRows() is not recursively // applied to the same data frame. @@ -242,12 +236,12 @@ void EmbedThread( EDM_Eval::WorkQueue &workQ, embedded, false, // const_predict verbose ); - + VectorError ve = ComputeError( S.VectorColumnName("Observations"), S.VectorColumnName("Predictions")); E_rho.WriteRow( i, std::valarray({ (double) E, ve.rho })); - + if ( verbose ) { std::lock_guard lck( EDM_Eval::mtx ); std::cout << "EmbedThread() workQ[" << workQ[i] << "] E " << E @@ -260,10 +254,10 @@ void EmbedThread( EDM_Eval::WorkQueue &workQ, std::lock_guard lck( EDM_Eval::q_mtx ); EDM_Eval::embedDimExceptQ.push( std::current_exception() ); } - + i = std::atomic_fetch_add(&EDM_Eval::embed_count_i, std::size_t(1)); } - + // Reset counter std::atomic_store( &EDM_Eval::embed_count_i, std::size_t(0) ); } @@ -273,39 +267,37 @@ void EmbedThread( EDM_Eval::WorkQueue &workQ, // API Overload 1: Explicit data file path/name // Implemented as a wrapper to API Overload 2: //----------------------------------------------------------------- -DataFrame PredictInterval( std::string pathIn, - std::string dataFile, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int maxTp, - int E, - int tau, - std::string colNames, - std::string targetName, - bool embedded, - bool verbose, - unsigned nThreads ) { - +DataFrame< double > PredictInterval( std::string pathIn, + std::string dataFile, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int maxTp, + int E, + int tau, + std::string colNames, + std::string targetName, + bool embedded, + bool verbose, + unsigned nThreads ) { + // Create DataFrame (constructor loads data) - DataFrame< double > *dataFrameIn = - new DataFrame< double > ( pathIn, dataFile ); - - DataFrame Tp_rho = PredictInterval( std::ref( *dataFrameIn ), - pathOut, - predictFile, - lib, - pred, - maxTp, - E, - tau, - colNames, - targetName, - embedded, - verbose ); - delete dataFrameIn; - + DataFrame< double > dataFrameIn( pathIn, dataFile ); + + DataFrame< double > Tp_rho = PredictInterval( std::ref( dataFrameIn ), + pathOut, + predictFile, + lib, + pred, + maxTp, + E, + tau, + colNames, + targetName, + embedded, + verbose, + nThreads ); return Tp_rho; } @@ -313,22 +305,22 @@ DataFrame PredictInterval( std::string pathIn, // PredictInterval() : Evaluate Simplex rho vs. predict interval Tp // API Overload 2: DataFrame provided //----------------------------------------------------------------- -DataFrame PredictInterval( DataFrame< double > &data, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int maxTp, - int E, - int tau, - std::string colNames, - std::string targetName, - bool embedded, - bool verbose, - unsigned nThreads ) { - +DataFrame< double > PredictInterval( DataFrame< double > & data, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + int maxTp, + int E, + int tau, + std::string colNames, + std::string targetName, + bool embedded, + bool verbose, + unsigned nThreads ) { + // Container for results - DataFrame Tp_rho( maxTp, 2, "Tp rho" ); + DataFrame< double > Tp_rho( maxTp, 2, "Tp rho" ); // Build work queue EDM_Eval::WorkQueue workQ( maxTp ); @@ -339,9 +331,9 @@ DataFrame PredictInterval( DataFrame< double > &data, } unsigned maxThreads = std::thread::hardware_concurrency(); - if ( maxThreads < nThreads ) { nThreads = maxThreads; } - if ( nThreads > maxTp ) { nThreads = maxTp; } - + if ( maxThreads < nThreads ) { nThreads = maxThreads; } + if ( (int) nThreads > maxTp ) { nThreads = maxTp; } + // thread container std::vector< std::thread > threads; for ( unsigned i = 0; i < nThreads; ++i ) { @@ -358,12 +350,12 @@ DataFrame PredictInterval( DataFrame< double > &data, embedded, verbose ) ); } - + // join threads for ( auto &thrd : threads ) { thrd.join(); } - + // If thread threw exception, get from queue and rethrow if ( not EDM_Eval::predictIntExceptQ.empty() ) { std::lock_guard lck( EDM_Eval::q_mtx ); @@ -373,12 +365,11 @@ DataFrame PredictInterval( DataFrame< double > &data, // Unroll all other exception from the thread/loops while( not EDM_Eval::predictIntExceptQ.empty() ) { - // JP When do these exception_ptr get deleted? Is it a leak? EDM_Eval::predictIntExceptQ.pop(); } std::rethrow_exception( exceptionPtr ); } - + if ( predictFile.size() ) { Tp_rho.WriteData( pathOut, predictFile ); } @@ -403,39 +394,39 @@ void PredictIntervalThread( EDM_Eval::WorkQueue &workQ, { std::size_t i = std::atomic_fetch_add( &EDM_Eval::tp_count_i, std::size_t(1) ); - + while( i < workQ.size() ) { - + // WorkQueue stores Tp int Tp = workQ[ i ]; - + // In a multthreaded application we need to pass a unique copy of // the data so that DeletePartialDataRows() is not recursively // applied to the same data frame. DataFrame< double > localData( data ); try { - DataFrame S = Simplex( std::ref( localData ), - "", // pathOut, - "", // predictFile, - lib, - pred, - E, - Tp, - 0, // knn - tau, - 0, // exclusionRadius - colNames, - targetName, - embedded, - false, // const_pred - verbose ); - + DataFrame< double > S = Simplex( std::ref( localData ), + "", // pathOut, + "", // predictFile, + lib, + pred, + E, + Tp, + 0, // knn + tau, + 0, // exclusionRadius + colNames, + targetName, + embedded, + false, // const_pred + verbose ); + VectorError ve = ComputeError( S.VectorColumnName("Observations"), S.VectorColumnName("Predictions")); Tp_rho.WriteRow( i, std::valarray({ (double) Tp, ve.rho })); - + if ( verbose ) { std::lock_guard lck( EDM_Eval::mtx ); std::cout << "PredictIntervalThread() workQ[" << workQ[i] @@ -449,10 +440,10 @@ void PredictIntervalThread( EDM_Eval::WorkQueue &workQ, std::lock_guard lck( EDM_Eval::q_mtx ); EDM_Eval::predictIntExceptQ.push( std::current_exception() ); } - + i = std::atomic_fetch_add( &EDM_Eval::tp_count_i, std::size_t(1) ); } - + // Reset counter std::atomic_store( &EDM_Eval::tp_count_i, std::size_t(0) ); } @@ -462,28 +453,27 @@ void PredictIntervalThread( EDM_Eval::WorkQueue &workQ, // API Overload 1: Explicit data file path/name // Implemented as a wrapper to API Overload 2: //---------------------------------------------------------------- -DataFrame PredictNonlinear( std::string pathIn, - std::string dataFile, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - std::string theta, - int E, - int Tp, - int knn, - int tau, - std::string colNames, - std::string targetName, - bool embedded, - bool verbose, - unsigned nThreads ) { - +DataFrame< double > PredictNonlinear( std::string pathIn, + std::string dataFile, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + std::string theta, + int E, + int Tp, + int knn, + int tau, + std::string colNames, + std::string targetName, + bool embedded, + bool verbose, + unsigned nThreads ) { + // Create DataFrame (constructor loads data) - DataFrame< double > *dataFrameIn = - new DataFrame< double > ( pathIn, dataFile ); - - DataFrame< double > Theta_rho = PredictNonlinear( std::ref( *dataFrameIn ), + DataFrame< double > dataFrameIn( pathIn, dataFile ); + + DataFrame< double > Theta_rho = PredictNonlinear( std::ref( dataFrameIn ), pathOut, predictFile, lib, @@ -496,9 +486,8 @@ DataFrame PredictNonlinear( std::string pathIn, colNames, targetName, embedded, - verbose ); - delete dataFrameIn; - + verbose, + nThreads ); return Theta_rho; } @@ -506,59 +495,59 @@ DataFrame PredictNonlinear( std::string pathIn, // PredictNonlinear() : Smap rho vs. localisation parameter theta // API Overload 2: DataFrame provided //---------------------------------------------------------------- -DataFrame PredictNonlinear( DataFrame< double > &data, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - std::string theta, - int E, - int Tp, - int knn, - int tau, - std::string colNames, - std::string targetName, - bool embedded, - bool verbose, - unsigned nThreads ) { +DataFrame< double > PredictNonlinear( DataFrame< double > & data, + std::string pathOut, + std::string predictFile, + std::string lib, + std::string pred, + std::string theta, + int E, + int Tp, + int knn, + int tau, + std::string colNames, + std::string targetName, + bool embedded, + bool verbose, + unsigned nThreads ) { std::vector ThetaValues( { 0.01, 0.1, 0.3, 0.5, 0.75, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 9 } ); - if ( theta.size() ) { - // Use theta values passed in as parameter string - ThetaValues.clear(); - - std::vector theta_vec = SplitString( theta, " \t,\n" ); - - try { - for ( auto ci = theta_vec.begin(); ci != theta_vec.end(); ++ci ) { - ThetaValues.push_back( std::stod( *ci ) ); - } - } - catch ( const std::invalid_argument &ia ) { - std::stringstream errMsg; - errMsg << "PredictNonlinear(): Unable to convert theta [" - << ia.what() << "] to numeric."; - throw std::runtime_error( errMsg.str() ); - } - } - + if ( theta.size() ) { + // Use theta values passed in as parameter string + ThetaValues.clear(); + + std::vector< std::string > theta_vec = SplitString( theta, " \t,\n" ); + + try { + for ( auto ci = theta_vec.begin(); ci != theta_vec.end(); ++ci ) { + ThetaValues.push_back( std::stod( *ci ) ); + } + } + catch ( const std::invalid_argument &ia ) { + std::stringstream errMsg; + errMsg << "PredictNonlinear(): Unable to convert theta [" + << ia.what() << "] to numeric."; + throw std::runtime_error( errMsg.str() ); + } + } + // Container for results - DataFrame Theta_rho( ThetaValues.size(), 2, "Theta rho" ); + DataFrame< double > Theta_rho( ThetaValues.size(), 2, "Theta rho" ); // Build work queue EDM_Eval::WorkQueue workQ( ThetaValues.size() ); // Insert ThetaValues indexes into work queue - for ( auto i = 0; i < ThetaValues.size(); i++ ) { + for ( size_t i = 0; i < ThetaValues.size(); i++ ) { workQ[ i ] = i; } unsigned maxThreads = std::thread::hardware_concurrency(); if ( maxThreads < nThreads ) { nThreads = maxThreads; } if ( nThreads > ThetaValues.size() ) { nThreads = ThetaValues.size(); } - + // thread container std::vector< std::thread > threads; for ( unsigned i = 0; i < nThreads; ++i ) { @@ -578,7 +567,7 @@ DataFrame PredictNonlinear( DataFrame< double > &data, embedded, verbose ) ); } - + // join threads for ( auto &thrd : threads ) { thrd.join(); @@ -593,16 +582,15 @@ DataFrame PredictNonlinear( DataFrame< double > &data, // Unroll all other exception from the thread/loops while( not EDM_Eval::predictNLExceptQ.empty() ) { - // JP When do these exception_ptr get deleted? Is it a leak? EDM_Eval::predictNLExceptQ.pop(); } std::rethrow_exception( exceptionPtr ); } - + if ( predictFile.size() ) { Theta_rho.WriteData( pathOut, predictFile ); } - + return Theta_rho; } @@ -624,14 +612,13 @@ void SMapThread( EDM_Eval::WorkQueue &workQ, bool embedded, bool verbose ) { - std::size_t i = std::atomic_fetch_add( &EDM_Eval::smap_count_i, std::size_t(1) ); while( i < workQ.size() ) { - + double theta = ThetaValues[ workQ[ i ] ]; - + // In a multthreaded application we need to pass a unique copy of // the data so that DeletePartialDataRows() is not recursively // applied to the same data frame. @@ -656,16 +643,16 @@ void SMapThread( EDM_Eval::WorkQueue &workQ, embedded, false, // const_predict verbose ); - + DataFrame< double > predictions = S.predictions; DataFrame< double > coefficients = S.coefficients; - + VectorError ve = ComputeError( predictions.VectorColumnName( "Observations" ), predictions.VectorColumnName( "Predictions" ) ); - + Theta_rho.WriteRow( i, std::valarray({ theta, ve.rho })); - + if ( verbose ) { std::lock_guard lck( EDM_Eval::mtx ); std::cout << "Theta " << theta @@ -678,10 +665,10 @@ void SMapThread( EDM_Eval::WorkQueue &workQ, std::lock_guard lck( EDM_Eval::q_mtx ); EDM_Eval::predictNLExceptQ.push( std::current_exception() ); } - + i = std::atomic_fetch_add( &EDM_Eval::smap_count_i, std::size_t(1) ); } - + // Reset counter std::atomic_store( &EDM_Eval::smap_count_i, std::size_t(0) ); } diff --git a/src/Interface.cc b/src/Interface.cc deleted file mode 100644 index 17c2671..0000000 --- a/src/Interface.cc +++ /dev/null @@ -1,7 +0,0 @@ -#include "Common.h" - -//---------------------------------------------------------------- -// -//---------------------------------------------------------------- -void Interface() { -} diff --git a/src/Multiview.cc b/src/Multiview.cc index 5acd63d..18f3567 100644 --- a/src/Multiview.cc +++ b/src/Multiview.cc @@ -12,42 +12,39 @@ // Parameters.Validate() with Method::Simplex sets knn equal to E+1 // if knn not specified, so we need to explicitly set knn to D + 1. // -// multiview is the number of top-ranked D-dimensional predictions -// to "average" for the final prediction. Corresponds to parameter -// k in Ye & Sugihara with default k = sqrt(m) where m is the -// number of combinations C(n,D) available from the n = D * E +// multiviewEnsemble is the number of top-ranked D-dimensional +// predictions to "average" for the final prediction. Corresponds +// to parameter k in Ye & Sugihara with default k = sqrt(m) where +// m is the number of combinations C(n,D) available from the n = D * E // columns taken D at-a-time. // // Ye H., and G. Sugihara, 2016. Information leverage in // interconnected ecosystems: Overcoming the curse of dimensionality. // Science 353:922–925. //-------------------------------------------------------------------- -// +//-------------------------------------------------------------------- // NOTE: Multiview evaluates the top projections using in-sample // library predictions. It can be shown that highly accurate // in-sample predictions can be made from arbitrary non- // constant, non-oscillatory vectors. Therefore, some attention // may be warranted to filter prospective embedding vectors. +// The multiviewTrainLib flag disables this default behavior +// so that the top k evalutions are done using the specified +// lib and pred. //-------------------------------------------------------------------- -#include -#include -#include -#include - -#include "Common.h" -#include "AuxFunc.h" +#include "Multiview.h" namespace EDM_Multiview { // Thread Work Queue : Vector of combos indices typedef std::vector< int > WorkQueue; - + // Thread exception_ptr queue std::queue< std::exception_ptr > exceptionQ; - + // atomic counter for all threads std::atomic eval_i(0); // initialize to 0 - + std::mutex mtx; std::mutex q_mtx; } @@ -57,293 +54,133 @@ namespace EDM_Multiview { //---------------------------------------------------------------- std::vector< std::vector< size_t > > Combination( int n, int k ); -DataFrame SimplexProjection( Parameters param, - DataEmbedNN embedNN, - bool checkDataRows = true ); - -void EvalComboThread( Parameters param, +void EvalComboThread( MultiviewClass & MV, EDM_Multiview::WorkQueue workQ, - std::vector< std::vector< size_t > > combos, - DataFrame< double > &embedding, - std::valarray< double > &targetVec, - DataFrame< double > &combos_rho, - std::vector< DataFrame< double > > &prediction ); + std::vector< std::vector< size_t > >& combos, + DataFrame< double > & combosRho, + std::vector< DataFrame< double > > & comboPrediction ); -std::vector< std::string > ComboRhoTable( DataFrame combosRho, +std::vector< std::string > ComboRhoTable( DataFrame< double > combosRho, std::vector< std::string > colNames ); //---------------------------------------------------------------- -// Multiview() : Evaluate Simplex rho vs. dimension E -// API Overload 1: Explicit data file path/name -// Implemented as a wrapper to API Overload 2: +// Constructor +// data & parameters initialise EDM::SimplexClass parent, and, +// both mapping objects to the same, initial parameters. //---------------------------------------------------------------- -MultiviewValues Multiview( std::string pathIn, - std::string dataFile, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int D, - int E, - int Tp, - int knn, - int tau, - std::string columns, - std::string target, - int multiview, - int exclusionRadius, - bool trainLib, - bool verbose, - unsigned nThreads ) { - - // Create DataFrame (constructor loads data) - DataFrame< double > dataFrameIn( pathIn, dataFile ); - - MultiviewValues result = Multiview( dataFrameIn, - pathOut, - predictFile, - lib, - pred, - D, - E, - Tp, - knn, - tau, - columns, - target, - multiview, - exclusionRadius, - trainLib, - verbose, - nThreads ); - return result; -} +MultiviewClass::MultiviewClass ( + DataFrame< double > & data, + Parameters & parameters ) : + SimplexClass{ data, parameters }, // base class initialise + predictOutputFileIn( parameters.predictOutputFile ) +{} //---------------------------------------------------------------- -// Multiview() -// API Overload 2: DataFrame provided +// Project : Polymorphic implementation //---------------------------------------------------------------- -MultiviewValues Multiview( DataFrame< double > data, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int D, - int E, - int Tp, - int knn, - int tau, - std::string columns, - std::string target, - int multiview, - int exclusionRadius, - bool trainLib, - bool verbose, - unsigned nThreads ) { +void MultiviewClass::Project( unsigned nThreads ) { - // Require at least E = 1 - if ( E < 1 ) { - std::stringstream errMsg; - errMsg << " Multiview(): E = " << E << " is invalid.\n" ; - throw std::runtime_error( errMsg.str() ); - } - - // Create local Parameters struct. Note embedded = true - Parameters param = Parameters( Method::Simplex, "", "", - pathOut, predictFile, - lib, pred, E, Tp, knn, tau, 0, - exclusionRadius, columns, target, - true, // embedded true - false, verbose, - "", "", "", 0, 0, 0, multiview ); - - if ( not param.columnNames.size() ) { - throw std::runtime_error( "Multiview() requires column names." ); - } - if ( not param.targetName.size() ) { - throw std::runtime_error( "Multiview() requires target name." ); - } - // Ensure that params are validated so columnNames are populated - if ( not param.validated ) { - throw std::runtime_error( "Multiview() params not validated." ); - } + CheckParameters(); - // Validate that columns & target are in data - for ( auto colName : param.columnNames ) { - auto ci = find( data.ColumnNames().begin(), - data.ColumnNames().end(), colName ); - - if ( ci == data.ColumnNames().end() ) { - std::stringstream errMsg; - errMsg << "Multiview(): Failed to find column " - << colName << " in dataFrame with columns: [ "; - for ( auto col : data.ColumnNames() ) { - errMsg << col << " "; - } errMsg << " ]\n"; - throw std::runtime_error( errMsg.str() ); - } - } - auto ti = find( data.ColumnNames().begin(), - data.ColumnNames().end(), param.targetName ); - if ( ti == data.ColumnNames().end() ) { - std::stringstream errMsg; - errMsg << "Multiview(): Failed to find target " - << param.targetName << " in dataFrame with columns: [ "; - for ( auto col : data.ColumnNames() ) { - errMsg << col << " "; - } errMsg << " ]\n"; - throw std::runtime_error( errMsg.str() ); - } + // Set embedded false so E-dimensional embedding is computed + parameters.embedded = false; - // Validate data rows against lib and pred indices - CheckDataRows( param, std::ref( data ), "Multiview()" ); - - //------------------------------------------------------------ - // Generate embedding on param.columns_str - // embedding will have tau * (E-1) fewer rows than data, - // the time column is not returned in embedding. - // Column names are changed from X to X(t-0), X(t-1),... - //------------------------------------------------------------ - DataFrame< double > embedding = Embed( data, - param.E, - param.tau, - param.columns_str, - param.verbose ); - - size_t shift = abs( param.tau ) * ( param.E - 1 ); - - // Delete data top rows of partial data - if ( not data.PartialDataRowsDeleted() ) { - // Not thread safe - std::lock_guard lck( EDM_Multiview::mtx ); - - data.DeletePartialDataRows( shift, param.tau ); - } - - // Adjust param.library and param.prediction vectors of indices - if ( shift > 0 ) { - param.DeleteLibPred(); - } + PrepareEmbedding(); // EmbedData(). target, data, lib/pred adjust - // Get target - std::valarray< double > targetVec = data.VectorColumnName(param.targetName); - - // Save param.predictOutputFile and reset so Simplex() does not write - std::string outputFile = param.predictOutputFile; - param.predictOutputFile = ""; - - // Establish the state-space dimension D - // default to the number of input columns (not embedded columns) - if ( D == 0 ) { - D = param.columnNames.size(); - } - if ( D > embedding.NColumns() ) { - std::stringstream msg; - msg << "WARNING: Multiview(): D = " << D - << " exceeds the number of columns in the embedding: " - << embedding.NColumns() << ". D set to " - << embedding.NColumns() << std::endl; - std::cout << msg.str(); - - D = param.columnNames.size(); + SetupParameters(); // Requires valid embedding + + // Set embedded true for subset columns in Simplex() + parameters.embedded = true; + + Multiview( nThreads ); // Simplex() and output +} + +//---------------------------------------------------------------- +// Multiview algorithm +//---------------------------------------------------------------- +void MultiviewClass::Multiview ( unsigned nThreads ) { + // Create column names for the results DataFrame + // One row for each combo: D columns (a combo), rho, MAE, RMSE + // JP: NOTE DataFrame is based on valarray, so we can't store non + // numeric values (column names) in the results DataFrame. + std::stringstream header; + for ( auto i = 1; i <= parameters.multiviewD; i++ ) { + header << "Col_" << i << " "; } - + header << "rho MAE RMSE"; + // Combinations of possible embedding variables, D at-a-time // Note that these combinations are not zero-offset, i.e. // Combination( 3, 2 ) = [(1, 2), (1, 3), (2, 3)] // These correspond to column indices +1 std::vector< std::vector< size_t > > combos = - Combination( embedding.NColumns(), D ); + Combination( embedding.NColumns(), parameters.multiviewD ); #ifdef DEBUG_ALL std::cout << "Multiview(): " << combos.size() << " combos:\n"; - for ( auto i = 0; i < combos.size(); i++ ) { + for ( size_t i = 0; i < combos.size(); i++ ) { std::vector< size_t > combo_i = combos[i]; std::cout << "["; - for ( auto j = 0; j < combo_i.size(); j++ ) { + for ( size_t j = 0; j < combo_i.size(); j++ ) { std::cout << combo_i[j] << ","; } std::cout << "] "; } std::cout << std::endl; #endif - + // Establish number of ensembles if not specified - if ( not param.MultiviewEnsemble ) { + if ( not parameters.multiviewEnsemble ) { // Ye & Sugihara suggest sqrt( m ) as the number of embeddings to avg - param.MultiviewEnsemble = std::max(2, (int) std::sqrt(combos.size())); - + parameters.multiviewEnsemble = std::max(2,(int)std::sqrt(combos.size())); + std::stringstream msg; msg << "Multiview() Set view sample size to " - << param.MultiviewEnsemble << std::endl; + << parameters.multiviewEnsemble << std::endl; std::cout << msg.str(); } - // validate number of combinations - if ( param.MultiviewEnsemble > combos.size() ) { + // Validate number of combinations + if ( parameters.multiviewEnsemble > (int) combos.size() ) { std::stringstream msg; msg << "WARNING: Multiview(): multiview ensembles " - << param.MultiviewEnsemble + << parameters.multiviewEnsemble << " exceeds the number of available combinations: " << combos.size() << ". Set to " << combos.size() << std::endl; std::cout << msg.str(); - - param.MultiviewEnsemble = combos.size(); - } - - //--------------------------------------------------------------- - // Evaluate variable combinations. - // Note that this is done within the library itself (in-sample). - //--------------------------------------------------------------- - // Save a copy of the specified prediction observation rows. - std::vector prediction = param.prediction; - if ( trainLib ) { - // Override param.prediction for in-sample forecast skill evaluation - param.prediction = param.library; + parameters.multiviewEnsemble = combos.size(); } - // This is not a good implementation... - // Replace param.E with the number of dimensions, recall embbeded = true - param.E = D; - - // Create column names for the results DataFrame - // One row for each combo: D columns (a combo), rho, MAE, RMSE - // JP: NOTE DataFrame is based on valarray, so we can't store non - // numeric values (column names) in the results DataFrame. - std::stringstream header; - for ( auto i = 1; i <= D; i++ ) { - header << "Col_" << i << " "; - } - header << "rho MAE RMSE"; - // Results Data Frame: D columns (a combo), rho, mae, rmse - DataFrame combos_rho( combos.size(), D + 3, header.str() ); + DataFrame< double > combosRho( combos.size(), + parameters.multiviewD + 3, header.str() ); // Results vector of DataFrame's with prediction results - std::vector< DataFrame< double > > combos_prediction( combos.size() ); + std::vector< DataFrame< double > > combosPrediction( combos.size() ); // Build work queue EDM_Multiview::WorkQueue workQ( combos.size() ); // Insert combos index into work queue - for ( auto i = 0; i < combos.size(); i++ ) { + for ( size_t i = 0; i < combos.size(); i++ ) { workQ[ i ] = i; } unsigned maxThreads = std::thread::hardware_concurrency(); if ( maxThreads < nThreads ) { nThreads = maxThreads; } + //--------------------------------------------------------------- + // Evaluate variable combinations. + //--------------------------------------------------------------- // thread container std::vector< std::thread > threads; for ( unsigned i = 0; i < nThreads; ++i ) { threads.push_back( std::thread( EvalComboThread, - param, + std::ref( *this ), workQ, - combos, - std::ref( embedding ), - std::ref( targetVec ), - std::ref( combos_rho ), - std::ref( combos_prediction ) ) ); + std::ref( combos ), + std::ref( combosRho ), + std::ref( combosPrediction ) ) ); } // join threads @@ -360,7 +197,6 @@ MultiviewValues Multiview( DataFrame< double > data, // Unroll all other exception from the thread/loops while( not EDM_Multiview::exceptionQ.empty() ) { - // JP When do these exception_ptr get deleted? Is it a leak? EDM_Multiview::exceptionQ.pop(); } std::rethrow_exception( exceptionPtr ); @@ -370,106 +206,107 @@ MultiviewValues Multiview( DataFrame< double > data, // Rank forecasts. If trainLib true these are in-sample (library) //----------------------------------------------------------------- // Make pairs of row indices and rho - std::valarray< double > rho = combos_rho.VectorColumnName( "rho" ); + std::valarray< double > rho = combosRho.VectorColumnName( "rho" ); // vector of indices std::valarray< size_t > indices( rho.size() ); std::iota( begin( indices ), end( indices ), 0 ); // Ensure that rho is the first of the pair so sort will work - std::vector< std::pair< double, int > > combo_sort( rho.size() ); + std::vector< std::pair< double, int > > comboSort( rho.size() ); for ( size_t i = 0; i < rho.size(); i++ ) { - combo_sort[ i ] = std::make_pair( rho[i], indices[i] ); + comboSort[ i ] = std::make_pair( rho[i], indices[i] ); } // sort pairs and reverse for largest rho first - std::sort ( combo_sort.begin(), combo_sort.end() ); - std::reverse( combo_sort.begin(), combo_sort.end() ); + std::sort ( comboSort.begin(), comboSort.end() ); + std::reverse( comboSort.begin(), comboSort.end() ); #ifdef DEBUG_ALL - std::cout << "Multiview(): combos:\n" << combos_rho << std::endl; + std::cout << "Multiview(): combos:\n" << combosRho << std::endl; std::cout << "Ranked combos:\n"; - for ( auto i = 0; i < combo_sort.size(); i++ ) { + for ( size_t i = 0; i < comboSort.size(); i++ ) { std::cout << "("; - std::pair< double, int > combo_pair = combo_sort[ i ]; - std::cout << combo_pair.first << "," - << combo_pair.second << ") "; + std::pair< double, int > comboPair = comboSort[ i ]; + std::cout << comboPair.first << "," + << comboPair.second << ") "; } std::cout << std::endl; #endif // --------------------------------------------------------------- // Perform predictions with the top multiview embeddings // --------------------------------------------------------------- - if ( trainLib ) { + if ( parameters.multiviewTrainLib ) { // Reset the user specified prediction vector - param.prediction = prediction; + parameters.prediction = predictionIn; } // Get top param.MultiviewEnsemble combos - size_t nEnsemble = std::min( (int) combo_sort.size(), - param.MultiviewEnsemble ); + size_t nEnsemble = std::min( (int) comboSort.size(), + parameters.multiviewEnsemble ); + std::vector< std::pair< double, int > > - combo_best( combo_sort.begin(), combo_sort.begin() + nEnsemble ); - + comboBest( comboSort.begin(), comboSort.begin() + nEnsemble ); + #ifdef DEBUG_ALL std::cout << "Multiview(): Best combos:\n"; - for ( auto i = 0; i < combo_best.size(); i++ ) { - std::pair< double, int > combo_pair = combo_best[ i ]; - std::vector< size_t > this_combo = combos[ combo_pair.second ]; - std::cout << "(" << combo_pair.first << " ["; - for ( auto j = 0; j < this_combo.size(); j++ ) { - std::cout << this_combo[j] << ","; + for ( size_t i = 0; i < comboBest.size(); i++ ) { + std::pair< double, int > comboPair = comboBest[ i ]; + std::vector< size_t > thisCombo = combos[ comboPair.second ]; + std::cout << "(" << comboPair.first << " ["; + for ( size_t j = 0; j < thisCombo.size(); j++ ) { + std::cout << thisCombo[j] << ","; } std::cout << "]) "; } std::cout << std::endl; #endif - // Create combos_best (vector of column numbers) from combo_best - std::vector< std::vector< size_t > > combos_best( param.MultiviewEnsemble ); - for ( auto i = 0; i < combo_best.size(); i++ ) { - std::pair< double, int > combo_pair = combo_best[ i ]; - std::vector< size_t > this_combo = combos[ combo_pair.second ]; - combos_best[ i ] = this_combo; + // Create combosBest (vector of column numbers) from comboBest + std::vector< std::vector< size_t > > + combosBest( parameters.multiviewEnsemble ); + + for ( size_t i = 0; i < comboBest.size(); i++ ) { + std::pair< double, int > comboPair = comboBest[ i ]; + std::vector< size_t > thisCombo = combos[ comboPair.second ]; + combosBest[ i ] = thisCombo; } - + // Results Data Frame: D columns (a combo), and rho mae rmse - DataFrame combos_rho_pred( param.MultiviewEnsemble, - D + 3, header.str() ); - + DataFrame combosRhoPred( parameters.multiviewEnsemble, + parameters.multiviewD + 3, header.str() ); + // Results vector of DataFrame's with prediction results // Used to compute the multiview ensemble average prediction std::vector< DataFrame< double > > - combos_rho_prediction( param.MultiviewEnsemble ); + combosRhoPrediction( parameters.multiviewEnsemble ); //-------------------------------------------------------------------- // If trainLib false, no need to compute these projections //-------------------------------------------------------------------- - if ( trainLib ) { + if ( parameters.multiviewTrainLib ) { // Build work queue - EDM_Multiview::WorkQueue workQ_pred( param.MultiviewEnsemble ); - + EDM_Multiview::WorkQueue workQPred( parameters.multiviewEnsemble ); + // Insert combos index into work queue - for ( auto i = 0; i < param.MultiviewEnsemble; i++ ) { - workQ_pred[ i ] = i; + for ( auto i = 0; i < parameters.multiviewEnsemble; i++ ) { + workQPred[ i ] = i; } - + // thread container - std::vector< std::thread > threads_pred; + std::vector< std::thread > threadsPred; for ( unsigned i = 0; i < nThreads; ++i ) { - threads_pred.push_back( + threadsPred.push_back( std::thread( EvalComboThread, - param, - workQ_pred, - combos_best, - std::ref( embedding ), - std::ref( targetVec ), - std::ref( combos_rho_pred ), - std::ref( combos_rho_prediction) ) ); + std::ref( *this ), + workQPred, + std::ref( combosBest ), + std::ref( combosRhoPred ), + std::ref( combosRhoPrediction) ) ); } - + // join threads - for ( auto &thrd : threads_pred ) { + for ( auto &thrd : threadsPred ) { thrd.join(); } - + // If thread threw exception, get from queue and rethrow if ( not EDM_Multiview::exceptionQ.empty() ) { std::lock_guard lck( EDM_Multiview::q_mtx ); @@ -479,17 +316,16 @@ MultiviewValues Multiview( DataFrame< double > data, // Unroll all other exception from the thread/loops while( not EDM_Multiview::exceptionQ.empty() ) { - // JP When do these exception_ptr get deleted? Is it a leak? EDM_Multiview::exceptionQ.pop(); } std::rethrow_exception( exceptionPtr ); } #ifdef DEBUG_ALL - for ( auto cpi = combos_rho_prediction.begin(); - cpi != combos_rho_prediction.end(); ++cpi ) { + for ( auto cpi = combosRhoPrediction.begin(); + cpi != combosRhoPrediction.end(); ++cpi ) { std::cout << *cpi; } - std::cout << combos_rho_pred; + std::cout << combosRhoPred; #endif } // if ( trainLib ) //------------------------------------------------------------------- @@ -500,13 +336,13 @@ MultiviewValues Multiview( DataFrame< double > data, // Insert top prediction results into combos_rho_pred // Insert top predictions into combos_rho_prediction int row; - for ( size_t row_i = 0; row_i < param.MultiviewEnsemble; row_i++ ) { - row = combo_sort[ row_i ].second; // row index of best rho - combos_rho_pred.WriteRow( row_i, combos_rho.Row( row ) ); - combos_rho_prediction[ row_i ] = combos_prediction[ row ]; + for ( int row_i = 0; row_i < parameters.multiviewEnsemble; row_i++ ) { + row = comboSort[ row_i ].second; // row index of best rho + combosRhoPred.WriteRow( row_i, combosRho.Row( row ) ); + combosRhoPrediction[ row_i ] = combosPrediction[ row ]; } } - + //---------------------------------------------------------- // Compute Multiview averaged prediction // combos_rho_prediction is a vector of DataFrames with @@ -514,79 +350,82 @@ MultiviewValues Multiview( DataFrame< double > data, //---------------------------------------------------------- // Get copy of Observations std::valarray< double > - Obs = combos_rho_prediction[0].VectorColumnName( "Observations" ); + Obs = combosRhoPrediction[0].VectorColumnName( "Observations" ); // Create ensemble average prediction vector std::valarray< double > Predictions( 0., Obs.size() ); // Compute ensemble prediction (this seems clunky...) - // combos_rho_prediction is a vector of DataFrames with prediction + // combosRhoPrediction is a vector of DataFrames with prediction // from each combo - for ( auto i = 0; i < param.MultiviewEnsemble; i++ ) { + for ( auto i = 0; i < parameters.multiviewEnsemble; i++ ) { std::valarray< double > prediction_i = - combos_rho_prediction[ i ].VectorColumnName( "Predictions" ); + combosRhoPrediction[ i ].VectorColumnName( "Predictions" ); // Accumulate prediction values - for ( auto j = 0; j < Predictions.size(); j++ ) { - Predictions[ j ] += prediction_i[ j ]; + for ( size_t j = 0; j < Predictions.size(); j++ ) { + Predictions[ j ] += prediction_i[ j ]; } } // Mean of prediction values - for ( auto i = 0; i < Predictions.size(); i++ ) { - Predictions[ i ] /= param.MultiviewEnsemble; + for ( size_t i = 0; i < Predictions.size(); i++ ) { + Predictions[ i ] /= parameters.multiviewEnsemble; } - // Error of ensemble prediction - VectorError ve = ComputeError( Obs, Predictions ); - - // Output Prediction DataFrame - DataFrame< double > Prediction( Predictions.size(), 2, - "Observations Predictions" ); + // Allocate output Prediction DataFrame + DataFrame< double > Prediction ( Predictions.size(), 2, + "Observations Predictions" ); // Output time vector - std::vector< std::string > predTime( param.prediction.size() + - abs( param.Tp ) ); - - FillTimes( param, data.Time(), std::ref( predTime ) ); - + std::vector< std::string > predTime( parameters.prediction.size() + + abs( parameters.Tp ) ); + + FillTimes( std::ref( predTime ) ); + Prediction.Time() = predTime; Prediction.TimeName() = data.TimeName(); Prediction.WriteColumn( 0, Obs ); Prediction.WriteColumn( 1, Predictions ); - if ( outputFile.size() ) { - Prediction.WriteData( param.pathOut, outputFile ); + if ( predictOutputFileIn.size() ) { + Prediction.WriteData( parameters.pathOut, predictOutputFileIn ); } // Create combos_rho table with column names std::vector< std::string > comboTable = - ComboRhoTable( combos_rho_pred, embedding.ColumnNames() ); + ComboRhoTable( combosRhoPred, embedding.ColumnNames() ); + + if ( parameters.verbose ) { + // Error of ensemble prediction + VectorError ve = ComputeError( Obs, Predictions ); - if ( param.verbose ) { std::cout << "Multiview(): rho " << ve.rho << " MAE " << ve.MAE << " RMSE " << ve.RMSE << std::endl; std::cout << std::endl << "Multiview Combinations:" << std::endl; + for ( auto tableRow : comboTable ) { std::cout << tableRow << std::endl; } std::cout << std::endl; - } + } + + // Allocate output + MultiviewValues MVout; + MVout.ComboRho = combosRhoPred; + MVout.Predictions = Prediction; + MVout.ComboRhoTable = comboTable; - struct MultiviewValues MV( combos_rho_pred, Prediction, comboTable ); - - return MV; + MVvalues = MVout; // Assign to Multiview Object } //---------------------------------------------------------------- // Worker thread -// Output: Write rho to combos_rho DataFrame, -// Simplex results to combos_prediction +// Output: Write rho to combosRho DataFrame, +// Simplex results to combosPrediction //---------------------------------------------------------------- -void EvalComboThread( Parameters param, - EDM_Multiview::WorkQueue workQ, - std::vector< std::vector< size_t > > combos, - DataFrame< double > &embedding, - std::valarray< double > &targetVec, - DataFrame< double > &combos_rho, - std::vector< DataFrame< double > > &combos_prediction ) +void EvalComboThread( MultiviewClass & MV, + EDM_Multiview::WorkQueue workQ, + std::vector< std::vector< size_t > > & combos, + DataFrame< double > & combosRho, + std::vector< DataFrame< double > > & combosPrediction ) { // atomic_fetch_add(): Adds val to the contained value and returns // the value it had immediately before the operation. @@ -594,63 +433,84 @@ void EvalComboThread( Parameters param, std::atomic_fetch_add( &EDM_Multiview::eval_i, std::size_t(1) ); while( eval_i < workQ.size() ) { - + // WorkQueue stores combo index in combos size_t combo_i = workQ[ eval_i ]; - + // Get the combo for this thread std::vector< size_t > combo = combos[ combo_i ]; try { - + // Local copy with combo column indices (zero-offset) - std::vector< size_t > combo_cols( combo ); + std::vector< size_t > comboCols( combo ); // Zero offset combo column indices for dataFrame - for ( auto ci = combo_cols.begin(); ci != combo_cols.end(); ++ci ) { + for ( auto ci = comboCols.begin(); ci != comboCols.end(); ++ci ) { *ci = *ci - 1; } + // Embedded column names are column names + (t-0), (t-1),... + std::vector< std::string > comboColumnNames; + for ( size_t i = 0; i < comboCols.size(); i++ ) { + comboColumnNames.push_back( + MV.embedding.ColumnNames()[ comboCols[ i ] ] ); + } + #ifdef DEBUG_ALL { std::lock_guard lck( EDM_Multiview::mtx ); std::cout << "EvalComboThread() Thread [" << std::this_thread::get_id() << "] "; std::cout << "combo: ["; - for ( auto i = 0; i < combo.size(); i++ ) { + for ( size_t i = 0; i < combo.size(); i++ ) { std::cout << combo[i] << ","; } std::cout << "] rho = "; } #endif - // Select combo columns from the data - DataFrame comboData = - embedding.DataFrameFromColumnIndex( combo_cols ); + // Target has been embedded too... add "(t-0)" + std::stringstream threadTarget; + threadTarget << MV.parameters.targetName << "(t-0)"; + + // Find target column in embedding + size_t targetColumn = + MV.embedding.ColumnNameToIndex()[ threadTarget.str() ]; + + // Add target column to comboCols for DataFrame subset + comboCols.push_back( targetColumn ); - // Compute neighbors on comboData - Neighbors neighbors = FindNeighbors( comboData, param ); + // Select combo columns from the data : columns and target + DataFrame< double > comboData = + MV.embedding.DataFrameFromColumnIndex( comboCols ); - // Pack embedding, target, neighbors for SimplexProjection - DataEmbedNN embedNN = DataEmbedNN( &embedding, comboData, - targetVec, neighbors ); + // Must use thread local Parameters since columns have been embedded + // and are different than base Parameters. x_t -> x_t(t-0)... + Parameters threadParameters( MV.parameters ); + + // Replace base copied columnNames and targetName with embedded ones + threadParameters.columnNames = comboColumnNames; + threadParameters.targetName = threadTarget.str(); + + // Simplex + SimplexClass S( std::ref( comboData ), std::ref( threadParameters ) ); - // combo prediction // This is an embedded = true, E = D columns prediction - DataFrame S = SimplexProjection( param, embedNN ); + S.Project(); // Write combo prediction DataFrame - combos_prediction[ eval_i ] = S; + combosPrediction[ eval_i ] = S.projection; // Evaluate combo prediction - VectorError ve = ComputeError( S.VectorColumnName( "Observations" ), - S.VectorColumnName( "Predictions" ) ); - + VectorError ve = + ComputeError( S.projection.VectorColumnName( "Observations" ), + S.projection.VectorColumnName( "Predictions" ) ); #ifdef DEBUG_ALL { std::lock_guard lck( EDM_Multiview::mtx ); std::cout << ve.rho << std::endl; std::cout << "-------------- embedding -------------------\n"; - std::cout << embedding; + std::cout << S.embedding; std::cout << "-------------- comboData -------------------\n"; std::cout << comboData; } @@ -658,15 +518,15 @@ void EvalComboThread( Parameters param, // Write combo and rho to the Data Frame // D columns (a combo), rho, MAE, RMSE - std::valarray< double > combo_row( combo.size() + 3 ); - for ( auto i = 0; i < combo.size(); i++ ) { - combo_row[ i ] = combo[ i ]; + std::valarray< double > comboRow( combo.size() + 3 ); + for ( size_t i = 0; i < combo.size(); i++ ) { + comboRow[ i ] = combo[ i ]; } - combo_row[ combo.size() ] = ve.rho; - combo_row[ combo.size() + 1 ] = ve.MAE; - combo_row[ combo.size() + 2 ] = ve.RMSE; - - combos_rho.WriteRow( eval_i, combo_row ); + comboRow[ combo.size() ] = ve.rho; + comboRow[ combo.size() + 1 ] = ve.MAE; + comboRow[ combo.size() + 2 ] = ve.RMSE; + + combosRho.WriteRow( eval_i, comboRow ); } // try catch(...) { @@ -674,30 +534,120 @@ void EvalComboThread( Parameters param, std::lock_guard lck( EDM_Multiview::q_mtx ); EDM_Multiview::exceptionQ.push( std::current_exception() ); } - + eval_i = std::atomic_fetch_add(&EDM_Multiview::eval_i, std::size_t(1)); } - + // Reset counter std::atomic_store( &EDM_Multiview::eval_i, std::size_t(0) ); } +//----------------------------------------------------------------- +// Populate EDM::MultiviewClass Parameters objects +//----------------------------------------------------------------- +void MultiviewClass::SetupParameters() { + // Clear parameters.predictOutputFile so Simplex() does not write + // predictOutputFile copied to member predictOutputFileIn in constructor + parameters.predictOutputFile = ""; + + // Establish the state-space dimension D + // default to the number of input columns (not embedding columns) + if ( parameters.multiviewD == 0 ) { + parameters.multiviewD = parameters.columnNames.size(); + } + if ( parameters.multiviewD > (int) embedding.NColumns() ) { + std::stringstream msg; + msg << "WARNING: Multiview(): D = " << parameters.multiviewD + << " exceeds the number of columns in the embedding: " + << embedding.NColumns() << ". D set to " + << embedding.NColumns() << std::endl; + std::cout << msg.str(); + + parameters.multiviewD = (int) embedding.NColumns(); + } + + // Save a copy of the specified prediction observation rows. + predictionIn = parameters.prediction; + + if ( parameters.multiviewTrainLib ) { + // Override parameters.prediction for in-sample forecast skill evaluation + parameters.prediction = parameters.library; + } + + // This is not a good implementation... + // Replace parameters.E with the number of dimensions, recall embbeded = true + parameters.E = parameters.multiviewD; +} + +//----------------------------------------------------------------- +// +//----------------------------------------------------------------- +void MultiviewClass::CheckParameters() { + // Require at least E = 1 + if ( parameters.E < 1 ) { + std::stringstream errMsg; + errMsg << " Multiview(): E = " << parameters.E << " is invalid.\n" ; + throw std::runtime_error( errMsg.str() ); + } + + if ( not parameters.columnNames.size() ) { + throw std::runtime_error( "Multiview() requires column names." ); + } + if ( not parameters.targetName.size() ) { + throw std::runtime_error( "Multiview() requires target name." ); + } + // Ensure that params are validated so columnNames are populated + if ( not parameters.validated ) { + throw std::runtime_error( "Multiview() params not validated." ); + } + + // Validate that columns & target are in data + for ( auto colName : parameters.columnNames ) { + auto ci = find( data.ColumnNames().begin(), + data.ColumnNames().end(), colName ); + + if ( ci == data.ColumnNames().end() ) { + std::stringstream errMsg; + errMsg << "Multiview(): Failed to find column " + << colName << " in dataFrame with columns: [ "; + for ( auto col : data.ColumnNames() ) { + errMsg << col << " "; + } errMsg << " ]\n"; + throw std::runtime_error( errMsg.str() ); + } + } + auto ti = find( data.ColumnNames().begin(), + data.ColumnNames().end(), parameters.targetName ); + if ( ti == data.ColumnNames().end() ) { + std::stringstream errMsg; + errMsg << "Multiview(): Failed to find target " + << parameters.targetName << " in dataFrame with columns: [ "; + for ( auto col : data.ColumnNames() ) { + errMsg << col << " "; + } errMsg << " ]\n"; + throw std::runtime_error( errMsg.str() ); + } + + // Validate data rows against lib and pred indices + CheckDataRows( "Multiview()" ); +} + //---------------------------------------------------------------- // Return combinations C(n,k) as a vector of vectors //---------------------------------------------------------------- std::vector< std::vector< size_t > > Combination( int n, int k ) { std::vector< bool > v(n); - for ( size_t i = 0; i < n; ++i ) { + for ( int i = 0; i < n; ++i ) { v[i] = ( i >= (n - k) ); } std::vector< std::vector< size_t > > combos; - + do { std::vector< size_t > this_combo( k ); size_t j = 0; - for ( size_t i = 0; i < n; ++i ) { + for ( int i = 0; i < n; ++i ) { if ( v[i] ) { this_combo[ j ] = i + 1; j++; @@ -705,7 +655,7 @@ std::vector< std::vector< size_t > > Combination( int n, int k ) { } // insert this tuple in the combos vector combos.push_back( this_combo ); - + } while ( std::next_permutation( v.begin(), v.end() ) ); return combos; @@ -732,7 +682,7 @@ std::vector< std::string > ComboRhoTable( } std::vector< std::string > table; - + // Header std::stringstream header; for ( size_t col = 0; col < nCol; col++ ) { // column indices @@ -748,9 +698,9 @@ std::vector< std::string > ComboRhoTable( for ( size_t row = 0; row < combos_rho_pred.NRows(); row++ ) { std::stringstream rowsstring; rowsstring.precision( 4 ); - + std::valarray< double > rowValues = combos_rho_pred.Row( row ); - + for ( size_t col = 0; col < nCol; col++ ) { rowsstring << std::setw(4) << rowValues[ col ] << ", "; } @@ -762,9 +712,9 @@ std::vector< std::string > ComboRhoTable( rowsstring << std::setw(6) << rowValues[ nCol ] << ", "; // rho rowsstring << std::setw(6) << rowValues[ nCol + 1 ] << ", "; // MAE rowsstring << std::setw(6) << rowValues[ nCol + 2 ]; // RMSE - + table.push_back( rowsstring.str() ); } - + return table; } diff --git a/src/Multiview.h b/src/Multiview.h new file mode 100644 index 0000000..76aff4c --- /dev/null +++ b/src/Multiview.h @@ -0,0 +1,34 @@ + +#ifndef EDM_MULTIVIEW_H +#define EDM_MULTIVIEW_H + +#include +#include +#include +#include + +#include "EDM.h" +#include "Simplex.h" + +//---------------------------------------------------------------- +// Multiview class inherits from Simplex class and defines +// CCM-specific projection methods +//---------------------------------------------------------------- +class MultiviewClass : public SimplexClass { +public: + std::string predictOutputFileIn; // copy from parameters + std::vector predictionIn; // copy from parameters + + struct MultiviewValues MVvalues; // output structure + + // Constructor + MultiviewClass ( DataFrame< double > & data, + Parameters & parameters ); + + // Method declarations + void Project( unsigned maxThreads ); + void CheckParameters(); + void SetupParameters(); + void Multiview( unsigned maxThreads ); +}; +#endif diff --git a/src/Neighbors.cc b/src/Neighbors.cc deleted file mode 100644 index 07276c5..0000000 --- a/src/Neighbors.cc +++ /dev/null @@ -1,357 +0,0 @@ - -#include "Neighbors.h" - -//---------------------------------------------------------------- -Neighbors:: Neighbors(): anyTies(false) {} -Neighbors::~Neighbors() {} - -namespace EDM_Neighbors { - // Define the initial maximum distance for neigbors to avoid sort() - // DBL_MAX is a Macro equivalent to: std::numeric_limits::max() - double DistanceMax = std::numeric_limits::max(); - double DistanceLimit = std::numeric_limits::max() / ( 1 + 1E-9 ); -} - -//---------------------------------------------------------------- -// It is assumed that the data frame has only columns of data for -// which knn will be computed. The (time) column is not present. -//---------------------------------------------------------------- -Neighbors FindNeighbors( - DataFrame dataFrame, - Parameters parameters ) -{ - -#ifdef DEBUG_ALL - PrintDataFrameIn( dataFrame, parameters ); -#endif - - if ( not parameters.validated ) { - std::string errMsg("FindNeighbors(): Parameters not validated." ); - throw( std::runtime_error( errMsg ) ); - } - - if ( parameters.embedded and parameters.E > dataFrame.NColumns() ) { - std::stringstream errMsg; - errMsg << "WARNING: FindNeighbors() Multivariate data " - << "(embedded = true): The number of dataFrame columns (" - << dataFrame.NColumns() << ") is less than the embedding " - << "dimension E (" << parameters.E << ")\n"; - std::cout << errMsg.str(); - } - - size_t N_library_rows = parameters.library.size(); - size_t N_prediction_rows = parameters.prediction.size(); - size_t N_columns = dataFrame.NColumns(); - - auto max_lib_it = std::max_element( parameters.library.begin(), - parameters.library.end() ); - size_t max_lib_index = *max_lib_it; - - // Maximum column index. - // We assume that the dataFrame has been selected to the proper columns - size_t maxCol_i = N_columns - 1; - - if ( parameters.verbose ) { - // Identify degenerate library : prediction points by - // set_intersection() of lib & pred indices, needs a result vector - std::vector< double > result( N_library_rows + N_prediction_rows, 0 ); - - std::vector< double >::iterator ii = set_intersection ( - parameters.prediction.begin(), parameters.prediction.end(), - parameters.library.begin(), parameters.library.end(), - result.begin() ); - - if ( ii != result.begin() ) { - // Overlapping indices exist - std::stringstream msg; - msg << "WARNING: FindNeighbors(): Degenerate library and " - << " prediction data found. Overlap indices: "; - for ( auto ri = result.begin(); ri != ii; ++ri ) { - msg << *ri << " "; - } msg << std::endl; - std::cout << msg.str(); - } - } - - // Compute distances of all pred : lib vectors - // DistLib.distances is DataFrame of pred rows x lib columns with - // distances for each pred row to all lib rows - Neighbors DistLib = Distances( std::ref(dataFrame), parameters ); - - // DistLib.neighbors are the lib row indices, 1 row x lib columns - std::valarray< size_t > rowLib = DistLib.neighbors.Row( 0 ); - - // Pair the distances and library row indices for sort on distance - // Each predPairs element correponds to a prediction row and - // holds a vector of < distance, lib_row > pairs for each lib_row - std::vector< std::vector< std::pair< double, size_t > > > - predPairs( N_prediction_rows ); - - for ( size_t pred_row = 0; pred_row < N_prediction_rows; pred_row++ ) { - std::valarray< double > rowDist = DistLib.distances.Row( pred_row ); - - std::vector< std::pair > rowPairs( rowDist.size() ); - for ( size_t i = 0; i < rowDist.size(); i++ ) { - rowPairs[ i ] = std::make_pair( rowDist[i], rowLib[i] ); - } - // insert into predPairs - predPairs[ pred_row ] = rowPairs; - } - -#ifdef DEBUG_ALL - std::cout << DistLib.neighbors; - std::cout << DistLib.distances; - for ( size_t pred_row = 0; pred_row < predPairs.size(); pred_row++ ) { - std::vector< std::pair > rowPair = predPairs[ pred_row ]; - for ( size_t i = 0; i < rowPair.size(); i++ ) { - std::pair thisPair = rowPair[ i ]; - std::cout << "[" << thisPair.first << ", " - << thisPair.second << "] "; - } std::cout << std::endl; - } std::cout << std::endl; -#endif - - // Neighbors: struct on local stack to be returned by copy - Neighbors neighbors = Neighbors(); - neighbors.neighbors = DataFrame(N_prediction_rows, parameters.knn); - neighbors.distances = DataFrame(N_prediction_rows, parameters.knn); - - // To be inserted in neighbors struct below - std::vector< bool > ties( N_prediction_rows, false ); - std::vector< std::vector< std::pair< double, size_t > > > - tiePairs( N_prediction_rows ); - - //------------------------------------------------------------------- - // For each prediction vector (row in prediction DataFrame) find the - // list of library indices that are within k_NN points - //------------------------------------------------------------------- - for ( size_t pred_row = 0; pred_row < predPairs.size(); pred_row++ ) { - - // rowPair is a vector of pairs of length library rows - // Get the rowPair for this prediction row - std::vector< std::pair > rowPair = predPairs[ pred_row ]; - - // sort < distance, lib_row > pairs for this pred_row - // distance must be .first - std::sort( rowPair.begin(), rowPair.end(), DistanceCompare ); - - // Insert knn distance / library row index into knn vectors - std::valarray< double > knnDistances( parameters.knn ); - std::valarray< size_t > knnLibRows ( parameters.knn ); - - size_t lib_row_i = 0; - size_t k = 0; - while ( k < parameters.knn ) { - double distance = rowPair[ lib_row_i ].first; - size_t lib_row = rowPair[ lib_row_i ].second; - - if ( not parameters.noNeighborLimit ) { - // Reach exceeding grasp : forecast point is outside library - if ( lib_row + parameters.Tp > max_lib_index or - lib_row + parameters.Tp < 0 ) { - lib_row_i++; - continue; // keep looking - } - } - - // Exclusion radius: units are data rows, not time - if ( parameters.exclusionRadius ) { - int xrad = (int) lib_row - (int) pred_row; - if ( std::abs( xrad ) <= parameters.exclusionRadius ) { - lib_row_i++; - continue; // skip this neighbor - } - } - - knnDistances[ k ] = distance; - knnLibRows [ k ] = lib_row; - lib_row_i++; - k++; - } - - neighbors.distances.WriteRow( pred_row, knnDistances ); - neighbors.neighbors.WriteRow( pred_row, knnLibRows ); - - // Check for ties 1.18e−38 is float 32-bit min - if ( k < rowPair.size() ) { - if ( rowPair[ k ].first <= rowPair[ k-1 ].first ) { - // At least one tie... - std::vector< std::pair< double, size_t > > rowTiePairs; - - while( k < rowPair.size() and rowPair[ k ].first > 0 and - rowPair[ k ].first <= rowPair[ k-1 ].first ) { - - // Set flag in ties and store tie pairs in tiePairs - ties[ pred_row ] = true; - - rowTiePairs.push_back(std::make_pair( rowPair[ k ].first, - rowPair[ k ].second )); - k++; - } - - if ( find( ties.begin(), ties.end(), true ) != ties.end() ) { - neighbors.anyTies = true; - tiePairs[ pred_row ] = rowTiePairs; - } - } - } - } // for ( pred_row = 0; pred_row < predPairs.size(); pred_row++ ) - - neighbors.ties = ties; - neighbors.tiePairs = tiePairs; - -#ifdef DEBUG_ALL - for ( size_t i = 0; i < neighbors.ties.size(); i++ ) { - if ( neighbors.ties[ i ] ) { - std::vector< std::pair< double, size_t > > rowTiePairs = - neighbors.tiePairs[ i ]; - std::cout << "Ties at pred_i " << i << ": "; - for ( size_t j = 0; j < rowTiePairs.size(); j++ ) { - double dist = rowTiePairs[ j ].first; - size_t prow = rowTiePairs[ j ].second; - std::cout << "[ " << dist << ", " << prow << "] "; - } std::cout << std::endl; - } - } - - const Neighbors &neigh = neighbors; - PrintNeighborsOut( neigh ); -#endif - - return neighbors; -} - -//--------------------------------------------------------------------- -// Compute all prediction row : library row distances. -// Note that dataBlock does NOT have the time in column 0. -// -// Hijack a Neighbors struct to return two DataFrames: -// distances: pred rows x lib columns matrix with distances. -// distance(i,j) hold distance between the E-dimensional -// phase space point prediction row i and library row j. -// neighbors: 1 row x lib cols matrix with lib rows -//--------------------------------------------------------------------- -Neighbors Distances( const DataFrame< double > &dataBlock, - Parameters param ) { - - size_t N_pred = param.prediction.size(); - size_t N_lib = param.library.size(); - - // Output distance matrix - DataFrame< double > D = DataFrame< double >( N_pred, N_lib ); - DataFrame< size_t > N = DataFrame< size_t >( 1, N_lib ); - - // Initialise D to DistanceMax - std::valarray< double > row_init( EDM_Neighbors::DistanceMax, N_lib ); - for ( size_t row = 0; row < N_pred; row++ ) { - D.WriteRow( row, row_init ); - } - - // Set lib indices into neighbors - for ( size_t col = 0; col < N_lib; col++ ) { - N( 0, col ) = param.library[ col ]; - } - - // Compute all prediction row : library row distances - for ( size_t row = 0; row < N_pred; row++ ) { - // Get E-dimensional vector from this prediction row - std::valarray< double > v1 = dataBlock.Row( param.prediction[ row ] ); - - for ( size_t col = 0; col < N_lib; col++ ) { - // Find distance between vector (v1) and library vector v2 - std::valarray< double > v2 = dataBlock.Row( param.library[ col ] ); - - D( row, col ) = Distance( v1, v2, DistanceMetric::Euclidean ); - } - } - - Neighbors DistLib = Neighbors(); - DistLib.distances = D; - DistLib.neighbors = N; - - return DistLib; -} - -//---------------------------------------------------------------- -// -//---------------------------------------------------------------- -double Distance( const std::valarray &v1, - const std::valarray &v2, - DistanceMetric metric ) -{ - double distance = 0; - - // For efficiency sake, we forego the usual validation of v1 & v2. - - if ( metric == DistanceMetric::Euclidean ) { - double sum = 0; - double delta = 0; - for ( size_t i = 0; i < v1.size(); i++ ) { - delta = v2[i] - v1[i]; - sum += delta * delta; // avoid call to pow() - } - distance = sqrt( sum ); - - // Note: this implicit implementation is slower - // std::valarray delta = v2 - v1; - // distance = sqrt( (delta * delta).sum() ); - } - else if ( metric == DistanceMetric::Manhattan ) { - double sum = 0; - for ( size_t i = 0; i < v1.size(); i++ ) { - sum += abs( v2[i] - v1[i] ); - } - distance = sum; - } - else { - std::stringstream errMsg; - errMsg << "Distance() Invalid DistanceMetric: " - << static_cast( metric ); - throw std::runtime_error( errMsg.str() ); - } - - return distance; -} - -#ifdef DEBUG_ALL -//---------------------------------------------------------------- -// -//---------------------------------------------------------------- -void PrintDataFrameIn( const DataFrame &dataFrame, - const Parameters ¶meters ) -{ - std::cout << "FindNeighbors(): library:" << std::endl; - for ( size_t row = 0; row < parameters.library.size(); row++ ) { - size_t row_i = parameters.library[row]; - std::cout << "row " << row_i << " : "; - for ( size_t col = 0; col < dataFrame.NColumns(); col++ ) { - std::cout << dataFrame(row_i,col) << " "; - } std::cout << std::endl; - } - std::cout << "FindNeighbors(): prediction:" << std::endl; - for ( size_t row = 0; row < parameters.prediction.size(); row++ ) { - size_t row_i = parameters.prediction[row]; - std::cout << "row " << row_i << " : "; - for ( size_t col = 0; col < dataFrame.NColumns(); col++ ) { - std::cout << dataFrame(row_i,col) << " "; - } std::cout << std::endl; - } -} - -//---------------------------------------------------------------- -// -//---------------------------------------------------------------- -void PrintNeighborsOut( const Neighbors &neighbors ) -{ - std::cout << "FindNeighbors(): neighbors:distances" << std::endl; - for ( size_t i = 0; i < neighbors.neighbors.NRows(); i++ ) { - std::cout << "Row " << i << " | "; - for ( size_t j = 0; j < neighbors.neighbors.NColumns(); j++ ) { - std::cout << neighbors.neighbors( i, j ) << " "; - } std::cout << " : "; - for ( size_t j = 0; j < neighbors.neighbors.NColumns(); j++ ) { - std::cout << neighbors.distances( i, j ) << " "; - } std::cout << std::endl; - } -} -#endif diff --git a/src/Neighbors.h b/src/Neighbors.h deleted file mode 100644 index e344ff0..0000000 --- a/src/Neighbors.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef NEIGHBORS_H -#define NEIGHBORS_H - -#include -#include -#include - -#include "Common.h" -#include "Parameter.h" - -// Return structure of FindNeighbors() & Distances() -struct Neighbors { - DataFrame neighbors; - DataFrame distances; - - bool anyTies; // Are there ties? - std::vector< bool > ties; // true : false for each prediction row - std::vector< std::vector< std::pair< double, size_t > > > tiePairs; - - Neighbors(); - ~Neighbors(); -}; - -// Prototypes -Neighbors Distances( const DataFrame< double > &dataBlock, - Parameters param ); - -Neighbors FindNeighbors( DataFrame dataFrame, - Parameters parameters ); - -void PrintDataFrameIn( const DataFrame &dataFrame, - const Parameters ¶meters ); - -void PrintNeighborsOut( const Neighbors &neighbors ); - -double Distance( const std::valarray &v1, - const std::valarray &v2, - DistanceMetric metric ); - -#endif diff --git a/src/Parameter.cc b/src/Parameter.cc index 62347a9..df98a6a 100644 --- a/src/Parameter.cc +++ b/src/Parameter.cc @@ -3,16 +3,18 @@ //---------------------------------------------------------------- // Constructor -// Default values set in Parameter.h +// Default values set in Parameter.h declaration //---------------------------------------------------------------- Parameters::Parameters( Method method, std::string pathIn, std::string dataFile, std::string pathOut, - std::string predictFile, + std::string predictOutputFile, + std::string lib_str, std::string pred_str, + int E, int Tp, int knn, @@ -22,42 +24,44 @@ Parameters::Parameters( std::string columns_str, std::string target_str, - + bool embedded, bool const_predict, bool verbose, - - std::string SmapFile, - std::string blockFile, - std::string derivatives_str, - - double svdSig, - double tikhonov, - double elasticNet, - - int multi, + + std::string SmapOutputFile, + std::string blockOutputFile, + + int multiviewEnsemble, + int multiviewD, + bool multiviewTrainLib, + std::string libSizes_str, - int sample, - bool random, + int subSamples, + bool randomLib, bool replacement, - unsigned rseed, - bool noNeigh + unsigned seed, + bool includeData, + bool colToTargetFlag, + bool noNeighborLimit ) : // Variable initialization from Parameters arguments method ( method ), pathIn ( pathIn ), dataFile ( dataFile ), pathOut ( pathOut ), - predictOutputFile( predictFile ), + predictOutputFile( predictOutputFile ), + lib_str ( lib_str ), pred_str ( pred_str ), + E ( E ), Tp ( Tp ), knn ( knn ), tau ( tau ), theta ( theta ), exclusionRadius ( exclusionRadius ), - + columns_str ( columns_str ), target_str ( target_str ), targetIndex ( 0 ), @@ -65,32 +69,32 @@ Parameters::Parameters( embedded ( embedded ), const_predict ( const_predict ), verbose ( verbose ), - - SmapOutputFile ( SmapFile ), - blockOutputFile ( blockFile ), - - derivatives_str ( derivatives_str ), - SVDSignificance ( svdSig ), - TikhonovAlpha ( tikhonov ), - ElasticNetAlpha ( elasticNet ), - - MultiviewEnsemble( multi ), + + SmapOutputFile ( SmapOutputFile ), + blockOutputFile ( blockOutputFile ), + + multiviewEnsemble( multiviewEnsemble ), + multiviewD ( multiviewD ), + multiviewTrainLib( multiviewTrainLib ), + libSizes_str ( libSizes_str ), - subSamples ( sample ), - randomLib ( random ), + subSamples ( subSamples ), + randomLib ( randomLib ), replacement ( replacement ), - seed ( rseed ), - noNeighborLimit ( noNeigh ), + seed ( seed ), + includeData ( includeData ), + colToTargetFlag ( colToTargetFlag ), + noNeighborLimit ( noNeighborLimit ), // Set validated flag and instantiate Version validated ( false ), - version ( 1, 3, 8, "2020-05-23" ) + version ( 2, 0, 0, "2020-06-03" ) { // Constructor code if ( method != Method::None ) { Validate(); - + if ( verbose ) { version.ShowVersion(); } @@ -102,11 +106,6 @@ Parameters::Parameters( //---------------------------------------------------------------- Parameters::~Parameters() {} -//---------------------------------------------------------------- -// -//---------------------------------------------------------------- -void Parameters::Load() {} - //---------------------------------------------------------------- // Index offsets, generate library and prediction indices, // and parameter validation @@ -146,9 +145,9 @@ void Parameters::Validate() { for ( auto thisPair : libPairs ) { size_t lib_start = thisPair.first; size_t lib_end = thisPair.second; - + nLib += lib_end - lib_start + 1; - + // Validate end > stop indices if ( method == Method::Simplex or method == Method::SMap ) { // Don't check if method == None, Embed or CCM since default @@ -198,9 +197,9 @@ void Parameters::Validate() { for ( auto thisPair : predPairs ) { size_t pred_start = thisPair.first; size_t pred_end = thisPair.second; - + nPred += pred_end - pred_start + 1; - + // Validate end > stop indices if ( method == Method::Simplex or method == Method::SMap ) { // Don't check if method == None, Embed or CCM since default @@ -224,7 +223,7 @@ void Parameters::Validate() { } } } - + if ( method == Method::Simplex or method == Method::SMap ) { if ( not library.size() ) { std::string errMsg( "Parameters::Validate(): " @@ -246,7 +245,7 @@ void Parameters::Validate() { prediction = std::vector( 1, 0 ); } } - + #ifdef DEBUG_ALL PrintIndices( library, prediction ); #endif @@ -254,23 +253,23 @@ void Parameters::Validate() { //-------------------------------------------------------------- // Convert multi argument parameters from string to vectors //-------------------------------------------------------------- - + // Columns // If columns are purely integer, then populate vector columnIndex // Otherwise fill in vector columnNames if ( columns_str.size() ) { - + std::vector columns_vec = SplitString( columns_str, " \t,\n" ); - + bool onlyDigits = false; - + for ( auto ci = columns_vec.begin(); ci != columns_vec.end(); ++ci ) { onlyDigits = OnlyDigits( *ci, true ); if ( not onlyDigits ) { break; } } - + if ( onlyDigits ) { for ( auto ci = columns_vec.begin(); ci != columns_vec.end(); ++ci ) { @@ -288,7 +287,7 @@ void Parameters::Validate() { << " No valid columns found." << std::endl; throw std::runtime_error( errMsg.str() ); } - + // target if ( target_str.size() ) { bool onlyDigits = OnlyDigits( target_str, true ); @@ -299,20 +298,6 @@ void Parameters::Validate() { targetName = target_str; } } - - // Derivatives - if ( derivatives_str.size() > 0 ) { - std::vector der_vec = SplitString(derivatives_str," \t,"); - if ( der_vec.size() < 2 ) { - std::string errMsg( "Parameters::Validate(): " - "derivatives must be integer pairs.\n"); - throw std::runtime_error( errMsg ); - } - derivatives = std::vector( der_vec.size() ); - for ( size_t i = 0; i < der_vec.size(); i++ ) { - derivatives.push_back( std::stoi( der_vec[i] ) ); - } - } //-------------------------------------------------------------------- // CCM sample not 0 if random is true @@ -334,7 +319,7 @@ void Parameters::Validate() { "CCM librarySizes must be three integers.\n" ); throw std::runtime_error( errMsg ); } - + size_t start = std::stoi( libsize_vec[0] ); size_t stop = std::stoi( libsize_vec[1] ); size_t increment = std::stoi( libsize_vec[2] ); @@ -346,7 +331,7 @@ void Parameters::Validate() { << " is invalid.\n"; throw std::runtime_error( errMsg.str() ); } - + if ( start > stop ) { std::stringstream errMsg; errMsg << "Parameters::Validate(): " @@ -354,16 +339,16 @@ void Parameters::Validate() { << " stop " << stop << " are invalid.\n"; throw std::runtime_error( errMsg.str() ); } - + size_t N_lib = std::floor( (stop-start)/increment + 1/increment ) + 1; - if ( start < E ) { + if ( (int) start < E ) { std::stringstream errMsg; errMsg << "Parameters::Validate(): " << "CCM librarySizes start < E = " << E << "\n"; throw std::runtime_error( errMsg.str() ); } - else if ( start < 3 ) { + else if ( (int) start < 3 ) { std::string errMsg( "Parameters::Validate(): " "CCM librarySizes start < 3.\n" ); throw std::runtime_error( errMsg ); @@ -444,43 +429,6 @@ void Parameters::Validate() { "data/dimension correspondance.\n" ); std::cout << msg; } - - // S-Map coefficient columns for derivatives start at 1 since the 0th - // column is the S-Map linear prediction bias term - if ( derivatives.size() > 1 ) { - std::vector::iterator it = std::find( derivatives.begin(), - derivatives.end(), 0); - if ( it != derivatives.end() ) { - std::string errMsg( "Parameters::Validate() S-Map coefficient " - " columns for derivatives can not use column 0.\n"); - throw std::runtime_error( errMsg ); - } - if ( derivatives.size() % 2 ) { - std::string errMsg( "Parameters::Validate() S-Map coefficient " - " columns for derivatives must be in pairs.\n"); - throw std::runtime_error( errMsg ); - } - } - - // Tikhonov and ElasticNet are mutually exclusive - if ( TikhonovAlpha and ElasticNetAlpha ) { - std::string errMsg( "Parameters::Validate() Multiple S-Map solve " - "methods specified. Use one or none of: " - "tikhonov, elasticNet.\n"); - throw std::runtime_error( errMsg ); - } - - // Very small alphas don't make sense in elastic net - if ( ElasticNetAlpha < 0.01 ) { - std::cout << "Parameters::Validate() ElasticNetAlpha too small." - " Setting to 0.01."; - ElasticNetAlpha = 0.01; - } - if ( ElasticNetAlpha > 1 ) { - std::cout << "Parameters::Validate() ElasticNetAlpha too large." - " Setting to 1."; - ElasticNetAlpha = 1; - } } else if ( method == Method::Embed ) { // no-op @@ -531,7 +479,7 @@ void Parameters::DeleteLibPred() { break; } } - + bool deletePredIndex = false; for ( auto element = deleted_pred_elements.begin(); element != deleted_pred_elements.end(); element++ ) { @@ -541,9 +489,9 @@ void Parameters::DeleteLibPred() { break; } } - + #ifdef DEBUG_ALL - std::cout << "DeleteLibPred(): Nrows: " << NRows + std::cout << "DeleteLibPred(): shift: " << shift << " deleteLibIndex: " << deleteLibIndex << " deletePredIndex: " << deletePredIndex << std::endl; std::cout << " deleted_lib_elements: "; @@ -556,12 +504,12 @@ void Parameters::DeleteLibPred() { std::cout << *element << ", "; } std::cout << std::endl; #endif - + // Erase elements of row indices that were deleted if ( deleteLibIndex ) { for ( auto element = deleted_lib_elements.begin(); element != deleted_lib_elements.end(); element++ ) { - + std::vector< size_t >::iterator it; it = std::find( library.begin(), library.end(), *element ); @@ -574,17 +522,17 @@ void Parameters::DeleteLibPred() { if ( deletePredIndex ) { for ( auto element = deleted_pred_elements.begin(); element != deleted_pred_elements.end(); element++ ) { - + std::vector< size_t >::iterator it; it = std::find( prediction.begin(), prediction.end(), *element ); - + if ( it != prediction.end() ) { prediction.erase( it ); } } } - + // Now offset all values by shift so that vectors indices // in library and prediction refer to the same data rows // before the deletion/shift. @@ -615,7 +563,7 @@ std::ostream& operator<< ( std::ostream &os, Parameters &p ) { else if ( p.method == Method::CCM ) { method = "CCM"; } else if ( p.method == Method::None ) { method = "None"; } else if ( p.method == Method::Embed ) { method = "Embed"; } - + os << "Method: " << method << " E=" << p.E << " Tp=" << p.Tp << " knn=" << p.knn << " tau=" << p.tau << " theta=" << p.theta @@ -639,9 +587,8 @@ std::ostream& operator<< ( std::ostream &os, Parameters &p ) { << p.prediction[ p.prediction.size() - 1 ] << "] " << std::endl; - os << "-------------------------------------------------------\n"; - + return os; } diff --git a/src/Parameter.h b/src/Parameter.h index 72f9ff9..071dad6 100644 --- a/src/Parameter.h +++ b/src/Parameter.h @@ -12,15 +12,16 @@ //------------------------------------------------------------ class Parameters { -public: // Not protected with accessors. - Method method; // Simplex or SMap enum class +public: // No need for protected or private + Method method; // Simplex or SMap enum class - std::string pathIn; // path for input dataFile - std::string dataFile; // input dataFile (assumed .csv) - std::string pathOut; // path for output files - std::string predictOutputFile;// - std::string lib_str; // multi argument parameters for library - std::string pred_str; // multi argument parameters for prediction + std::string pathIn; // path for input dataFile + std::string dataFile; // input dataFile (assumed .csv) + std::string pathOut; // path for output files + std::string predictOutputFile; // + + std::string lib_str; // multi argument parameters for library + std::string pred_str; // multi argument parameters for prediction std::vector library; // library row indices std::vector prediction; // prediction row indices @@ -31,92 +32,89 @@ class Parameters { int tau; // block embedding delay double theta; // S Map localization int exclusionRadius; // temporal rows to ignore in predict - - std::string columns_str; - std::string target_str; - std::vector columnNames; // column names - std::vector columnIndex; // column indices - std::string targetName; // target column name - size_t targetIndex; // target column index + std::string columns_str; + std::string target_str; + std::vector< std::string > columnNames; // column names + std::vector< size_t > columnIndex; // column indices - bool embedded; // true if data is already embedded/block - bool const_predict; // true to compute non "predictor" stats - bool verbose; + std::string targetName; // target column name + size_t targetIndex; // target column index - std::string SmapOutputFile; // - std::string blockOutputFile; // Embed() output file + bool embedded; // true if data is already embedded/block + bool const_predict; // true to compute non "predictor" stats + bool verbose; - std::string derivatives_str; - std::vector derivatives;// list of column indices for derivatives - - double SVDSignificance; // SVD singular value cutoff - double TikhonovAlpha; // Initial alpha parameter - double ElasticNetAlpha; // Initial alpha parameter + std::string SmapOutputFile; // + std::string blockOutputFile; // Embed() output file - int MultiviewEnsemble; // Number of ensembles in multiview + int multiviewEnsemble; // Number of ensembles in multiview + int multiviewD; // Multiview state-space dimension + bool multiviewTrainLib; // Use prediction as training library std::string libSizes_str; - std::vector librarySizes;// CCM library sizes to evaluate + std::vector< size_t > librarySizes;// CCM library sizes to evaluate int subSamples; // CCM number of samples to draw bool randomLib; // CCM randomly select subsets if true bool replacement; // CCM random select with replacement if true unsigned seed; // CCM random selection RNG seed - + bool includeData; // CCM include all simplex projection results + bool colToTargetFlag; // CCM thread flag to select output objects + bool noNeighborLimit; // Strictly forbid neighbors outside library bool validated; - - Version version; // Version object, instantiated in constructor - friend std::ostream& operator<<(std::ostream &os, Parameters ¶ms); + Version version; // Version object, instantiated in constructor + + friend std::ostream& operator<<( std::ostream & os, Parameters & params ); // Constructor declaration and default arguments Parameters( - Method method = Method::None, - std::string pathIn = "./", - std::string dataFile = "", - std::string pathOut = "./", - std::string predictFile = "", - std::string lib_str = "", - std::string pred_str = "", - int E = 0, - int Tp = 0, - int knn = 0, - int tau = -1, - double theta = 0, - int exclusionRadius = 0, - - std::string columns_str = "", - std::string target_str = "", - - bool embedded = false, - bool const_predict = false, - bool verbose = false, - - std::string SmapFile = "", - std::string blockFile = "", - std::string derivatives_str = "", - - double svdSig = 1E-5, - double tikhonov = 0, - double elasticNet = 0.1, - - int multi = 0, - std::string libSizes_str = "", - int sample = 0, - bool random = true, - bool replacement = false, - unsigned seed = 0, // 0: Generate random seed in CCM - bool noNeighbor = false + Method method = Method::None, + std::string pathIn = "./", + std::string dataFile = "", + std::string pathOut = "./", + std::string predictOutputFile = "", + + std::string lib_str = "", + std::string pred_str = "", + + int E = 0, + int Tp = 0, + int knn = 0, + int tau = -1, + double theta = 0, + int exclusionRadius = 0, + + std::string columns_str = "", + std::string target_str = "", + + bool embedded = false, + bool const_predict = false, + bool verbose = false, + + std::string SmapOutputFile = "", + std::string blockOutputFile = "", + + int multiviewEnsemble = 0, + int multiviewD = 0, + bool multiviewTrainLib = true, + + std::string libSizes_str = "", + int subSamples = 0, + bool randomLib = true, + bool replacement = false, + unsigned seed = 0, // 0: Generate random seed in CCM + bool includeData = false, + bool colToTargetFlag = true, + bool noNeighborLimit = false ); ~Parameters(); - void Validate(); // Parameter validation and index offsets - void Load(); // Populate the parameters from arguments + void Validate(); // Parameter validation and index offsets void DeleteLibPred(); // Adjust for embedding - void PrintIndices( std::vector library, - std::vector prediction ); + void PrintIndices( std::vector< size_t > library, + std::vector< size_t > prediction ); }; - #endif diff --git a/src/SMap.cc b/src/SMap.cc index 42e0d89..e03bf81 100644 --- a/src/SMap.cc +++ b/src/SMap.cc @@ -1,250 +1,102 @@ -#include "Common.h" -#include "Parameter.h" -#include "Embed.h" -#include "Neighbors.h" -#include "AuxFunc.h" - -// forward declarations -std::valarray< double > Lapack_SVD( int m, - int n, - double *a, - double *b, - double rcond ); - -//--------------------------------------------------------------------- -// Overload 1: Explicit data file path/name with internal SVD (LAPACK) -// Implemented as a wrapper to API overload 2 -> 4 -//--------------------------------------------------------------------- -SMapValues SMap( std::string pathIn, - std::string dataFile, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int E, - int Tp, - int knn, - int tau, - double theta, - int exclusionRadius, - std::string columns, - std::string target, - std::string smapFile, - std::string derivatives, - bool embedded, - bool const_predict, - bool verbose ) -{ - // DataFrame constructor loads data - DataFrame< double > dataFrameIn( pathIn, dataFile ); - - // Call overload 2 with the dataFrameIn - SMapValues SMapOutput = SMap( dataFrameIn, pathOut, predictFile, - lib, pred, E, Tp, knn, tau, theta, - exclusionRadius, - columns, target, smapFile, derivatives, - embedded, const_predict, verbose ); - return SMapOutput; -} +#include "SMap.h" + +// NOTE: Contains SMapClass method implementations, AND: +// General SVD functions: SVD, Lapack_SVD, dgelss_ //---------------------------------------------------------------- -// Overload 2: DataFrame provided with internal SVD (LAPACK) -// Implemented as a wrapper to API overload 4 +// Constructor //---------------------------------------------------------------- -SMapValues SMap( DataFrame< double > &data, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int E, - int Tp, - int knn, - int tau, - double theta, - int exclusionRadius, - std::string columns, - std::string target, - std::string smapFile, - std::string derivatives, - bool embedded, - bool const_predict, - bool verbose ) { - - // Call overload 4 with default SVD function (below) - SMapValues SMapOutput = SMap( data, pathOut, predictFile, - lib, pred, E, Tp, knn, tau, theta, - exclusionRadius, - columns, target, smapFile, derivatives, - &SVD, - embedded, const_predict, verbose); - - return SMapOutput; +SMapClass::SMapClass ( + DataFrame< double > & data, + Parameters & parameters ): + EDM{ data, parameters } { } //---------------------------------------------------------------- -// Overload 3: Explicit data file path/name and solver -// Implemented as a wrapper to API overload 4 +// Project : Polymorphic implementation //---------------------------------------------------------------- -SMapValues SMap( std::string pathIn, - std::string dataFile, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int E, - int Tp, - int knn, - int tau, - double theta, - int exclusionRadius, - std::string columns, - std::string target, - std::string smapFile, - std::string derivatives, - std::valarray (*solver) (DataFrame < double >, - std::valarray < double > ), - bool embedded, - bool const_predict, - bool verbose ) -{ - // DataFrame constructor loads data - DataFrame< double > dataFrameIn( pathIn, dataFile ); +void SMapClass::Project ( Solver solver ) { - // Call overload 4 with dataFrameIn and solver object - SMapValues SMapOutput = SMap( dataFrameIn, pathOut, predictFile, - lib, pred, E, Tp, knn, tau, theta, - exclusionRadius, - columns, target, smapFile, derivatives, - solver, embedded, const_predict, verbose ); - return SMapOutput; + PrepareEmbedding(); + + Distances(); // all pred : lib vector distances into allDistances + + FindNeighbors(); + + SMap( solver ); + + FormatOutput(); // Common formatting + + WriteOutput(); // SMap specific formatting & output } //---------------------------------------------------------------- -// Overload 4: Solver & DataFrame provided +// SMap algorithm //---------------------------------------------------------------- -SMapValues SMap( DataFrame< double > &data, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int E, - int Tp, - int knn, - int tau, - double theta, - int exclusionRadius, - std::string columns, - std::string target, - std::string smapFile, - std::string derivatives, - std::valarray (*solver) (DataFrame < double >, - std::valarray < double > ), - bool embedded, - bool const_predict, - bool verbose ) -{ +void SMapClass::SMap ( Solver solver ) { - Parameters param = Parameters( Method::SMap, "", "", - pathOut, predictFile, - lib, pred, E, Tp, knn, tau, theta, - exclusionRadius, columns, target, - embedded, const_predict, verbose, - smapFile, "", derivatives ); - - //---------------------------------------------------------- - // Load data, Embed, compute Neighbors - //---------------------------------------------------------- - DataEmbedNN dataEmbedNN = EmbedNN( &data, std::ref( param ) ); - - // Unpack the dataEmbedNN for convenience - DataFrame *dataInRef = dataEmbedNN.dataIn; - DataFrame dataBlock = dataEmbedNN.dataFrame; - std::valarray target_vec = dataEmbedNN.targetVec; - Neighbors neighbors = dataEmbedNN.neighbors; + // Allocate output vectors to populate EDM class projections DataFrame. + // Must be after FindNeighbors() + size_t Npred = knn_neighbors.NRows(); - DataFrame &dataIn = std::ref( *dataInRef ); - - //---------------------------------------------------------- - // SMap projection - //---------------------------------------------------------- - size_t library_N_row = param.library.size(); - size_t predict_N_row = param.prediction.size(); - size_t N_row = neighbors.neighbors.NRows(); - - auto max_lib_it = std::max_element( param.library.begin(), - param.library.end() ); - size_t max_lib_index = *max_lib_it; + predictions = std::valarray< double > ( 0., Npred ); + const_predictions = std::valarray< double > ( 0., Npred ); + variance = std::valarray< double > ( 0., Npred ); - if ( predict_N_row != N_row ) { - std::stringstream errMsg; - errMsg << "SMap(): Number of prediction rows (" << predict_N_row - << ") does not match the number of neighbor rows (" - << N_row << ").\n"; - throw std::runtime_error( errMsg.str() ); - } - if ( neighbors.distances.NColumns() != param.knn ) { - std::stringstream errMsg; - errMsg << "SMap(): Number of neighbor columns (" - << neighbors.distances.NColumns() - << ") does not match knn (" << param.knn << ").\n"; - throw std::runtime_error( errMsg.str() ); - } - - std::valarray< double > predictions = std::valarray< double >( N_row ); - std::valarray< double > variance = std::valarray< double >( N_row ); - // Init coefficients to NAN ? - DataFrame< double > coefficients = DataFrame< double >( N_row, - param.E + 1 ); - DataFrame< double > derivative; - DataFrame< double > tangents; - - //------------------------------------------------------------ - // Process each prediction row - //------------------------------------------------------------ - for ( size_t row = 0; row < N_row; row++ ) { + // Allocate +Tp rows. Coefficients/nan will be shifted in Project() + coefficients = DataFrame< double >( Npred + abs( parameters.Tp ), + parameters.E + 1 ); + + auto maxLibit = std::max_element( parameters.library.begin(), + parameters.library.end() ); + int maxLibIndex = *maxLibit; // int for compare to libRow int + + // Process each prediction row in neighbors : distances + for ( size_t row = 0; row < Npred; row++ ) { - double D_avg = neighbors.distances.Row( row ).sum() / param.knn; + double D_avg = knn_distances.Row( row ).sum() / parameters.knn; // Compute weight vector - std::valarray< double > w = std::valarray< double >( param.knn ); - if ( param.theta > 0 ) { - w = std::exp( (-param.theta/D_avg) * neighbors.distances.Row(row) ); + std::valarray< double > w = std::valarray< double >( parameters.knn ); + if ( parameters.theta > 0 ) { + w = std::exp( (-parameters.theta / D_avg) * knn_distances.Row(row) ); } else { - w = std::valarray< double >( 1, param.knn ); + w = std::valarray< double >( 1, parameters.knn ); } - DataFrame< double > A = DataFrame< double >(param.knn, param.E + 1); - std::valarray< double > B = std::valarray< double >( param.knn ); + DataFrame< double > A = DataFrame< double >( parameters.knn, + parameters.E + 1 ); + std::valarray< double > B = std::valarray< double >( parameters.knn ); // Populate matrix A (exp weighted future prediction), and // vector B (target BC's) for this row (observation). - int lib_row; - size_t lib_row_base; + int libRow; + size_t libRowBase; - for ( size_t k = 0; k < param.knn; k++ ) { - lib_row_base = neighbors.neighbors( row, k ); - lib_row = lib_row_base + param.Tp; + for ( int k = 0; k < parameters.knn; k++ ) { + libRowBase = knn_neighbors( row, k ); + libRow = libRowBase + parameters.Tp; - if ( lib_row > max_lib_index ) { + if ( libRow > maxLibIndex ) { // The knn index + Tp is outside the library domain // Can only happen if noNeighborLimit = true is used. - if ( param.verbose ) { + if ( parameters.verbose ) { std::stringstream msg; - msg << "SMap() in row " << row << " libRow " << lib_row + msg << "SMap() in row " << row << " libRow " << libRow << " exceeds library domain.\n"; std::cout << msg.str(); } // Use the neighbor at the 'base' of the trajectory - B[ k ] = target_vec[ lib_row_base ]; + B[ k ] = target[ libRowBase ]; } - else if ( lib_row < 0 ) { - B[ k ] = target_vec[ 0 ]; + else if ( libRow < 0 ) { + B[ k ] = target[ 0 ]; } else { - B[ k ] = target_vec[ lib_row ]; + B[ k ] = target[ libRow ]; } //--------------------------------------------------------------- @@ -252,14 +104,14 @@ SMapValues SMap( DataFrame< double > &data, //--------------------------------------------------------------- // NOTE: The matrix A has a (weighted) constant (1) first column // to enable a linear intercept/bias term. - // NOTE: The dataBlock does not have a time vector, and only + // NOTE: The embedding does not have a time vector, and only // has columns from the embedding. So the coefficient - // matrix A has E+1 columns, while the dataBlock has E. + // matrix A has E+1 columns, while the embedding has E. //--------------------------------------------------------------- A( k, 0 ) = w[ k ]; // Intercept bias terms in column 0 (weighted) - for ( size_t j = 1; j < param.E + 1; j++ ) { - A( k, j ) = w[k] * dataBlock( lib_row_base, j-1 ); + for ( int j = 1; j < parameters.E + 1; j++ ) { + A( k, j ) = w[ k ] * embedding( libRowBase, j - 1 ); } } @@ -271,9 +123,9 @@ SMapValues SMap( DataFrame< double > &data, // Prediction is local linear projection double prediction = C[ 0 ]; // C[ 0 ] is the bias term - for ( size_t e = 1; e < param.E + 1; e++ ) { + for ( int e = 1; e < parameters.E + 1; e++ ) { prediction = prediction + - C[ e ] * dataBlock( param.prediction[ row ], e-1 ); + C[ e ] * embedding( parameters.prediction[ row ], e-1 ); } predictions[ row ] = prediction; @@ -284,90 +136,71 @@ SMapValues SMap( DataFrame< double > &data, std::pow( B - predictions[ row ], 2); variance[ row ] = ( w * deltaSqr ).sum() / w.sum(); - } // for ( row = 0; row < predict_N_row; row++ ) - + } // for ( row = 0; row < Npred; row++ ) + // non "predictions" X(t+1) = X(t) if const_predict specified - std::valarray< double > const_predictions( 0., N_row ); - if ( param.const_predict ) { + const_predictions = std::valarray< double >( 0., Npred ); + if ( parameters.const_predict ) { std::slice pred_slice = - std::slice( param.prediction[ 0 ], param.prediction.size(), 1 ); + std::slice( parameters.prediction[ 0 ], + parameters.prediction.size(), 1 ); - const_predictions = target_vec[ pred_slice ]; + const_predictions = target[ pred_slice ]; } - - //----------------------------------------------------- - // Derivatives - //----------------------------------------------------- - - - //---------------------------------------------------- - // Ouput - //---------------------------------------------------- - // Observations & predictions: Adjust rows/nan for Tp - DataFrame dataOut = FormatOutput( param, - predictions, - const_predictions, - variance, - target_vec, - dataIn.Time(), - dataIn.TimeName() ); - - // Coefficient output DataFrame: N_row + Tp rows - DataFrame< double > coefOut = DataFrame< double >( dataOut.NRows(), - param.E + 1 ); - - // Set time from: dataIn -> FormatOutput() -> FillTimes() -> dataOut - if ( dataOut.Time().size() ) { - coefOut.Time() = dataOut.Time(); - coefOut.TimeName() = dataOut.TimeName(); +} + +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +void SMapClass::WriteOutput () { + + // Process SMap coefficients output + // Set time from: data -> FormatOutput() -> FillTimes() -> projection + if ( projection.Time().size() ) { + coefficients.Time() = projection.Time(); + coefficients.TimeName() = projection.TimeName(); } // else { throw ? } JP - // Populate coefOut column names: C0, C1, C2, ... + // coefficients column names: C0, C1, C2, ... std::vector coefNames; for ( size_t col = 0; col < coefficients.NColumns(); col++ ) { std::stringstream coefName; coefName << "C" << col; coefNames.push_back( coefName.str() ); } - coefOut.ColumnNames() = coefNames; + coefficients.ColumnNames() = coefNames; - // coefficients have N_row's; coefOut has N_row + Tp + // coefficients has Npred + Tp rows, but coef were written in first Npred // Create coefficient column vector with Tp nan rows at the - // beginning/end of coefOut as in FormatOutput() - std::valarray coefColumnVec( NAN, dataOut.NRows() ); + // beginning/end of coefficients as in FormatOutput() + std::valarray< double > coefColumnVec( NAN, coefficients.NRows() ); - // Copy coefficients vectors into coefOut - std::slice coef_i; - if ( param.Tp > -1 ) { - coef_i = std::slice( param.Tp, N_row, 1 ); + // Copy/shift coefficients vectors + std::slice slice_in = std::slice( 0, knn_neighbors.NRows(), 1 ); + std::slice slice_out; + if ( parameters.Tp > -1 ) { + slice_out = std::slice( parameters.Tp, knn_neighbors.NRows(), 1 ); } else { - coef_i = std::slice( 0, N_row + param.Tp, 1 ); + slice_out = std::slice( 0, knn_neighbors.NRows() + parameters.Tp, 1 ); } - for ( size_t col = 0; col < coefOut.NColumns(); col++ ) { - coefColumnVec[ coef_i ] = coefficients.Column( col ); - coefOut.WriteColumn( col, coefColumnVec ); + for ( size_t col = 0; col < coefficients.NColumns(); col++ ) { + coefColumnVec[ slice_out ] = coefficients.Column( col )[ slice_in ]; + coefficients.WriteColumn( col, coefColumnVec ); } - if ( param.predictOutputFile.size() ) { - // Write predictions to disk - dataOut.WriteData( param.pathOut, param.predictOutputFile ); + if ( parameters.predictOutputFile.size() ) { + projection.WriteData( parameters.pathOut, + parameters.predictOutputFile ); } - if ( param.SmapOutputFile.size() ) { - // Write Smap coefficients to disk - coefOut.WriteData( param.pathOut, param.SmapOutputFile ); + if ( parameters.SmapOutputFile.size() ) { + coefficients.WriteData( parameters.pathOut, parameters.SmapOutputFile ); } - - SMapValues values = SMapValues(); - values.predictions = dataOut; - values.coefficients = coefOut; - - return values; } //---------------------------------------------------------------- -// Singular Value Decomposition +// Singular Value Decomposition : wrapper for Lapack_SVD() //---------------------------------------------------------------- std::valarray < double > SVD( DataFrame < double > A, std::valarray< double > B ) { @@ -397,7 +230,7 @@ std::valarray < double > SVD( DataFrame < double > A, } //------------------------------------------------------------------------- -// subroutine dgelss() +// subroutine dgelss() : LAPACK function call in Lapack_SVD() //----------------------------------------------------------------------- // DGELSS computes the minimum norm solution to a real linear least // squares problem: @@ -443,11 +276,11 @@ extern "C" { } //----------------------------------------------------------------------- -// +// Wrapper for LAPACK dgelss_() //----------------------------------------------------------------------- -std::valarray< double > Lapack_SVD( int m, // number of rows in matrix - int n, // number of columns in matrix - double *a, // pointer to top-left corner +std::valarray< double > Lapack_SVD( int m, // rows in matrix + int n, // columns in matrix + double *a, // ptr to top-left double *b, double rcond ) { @@ -474,7 +307,7 @@ std::valarray< double > Lapack_SVD( int m, // number of rows in matrix std::cout << "m.row=" << m << " n.col=" << n << " lda=" << lda << " s.n=" << N_SingularValues << std::endl; - for ( size_t i = 0; i < m*n; i++ ) { + for ( int i = 0; i < m*n; i++ ) { std::cout << a[i] << " "; } std::cout << std::endl; #endif diff --git a/src/SMap.h b/src/SMap.h new file mode 100644 index 0000000..fc069ff --- /dev/null +++ b/src/SMap.h @@ -0,0 +1,37 @@ + +#ifndef EDM_SMAP_H +#define EDM_SMAP_H + +#include "EDM.h" + +// Prototype & alias of solver function pointer +using Solver = std::valarray< double > (*) ( DataFrame < double >, + std::valarray < double > ); + +// Prototype declaration of general functions +std::valarray < double > SVD( DataFrame < double > A, + std::valarray< double > B ); + +std::valarray< double > Lapack_SVD( int m, // number of rows in matrix + int n, // number of columns in matrix + double *a, // pointer to top-left corner + double *b, + double rcond ); + +//---------------------------------------------------------------- +// SMap class inherits from EDM class and defines +// SMap-specific projection & output methods +//---------------------------------------------------------------- +class SMapClass : public EDM { + +public: + // Constructor + SMapClass ( DataFrame & data, + Parameters & parameters ); + + // Method declarations + void Project( Solver ); + void SMap ( Solver ); + void WriteOutput(); +}; +#endif diff --git a/src/Simplex.cc b/src/Simplex.cc index c29bcf0..71d155b 100644 --- a/src/Simplex.cc +++ b/src/Simplex.cc @@ -1,154 +1,66 @@ -#include "Common.h" -#include "Parameter.h" -#include "Neighbors.h" -#include "Embed.h" -#include "AuxFunc.h" - -// Forward declaration -DataFrame SimplexProjection( Parameters param, - DataEmbedNN embedNN, - bool checkDataRows = true ); +#include "Simplex.h" //---------------------------------------------------------------- -// API Overload 1: Explicit data file path/name -// Implemented as a wrapper to API Overload 2: +// Constructor //---------------------------------------------------------------- -DataFrame Simplex( std::string pathIn, - std::string dataFile, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int E, - int Tp, - int knn, - int tau, - int exclusionRadius, - std::string columns, - std::string target, - bool embedded, - bool const_predict, - bool verbose ) { - - // DataFrame constructor loads data - DataFrame< double > *dataFrameIn = - new DataFrame< double > ( pathIn, dataFile ); - - // Pass data frame to Simplex - DataFrame< double > S = Simplex( std::ref( *dataFrameIn ), - pathOut, - predictFile, - lib, - pred, - E, - Tp, - knn, - tau, - exclusionRadius, - columns, - target, - embedded, - const_predict, - verbose ); - delete dataFrameIn; - - return S; +SimplexClass::SimplexClass ( + DataFrame< double > & data, + Parameters & parameters ): + EDM{ data, parameters } { } //---------------------------------------------------------------- -// API Overload 2: DataFrame provided +// Project : Polymorphic implementation //---------------------------------------------------------------- -DataFrame Simplex( DataFrame< double > &data, - std::string pathOut, - std::string predictFile, - std::string lib, - std::string pred, - int E, - int Tp, - int knn, - int tau, - int exclusionRadius, - std::string columns, - std::string target, - bool embedded, - bool const_predict, - bool verbose ) { - - Parameters param = Parameters( Method::Simplex, "", "", - pathOut, predictFile, - lib, pred, E, Tp, knn, tau, 0, - exclusionRadius, - columns, target, embedded, - const_predict, verbose ); +void SimplexClass::Project () { + + PrepareEmbedding(); + + Distances(); // all pred : lib vector distances into allDistances + + FindNeighbors(); - //---------------------------------------------------------- - // Embed, compute Neighbors - //---------------------------------------------------------- - DataEmbedNN embedNN = EmbedNN( &data, std::ref( param ) ); + Simplex(); - DataFrame S = SimplexProjection( param, embedNN ); + FormatOutput(); - return S; + WriteOutput(); } //---------------------------------------------------------------- -// Simplex Projection +// Simplex algorithm //---------------------------------------------------------------- -DataFrame SimplexProjection( Parameters param, - DataEmbedNN embedNN, - bool checkDataRows ) { - - // Unpack the data, (embedding dataBlock not used), target & neighbors - DataFrame *dataIn = embedNN.dataIn; // used for output - std::valarray target_vec = embedNN.targetVec; - Neighbors neighbors = embedNN.neighbors; +void SimplexClass::Simplex () { - size_t library_N_row = param.library.size(); - size_t N_row = neighbors.neighbors.NRows(); - - auto max_lib_it = std::max_element( param.library.begin(), - param.library.end() ); - int max_lib_index = *max_lib_it; // int for compare to libRow int - -#ifdef DEBUG_ALL - std::cout << "SimplexProjection -------------------------\n"; - std::cout << "Neighbors: (" << neighbors.neighbors.NRows() << "x" - << neighbors.neighbors.NColumns() << ")\n"; - std::cout << neighbors.neighbors; - std::cout << "Target: (" << target_vec.size() << ")\n"; - for ( size_t row = 0; row < 10; row++ ) { - std::cout << target_vec[ row ] << " "; - } std::cout << std::endl; - std::cout << "-------------------------------------------\n\n"; -#endif + // Allocate output vectors to populate EDM class projections DataFrame. + // Must be after FindNeighbors() + size_t Npred = knn_neighbors.NRows(); + + predictions = std::valarray< double > ( 0., Npred ); + const_predictions = std::valarray< double > ( 0., Npred ); + variance = std::valarray< double > ( 0., Npred ); + + auto maxLibit = std::max_element( parameters.library.begin(), + parameters.library.end() ); + int maxLibIndex = *maxLibit; // int for compare to libRow int - if ( N_row != neighbors.distances.NRows() ) { - std::stringstream errMsg; - errMsg << "Simplex(): Number of neighbor rows " << N_row - << " doesn't match the number of distances rows " - << neighbors.distances.NRows() << std::endl; - throw std::runtime_error( errMsg.str() ); - } - double minWeight = 1.E-6; - std::valarray predictions( 0., N_row ); - std::valarray variance ( 0., N_row ); - // Process each prediction row in neighbors - for ( size_t row = 0; row < N_row; row++ ) { + // Process each prediction row in neighbors : distances + for ( size_t row = 0; row < Npred; row++ ) { - std::valarray distanceRow = neighbors.distances.Row( row ); + std::valarray< double > distanceRow = knn_distances.Row( row ); // Establish exponential weight reference, the 'distance scale' double minDistance = distanceRow.min(); - // Compute weight (vector) for each k_NN - std::valarray weightedDistances( minWeight, param.knn ); - + // Compute weight vector for each k_NN + std::valarray< double > weightedDistances( minWeight, parameters.knn ); + if ( minDistance == 0 ) { // Handle cases of distanceRow = 0 : can't divide by minDistance - for ( size_t i = 0; i < param.knn; i++ ) { + for ( int i = 0; i < parameters.knn; i++ ) { if ( distanceRow[i] > 0 ) { weightedDistances[i] = exp( -distanceRow[i] / minDistance ); } @@ -156,7 +68,7 @@ DataFrame SimplexProjection( Parameters param, // Setting weight = 1 implies that the corresponding // library target vector is the same as the observation // so it will be given full-weight in the prediction. - weightedDistances[i] = 1; + weightedDistances[ i ] = 1; } } } @@ -166,102 +78,102 @@ DataFrame SimplexProjection( Parameters param, } // weight vector - std::valarray weights( param.knn ); - for ( size_t i = 0; i < param.knn; i++ ) { + std::valarray< double > weights( parameters.knn ); + for ( int i = 0; i < parameters.knn; i++ ) { weights[i] = std::max( weightedDistances[i], minWeight ); } // target library vector, one element for each knn - std::valarray libTarget( param.knn ); + std::valarray< double > libTarget( parameters.knn ); - for ( size_t k = 0; k < param.knn; k++ ) { - int libRow = neighbors.neighbors( row, k ) + param.Tp; + for ( int k = 0; k < parameters.knn; k++ ) { + int libRow = knn_neighbors( row, k ) + parameters.Tp; - if ( libRow > max_lib_index ) { + if ( libRow > maxLibIndex ) { // The k_NN index + Tp is outside the library domain // Can only happen if noNeighborLimit = true is used. - if ( param.verbose ) { + if ( parameters.verbose ) { std::stringstream msg; msg << "Simplex() in row " << row << " libRow " << libRow << " exceeds library domain.\n"; std::cout << msg.str(); } // Use the neighbor at the 'base' of the trajectory - libTarget[ k ] = target_vec[ libRow - abs( param.Tp ) ]; + libTarget[ k ] = target[ libRow - abs( parameters.Tp ) ]; } else if ( libRow < 0 ) { - if ( param.verbose ) { + if ( parameters.verbose ) { std::stringstream msg; msg << "Simplex() in row " << row << " libRow " << libRow << " precedes library domain.\n"; std::cout << msg.str(); } // Use the neighbor at the 'base' of the trajectory - libTarget[ k ] = target_vec[ 0 ]; + libTarget[ k ] = target[ 0 ]; } else { - libTarget[ k ] = target_vec[ libRow ]; + libTarget[ k ] = target[ libRow ]; } } //------------------------------------------------------------------ // If ties, expand & adjust libTarget & weights //------------------------------------------------------------------ - if ( neighbors.anyTies ) { - if ( neighbors.ties[ row ] ) { + if ( anyTies ) { + if ( ties[ row ] ) { std::vector< std::pair< double, size_t > > rowTiePairs = - neighbors.tiePairs[ row ]; + tiePairs[ row ]; - double tieDistance = rowTiePairs[ 0 ].first; // all dist same... - size_t numTies = rowTiePairs.size(); - double tieFactor = 1; + size_t numTies = rowTiePairs.size(); + double tieFactor = 1; if ( numTies ) { tieFactor = 1 / double( numTies ); } // resize libTarget std::valarray< double > libTargetCopy( libTarget ); - libTarget.resize( param.knn + numTies ); // destroys contents + libTarget.resize( parameters.knn + numTies );// destroys contents // Copy original libTarget knn values - libTarget[ std::slice( 0, param.knn, 1 ) ] = libTargetCopy; + libTarget[ std::slice( 0, parameters.knn, 1 ) ] = libTargetCopy; // Add numTies values for ( size_t k2 = 0; k2 < rowTiePairs.size(); k2++ ) { - int libRow = rowTiePairs[ k2 ].second + param.Tp; - if ( libRow > max_lib_index ) { - libTarget[ k2 + param.knn ] = - target_vec[ libRow - abs( param.Tp ) ]; + int libRow = rowTiePairs[ k2 ].second + parameters.Tp; + if ( libRow > maxLibIndex ) { + libTarget[ k2 + parameters.knn ] = + target[ libRow - abs( parameters.Tp ) ]; } else if ( libRow < 0 ) { - libTarget[ k2 + param.knn ] = target_vec[ 0 ]; + libTarget[ k2 + parameters.knn ] = target[ 0 ]; } else { - libTarget[ k2 + param.knn ] = target_vec[ libRow ]; + libTarget[ k2 + parameters.knn ] = target[ libRow ]; } } // Weights // Resize distanceRow std::valarray distanceRowCopy( distanceRow ); - distanceRow.resize( param.knn + numTies ); // destroys contents + distanceRow.resize(parameters.knn + numTies);// destroys contents // Copy original distanceRow knn values - distanceRow[ std::slice( 0, param.knn, 1 ) ] = distanceRowCopy; + distanceRow[ std::slice( 0, parameters.knn, 1 ) ] = + distanceRowCopy; // Add numTies values - for ( size_t k2 = 0; k2 < rowTiePairs.size(); k2++ ) { - distanceRow[ k2 + param.knn ] = rowTiePairs[ k2 ].first; + for (size_t k2 = 0; k2 < rowTiePairs.size(); k2++ ) { + distanceRow[ k2 + parameters.knn ] = rowTiePairs[ k2 ].first; } minDistance = distanceRow.min(); // Resize weightedDistances std::valarray weightedDistancesCopy( weightedDistances ); - weightedDistances.resize( param.knn + numTies ); // destroys + weightedDistances.resize( parameters.knn + numTies ); // destroys if ( minDistance == 0 ) { // Handle cases of distanceRow = 0 - for ( size_t i = 0; i < param.knn + numTies; i++ ) { + for ( int i = 0; i < parameters.knn + (int) numTies; i++ ) { if ( distanceRow[i] > 0 ) { weightedDistances[i] = exp( -distanceRow[i] / minDistance ); @@ -277,33 +189,18 @@ DataFrame SimplexProjection( Parameters param, // Resize weights std::valarray weightsCopy( weights ); - weights.resize( param.knn + numTies ); // destroys + weights.resize( parameters.knn + numTies ); // destroys // Copy original knn weight values - weights[ std::slice( 0, param.knn, 1 ) ] = weightsCopy; - + weights[ std::slice( 0, parameters.knn, 1 ) ] = weightsCopy; + // Apply weight adjusment for ties - for ( size_t k2 = param.knn; k2 < weights.size(); k2++ ) { + for ( size_t k2 = parameters.knn; k2 < weights.size(); k2++ ) { weights[k2] = tieFactor * std::max( weightedDistances[k2], minWeight ); } - } - -#ifdef DEBUG_ALL - for ( size_t i = 0; i < neighbors.ties.size(); i++ ) { - if ( neighbors.ties[ i ] ) { - std::vector< std::pair< double, size_t > > rowTiePairs = - neighbors.tiePairs[ i ]; - std::cout << "Ties at pred_i " << i << ": "; - for ( size_t j = 0; j < rowTiePairs.size(); j++ ) { - double dist = rowTiePairs[ j ].first; - size_t prow = rowTiePairs[ j ].second; - std::cout << "[ " << dist << ", " << prow << "] "; - } std::cout << std::endl; - } - } -#endif - } + } // if ( ties[ row ] ) + } // if ( anyTies ) //------------------------------------------------------------------ //------------------------------------------------------------------ @@ -315,43 +212,30 @@ DataFrame SimplexProjection( Parameters param, std::pow(libTarget - predictions[ row ], 2); variance[ row ] = ( weights * deltaSqr ).sum() / weights.sum(); - } // for ( row = 0; row < N_row; row++ ) + } // for ( row = 0; row < Npred; row++ ) // non "predictions" X(t+1) = X(t) if const_predict specified - std::valarray< double > const_predictions( 0., N_row ); - if ( param.const_predict ) { + const_predictions = std::valarray< double > ( 0., Npred ); + if ( parameters.const_predict ) { std::slice pred_slice = - std::slice( param.prediction[ 0 ], param.prediction.size(), 1 ); + std::slice( parameters.prediction[ 0 ], + parameters.prediction.size(), 1 ); - const_predictions = target_vec[ pred_slice ]; + const_predictions = target[ pred_slice ]; } +} - //---------------------------------------------------- - // Ouput - //---------------------------------------------------- - DataFrame dataFrame = FormatOutput( param, - predictions, - const_predictions, - variance, - target_vec, - dataIn->Time(), - dataIn->TimeName() ); - - if ( param.predictOutputFile.size() ) { - // Write to disk - dataFrame.WriteData( param.pathOut, param.predictOutputFile ); +//---------------------------------------------------------------- +// +//---------------------------------------------------------------- +void SimplexClass::WriteOutput () { + if ( parameters.predictOutputFile.size() ) { + projection.WriteData( parameters.pathOut, + parameters.predictOutputFile ); } - -#ifdef DEBUG_ALL - std::cout << dataFrame; - VectorError ve = ComputeError( - dataFrame.VectorColumnName( "Observations" ), - dataFrame.VectorColumnName( "Predictions" ) ); - std::cout << "-------------------------------------------\n"; - std::cout << "rho " << ve.rho << " RMSE " << ve.RMSE - << " MAE " << ve.MAE << std::endl; - std::cout << "-------------------------------------------\n"; -#endif - - return dataFrame; } + +//---------------------------------------------------------------- +// Implemented in CCMClass +//---------------------------------------------------------------- +void SimplexClass::CopyData () {} diff --git a/src/Simplex.h b/src/Simplex.h new file mode 100644 index 0000000..72ae490 --- /dev/null +++ b/src/Simplex.h @@ -0,0 +1,29 @@ + +#ifndef EDM_SIMPLEX_H +#define EDM_SIMPLEX_H + +#include "EDM.h" + +//---------------------------------------------------------------- +// Simplex class inherits from EDM class and defines +// Simplex-specific projection methods +//---------------------------------------------------------------- +class SimplexClass : public EDM { +public: + // CCMClass includes two instances of SimplexClass. One for + // forward mapping, one for reverse. These objects hold the + // original input data subsetted for each library size. + DataFrame < double > dataCCM; // Original, full data + std::valarray< double > targetCCM; // Original, full target + + // Constructor + SimplexClass ( DataFrame & data, + Parameters & parameters ); + + // Method declarations + void Project(); + void Simplex(); + void CopyData(); // CCMClass + void WriteOutput(); +}; +#endif diff --git a/src/Version.h b/src/Version.h index b367bef..79d5f75 100644 --- a/src/Version.h +++ b/src/Version.h @@ -1,6 +1,9 @@ #ifndef VERSION_H #define VERSION_H +#include +#include + //------------------------------------------------------------ // Instantiated in Parameters() constructor //------------------------------------------------------------ diff --git a/src/makefile b/src/makefile index b6dacf7..080c699 100644 --- a/src/makefile +++ b/src/makefile @@ -1,16 +1,24 @@ -CC = g++ -OBJ = Common.o AuxFunc.o DateTimeUtil.o Parameter.o Embed.o Interface.o\ - Neighbors.o Simplex.o Eval.o CCM.o Multiview.o SMap.o +.PHONY: all clean distclean depend + +CC = g++ + +HEADERS = API.h CCM.h Common.h DataFrame.h DateTime.h EDM.h EDM_Neighbors.h\ + Multiview.h Parameter.h Simplex.h SMap.h Version.h + +SRCS = API.cc CCM.cc Common.cc DateTime.cc EDM.cc EDM_Formatting.cc\ + EDM_Neighbors.cc Eval.cc Multiview.cc Parameter.cc Simplex.cc SMap.cc + +OBJ = $(SRCS:%.cc=%.o) LIB = libEDM.a -CFLAGS = -std=c++11 -DCCM_THREADED -DMULTIVIEW_VALUES_OVERLOAD -O3 -Wreorder -#CFLAGS += -g -DDEBUG -DDEBUG_ALL -#LFLAGS = -L./ -lstdc++ -lEDM -lpthread # -llapacke -llapack -lblas +CFLAGS += -std=c++11 -Wpedantic -Wall -Wextra -Wreorder -O3 +CFLAGS += -DCCM_THREADED +# CFLAGS += -g # -DDEBUG_ALL +# LFLAGS = -L./ -lstdc++ -lEDM -lpthread -llapacke -llapack -lblas all: $(LIB) - ar -rcs $(LIB) $(OBJ) cp $(LIB) ../lib/ clean: @@ -20,61 +28,29 @@ distclean: rm -f $(OBJ) $(LIB) ../lib/$(LIB) *~ *.bak *.csv $(LIB): $(OBJ) + ar -rcs $(LIB) $(OBJ) -AuxFunc.o: AuxFunc.cc - $(CC) -c AuxFunc.cc $(CFLAGS) - -DateTimeUtil.o: DateTimeUtil.cc - $(CC) -c DateTimeUtil.cc $(CFLAGS) - -Common.o: Common.cc - $(CC) -c Common.cc $(CFLAGS) - -Parameter.o: Parameter.cc - $(CC) -c Parameter.cc $(CFLAGS) - -Embed.o: Embed.cc - $(CC) -c Embed.cc $(CFLAGS) - -Interface.o: Interface.cc - $(CC) -c Interface.cc $(CFLAGS) - -Neighbors.o: Neighbors.cc - $(CC) -c Neighbors.cc $(CFLAGS) - -Simplex.o: Simplex.cc - $(CC) -c Simplex.cc $(CFLAGS) - -Eval.o: Eval.cc - $(CC) -c Eval.cc $(CFLAGS) - -CCM.o: CCM.cc - $(CC) -c CCM.cc $(CFLAGS) - -Multiview.o: Multiview.cc - $(CC) -c Multiview.cc $(CFLAGS) - -SMap.o: SMap.cc - $(CC) -c SMap.cc $(CFLAGS) +%.o : %.cc + $(CC) $(CFLAGS) -c $< -SRCS = `echo ${OBJ} | sed -e 's/.o /.cc /g'` depend: @echo ${SRCS} makedepend -Y $(SRCS) # DO NOT DELETE +API.o: API.h Common.h DataFrame.h Parameter.h Version.h Simplex.h EDM.h +API.o: SMap.h CCM.h Multiview.h +CCM.o: CCM.h EDM.h Common.h DataFrame.h Parameter.h Version.h Simplex.h Common.o: Common.h DataFrame.h -AuxFunc.o: AuxFunc.h Common.h DataFrame.h Neighbors.h Parameter.h Version.h -AuxFunc.o: Embed.h DateTime.h -DateTimeUtil.o: DateTime.h +DateTime.o: DateTime.h +EDM.o: EDM.h Common.h DataFrame.h Parameter.h Version.h +EDM_Formatting.o: EDM.h Common.h DataFrame.h Parameter.h Version.h DateTime.h +EDM_Neighbors.o: EDM_Neighbors.h EDM.h Common.h DataFrame.h Parameter.h +EDM_Neighbors.o: Version.h +Eval.o: API.h Common.h DataFrame.h Parameter.h Version.h Simplex.h EDM.h +Eval.o: SMap.h CCM.h Multiview.h +Multiview.o: Multiview.h EDM.h Common.h DataFrame.h Parameter.h Version.h +Multiview.o: Simplex.h Parameter.o: Parameter.h Common.h DataFrame.h Version.h -Embed.o: Embed.h Common.h DataFrame.h Parameter.h Version.h -Interface.o: Common.h DataFrame.h -Neighbors.o: Neighbors.h Common.h DataFrame.h Parameter.h Version.h -Simplex.o: Common.h DataFrame.h Parameter.h Version.h Neighbors.h Embed.h -Simplex.o: AuxFunc.h -Eval.o: Common.h DataFrame.h -CCM.o: Common.h DataFrame.h Embed.h Parameter.h Version.h AuxFunc.h -CCM.o: Neighbors.h -Multiview.o: Common.h DataFrame.h AuxFunc.h Neighbors.h Parameter.h Version.h -Multiview.o: Embed.h +Simplex.o: Simplex.h EDM.h Common.h DataFrame.h Parameter.h Version.h +SMap.o: SMap.h EDM.h Common.h DataFrame.h Parameter.h Version.h diff --git a/src/makefile.windows b/src/makefile.windows deleted file mode 100644 index 2d9de31..0000000 --- a/src/makefile.windows +++ /dev/null @@ -1,74 +0,0 @@ - -CC = cl -OBJ = Common.obj AuxFunc.obj DateTimeUtil.obj Parameter.obj Embed.obj\ - Interface.obj Neighbors.obj Simplex.obj Eval.obj CCM.obj\ - Multiview.obj SMap.obj - -LIB = EDM.lib - -CFLAGS = -DCCM_THREADED -DMULTIVIEW_VALUES_OVERLOAD /EHsc /MD # /MT -DDEBUG -DDEBUG_ALL - -all: $(LIB) - lib /NODEFAULTLIB:LIBCMT /NODEFAULTLIB:library /OUT:$(LIB) $(OBJ) - cp $(LIB) ..\lib - -clean: - del -f $(OBJ) $(LIB) - -distclean: - del -f $(OBJ) $(LIB) ../lib/$(LIB) *~ *.bak *.csv - -$(LIB): $(OBJ) - -AuxFunc.obj: AuxFunc.cc - $(CC) /c AuxFunc.cc $(CFLAGS) - -DateTimeUtil.obj: DateTimeUtil.cc - $(CC) /c DateTimeUtil.cc $(CFLAGS) - -Common.obj: Common.cc - $(CC) /c Common.cc $(CFLAGS) - -Parameter.obj: Parameter.cc - $(CC) /c Parameter.cc $(CFLAGS) - -Embed.obj: Embed.cc - $(CC) /c Embed.cc $(CFLAGS) - -Interface.obj: Interface.cc - $(CC) /c Interface.cc $(CFLAGS) - -Neighbors.obj: Neighbors.cc - $(CC) /c Neighbors.cc $(CFLAGS) - -Simplex.obj: Simplex.cc - $(CC) /c Simplex.cc $(CFLAGS) - -Eval.obj: Eval.cc - $(CC) /c Eval.cc $(CFLAGS) - -CCM.obj: CCM.cc - $(CC) /c CCM.cc $(CFLAGS) - -Multiview.obj: Multiview.cc - $(CC) /c Multiview.cc $(CFLAGS) - -SMap.obj: SMap.cc - $(CC) /c SMap.cc $(CFLAGS) - -# Depedencies from makedepend on Linux -Common.obj: Common.h DataFrame.h -AuxFunc.obj: AuxFunc.h Common.h DataFrame.h Neighbors.h Parameter.h Version.h -AuxFunc.obj: Embed.h DateTime.h -DateTimeUtil.obj: DateTime.h -Parameter.obj: Parameter.h Common.h DataFrame.h Version.h -Embed.obj: Embed.h Common.h DataFrame.h Parameter.h Version.h -Interface.obj: Common.h DataFrame.h -Neighbors.obj: Neighbors.h Common.h DataFrame.h Parameter.h Version.h -Simplex.obj: Common.h DataFrame.h Parameter.h Version.h Neighbors.h Embed.h -Simplex.obj: AuxFunc.h -Eval.obj: Common.h DataFrame.h -CCM.obj: Common.h DataFrame.h Embed.h Parameter.h Version.h AuxFunc.h -CCM.obj: Neighbors.h -Multiview.obj: Common.h DataFrame.h AuxFunc.h Neighbors.h Parameter.h Version.h -Multiview.obj: Embed.h diff --git a/tests/DateTimeTest.cc b/tests/DateTimeTest.cc index b38c56a..bff5a24 100644 --- a/tests/DateTimeTest.cc +++ b/tests/DateTimeTest.cc @@ -14,15 +14,15 @@ void check_increment_correct( std::string date_str_1, std::string date_str_2, std::string correct_incremented, int tp ) { - std::string output = increment_datetime_str( date_str_1, date_str_2, tp ); + std::string output = IncrementDatetime( date_str_1, date_str_2, tp ); if ( output == correct_incremented ) std::cout <<"Correct."< combos = MV.Combo_rho; + DataFrame< double > combos = MV.ComboRho; DataFrame< double > output = MV.Predictions; combos.WriteData( "./", "Multiview_combos.csv" ); diff --git a/tests/TestCommon.h b/tests/TestCommon.h index 47c3e49..1257005 100644 --- a/tests/TestCommon.h +++ b/tests/TestCommon.h @@ -3,11 +3,8 @@ // #define PRINT_DIFFERENCE_IN_RESULTS -#include -#include -#include -#include -#include "Common.h" +#include "API.h" +#include "DateTime.h" #define STR_LINE_SEP "-------------------------------------------------" #define TAB_CHAR '\t' diff --git a/tests/data/CCM_anch_sst_cppEDM_valid.csv b/tests/data/CCM_anch_sst_cppEDM_valid.csv index dd6271f..6200902 100644 --- a/tests/data/CCM_anch_sst_cppEDM_valid.csv +++ b/tests/data/CCM_anch_sst_cppEDM_valid.csv @@ -1,15 +1,15 @@ LibSize,anchovy:np_sst,np_sst:anchovy -10.0000,-0.6529,0.8210 -15.0000,-0.7296,0.2161 -20.0000,-0.7596,-0.3209 -25.0000,-0.3350,0.0486 -30.0000,-0.0924,-0.2024 -35.0000,-0.0691,-0.1884 -40.0000,-0.0588,-0.2082 -45.0000,0.0591,-0.1272 -50.0000,0.0880,-0.1893 -55.0000,0.0137,-0.0726 -60.0000,0.1287,-0.0749 -65.0000,0.1101,-0.0755 -70.0000,0.0844,-0.0776 -75.0000,0.2165,-0.0700 +10.0000,-0.6530,-0.8953 +15.0000,-0.7293,0.2875 +20.0000,-0.7014,-0.4845 +25.0000,-0.3149,0.6335 +30.0000,0.1033,-0.2095 +35.0000,-0.0756,-0.1894 +40.0000,-0.0699,-0.0061 +45.0000,0.0126,0.0165 +50.0000,0.0958,-0.1779 +55.0000,0.0233,-0.0728 +60.0000,0.1149,-0.0749 +65.0000,0.1374,-0.0755 +70.0000,0.0726,-0.0744 +75.0000,0.2008,-0.0681 diff --git a/tests/data/CCM_anch_sst_pyEDM.csv b/tests/data/CCM_anch_sst_pyEDM.csv deleted file mode 100644 index 5f27b9f..0000000 --- a/tests/data/CCM_anch_sst_pyEDM.csv +++ /dev/null @@ -1,15 +0,0 @@ -lib_size,ρ anchovy np_sst,ρ np_sst anchovy -10,-0.66,0 -15,-0.73,0.18 -20,-0.76,-0.31 -25,-0.34,-0.14 -30,-0.22,-0.2 -35,-0.03,-0.08 -40,-0.06,-0.12 -45,0.06,-0.3 -50,0.08,-0.19 -55,0.03,-0.07 -60,0.15,-0.07 -65,0.07,-0.08 -70,0.12,-0.08 -75,0.22,-0.09 diff --git a/tests/data/Multiview_combos_valid.csv b/tests/data/Multiview_combos_valid.csv index cff978b..a26f439 100644 --- a/tests/data/Multiview_combos_valid.csv +++ b/tests/data/Multiview_combos_valid.csv @@ -1,10 +1,10 @@ Col_1,Col_2,Col_3,rho,MAE,RMSE -1.0000,2.0000,6.0000,0.8841,0.3100,0.3875 1.0000,2.0000,7.0000,0.9269,0.2454,0.3060 +1.0000,2.0000,6.0000,0.8841,0.3100,0.3875 1.0000,2.0000,3.0000,0.9341,0.2268,0.2895 -1.0000,5.0000,9.0000,0.8545,0.3409,0.4293 -1.0000,6.0000,8.0000,0.8413,0.3511,0.4427 -1.0000,4.0000,9.0000,0.7598,0.4271,0.5296 -1.0000,2.0000,8.0000,0.9037,0.2647,0.3473 +1.0000,7.0000,9.0000,0.8970,0.2903,0.3577 1.0000,2.0000,5.0000,0.9001,0.2842,0.3579 -1.0000,6.0000,9.0000,0.7906,0.4061,0.4948 +1.0000,3.0000,7.0000,0.8818,0.3066,0.3854 +1.0000,2.0000,8.0000,0.9037,0.2647,0.3473 +1.0000,4.0000,9.0000,0.7598,0.4271,0.5296 +1.0000,3.0000,8.0000,0.8882,0.3042,0.3787 diff --git a/tests/data/Multiview_pred_valid.csv b/tests/data/Multiview_pred_valid.csv index 64acfb3..d42387a 100644 --- a/tests/data/Multiview_pred_valid.csv +++ b/tests/data/Multiview_pred_valid.csv @@ -1,100 +1,100 @@ time,Observations,Predictions 103,0.4394,nan -104,-0.1194,0.1451 -105,1.4553,0.9389 -106,-1.4522,-1.2895 -107,-0.2129,0.2105 -108,1.1403,0.8587 -109,-1.1523,-1.2694 -110,0.5858,0.6076 -111,0.0227,-0.1043 -112,0.8775,0.8453 -113,-0.5192,-0.6579 -114,1.1396,1.2637 -115,-1.4618,-1.0305 -116,0.3419,0.3253 -117,0.7915,0.4555 -118,-0.7694,-0.2611 -119,0.8119,0.9033 -120,-0.7124,-0.9318 -121,1.4832,0.9287 -122,-1.3628,-1.0273 -123,0.2406,0.3210 -124,0.7683,0.2964 -125,-0.5327,-0.1064 -126,1.4174,0.9779 -127,-1.2639,-1.2033 -128,0.5169,0.6018 -129,0.4767,0.2275 -130,0.1681,0.0830 -131,0.5158,0.6573 -132,0.1068,-0.0637 -133,0.9393,0.7714 -134,-0.9698,-0.7118 -135,0.8169,0.8951 -136,-0.8705,-0.4924 -137,1.3428,0.9238 -138,-1.3902,-1.0241 -139,0.0282,0.2835 -140,1.2739,0.7547 -141,-0.9491,-1.2088 -142,1.0982,0.7828 -143,-0.4743,-0.4192 -144,1.2960,1.0176 -145,-1.3171,-1.1697 -146,0.5317,0.4853 -147,0.4973,0.1874 -148,0.1015,0.3314 -149,1.0045,0.6859 -150,-0.7131,-1.0195 -151,1.1896,1.0223 -152,-1.3306,-1.0589 -153,0.5372,0.4504 -154,0.3574,0.0609 -155,0.1723,0.4085 -156,0.4002,0.3706 -157,0.4691,0.2181 -158,0.4740,0.2333 -159,0.4869,0.1751 -160,-0.3526,0.2603 -161,0.7443,1.0279 -162,-1.1307,-0.8522 -163,0.4438,0.7386 -164,0.5323,0.3120 -165,0.7010,0.1170 -166,-0.3612,-0.3372 -167,1.3866,1.1314 -168,-1.1913,-1.1453 -169,0.3576,0.6220 -170,0.2594,0.3757 -171,0.3768,0.4205 -172,0.2548,0.2987 -173,0.1863,0.5563 -174,0.8941,0.4638 -175,-0.0269,-0.5002 -176,1.1631,0.8613 -177,-1.2747,-0.8117 -178,0.1799,0.5246 -179,0.4745,0.5657 -180,-0.1333,0.1713 -181,1.0006,0.8994 -182,-0.7447,-0.8426 -183,0.9934,1.0826 -184,-1.3225,-0.4851 -185,0.5358,0.4487 -186,0.4903,0.1532 -187,0.5186,0.1800 -188,0.5642,-0.0012 -189,0.1488,0.1205 -190,0.6269,0.6290 -191,0.1110,-0.0050 -192,0.8966,0.8125 -193,-0.7596,-0.5629 -194,0.7844,1.0960 -195,-1.2774,-0.5221 -196,0.5587,0.4636 -197,0.6340,0.1812 -198,0.0206,0.0293 -199,0.5489,0.7812 -200,0.0403,-0.0206 -201,nan,0.8723 +104,-0.1194,0.2495 +105,1.4553,0.9254 +106,-1.4522,-1.3405 +107,-0.2129,0.0730 +108,1.1403,0.8663 +109,-1.1523,-1.3848 +110,0.5858,0.5590 +111,0.0227,0.0232 +112,0.8775,0.8987 +113,-0.5192,-0.7901 +114,1.1396,1.2580 +115,-1.4618,-1.1741 +116,0.3419,0.3676 +117,0.7915,0.4652 +118,-0.7694,-0.5949 +119,0.8119,0.9741 +120,-0.7124,-0.8227 +121,1.4832,0.9921 +122,-1.3628,-1.1522 +123,0.2406,0.2102 +124,0.7683,0.5221 +125,-0.5327,-0.3221 +126,1.4174,1.0712 +127,-1.2639,-1.2906 +128,0.5169,0.5167 +129,0.4767,0.2731 +130,0.1681,0.2379 +131,0.5158,0.6967 +132,0.1068,-0.0962 +133,0.9393,0.8033 +134,-0.9698,-0.4814 +135,0.8169,0.9064 +136,-0.8705,-0.5001 +137,1.3428,0.8731 +138,-1.3902,-1.1017 +139,0.0282,0.2341 +140,1.2739,0.8353 +141,-0.9491,-1.3014 +142,1.0982,0.8053 +143,-0.4743,-0.3079 +144,1.2960,1.0650 +145,-1.3171,-1.2971 +146,0.5317,0.4276 +147,0.4973,0.2797 +148,0.1015,0.2435 +149,1.0045,0.6489 +150,-0.7131,-1.0350 +151,1.1896,1.0843 +152,-1.3306,-1.0925 +153,0.5372,0.4053 +154,0.3574,0.2240 +155,0.1723,0.3289 +156,0.4002,0.4903 +157,0.4691,0.3401 +158,0.4740,0.2289 +159,0.4869,0.1867 +160,-0.3526,0.2479 +161,0.7443,1.0504 +162,-1.1307,-0.9397 +163,0.4438,0.6968 +164,0.5323,0.2333 +165,0.7010,0.1802 +166,-0.3612,-0.1809 +167,1.3866,1.1162 +168,-1.1913,-1.2317 +169,0.3576,0.6106 +170,0.2594,0.4120 +171,0.3768,0.3568 +172,0.2548,0.2269 +173,0.1863,0.5716 +174,0.8941,0.4977 +175,-0.0269,-0.4281 +176,1.1631,0.8222 +177,-1.2747,-0.8763 +178,0.1799,0.5999 +179,0.4745,0.5120 +180,-0.1333,0.0946 +181,1.0006,0.8889 +182,-0.7447,-0.9915 +183,0.9934,1.0886 +184,-1.3225,-0.5507 +185,0.5358,0.4449 +186,0.4903,0.2235 +187,0.5186,0.1127 +188,0.5642,0.0899 +189,0.1488,0.1783 +190,0.6269,0.6377 +191,0.1110,-0.2217 +192,0.8966,0.8168 +193,-0.7596,-0.8783 +194,0.7844,1.1104 +195,-1.2774,-0.6189 +196,0.5587,0.5281 +197,0.6340,0.1885 +198,0.0206,-0.0329 +199,0.5489,0.7501 +200,0.0403,-0.1912 +201,nan,0.8926 diff --git a/tests/data/Smap_circle_coef.csv b/tests/data/Smap_circle_coef.csv deleted file mode 100644 index 732d27b..0000000 --- a/tests/data/Smap_circle_coef.csv +++ /dev/null @@ -1,100 +0,0 @@ -Time,C0,C1,C2 -101,nan,nan,nan -102,-0.0000,0.9980,0.0631 -103,0.0000,0.9980,0.0631 -104,0.0000,0.9980,0.0631 -105,-0.0000,0.9980,0.0631 -106,-0.0000,0.9980,0.0631 -107,-0.0000,0.9980,0.0631 -108,-0.0000,0.9980,0.0631 -109,0.0000,0.9980,0.0631 -110,0.0000,0.9980,0.0631 -111,0.0000,0.9980,0.0631 -112,-0.0000,0.9980,0.0631 -113,-0.0000,0.9980,0.0631 -114,0.0000,0.9980,0.0631 -115,0.0000,0.9980,0.0631 -116,-0.0000,0.9980,0.0631 -117,-0.0000,0.9980,0.0631 -118,-0.0000,0.9980,0.0631 -119,-0.0000,0.9980,0.0631 -120,-0.0000,0.9980,0.0631 -121,-0.0000,0.9980,0.0631 -122,-0.0000,0.9980,0.0631 -123,-0.0000,0.9980,0.0631 -124,-0.0000,0.9980,0.0631 -125,-0.0000,0.9980,0.0631 -126,0.0000,0.9980,0.0631 -127,0.0000,0.9980,0.0631 -128,0.0000,0.9980,0.0631 -129,0.0000,0.9980,0.0631 -130,0.0000,0.9980,0.0631 -131,0.0000,0.9980,0.0631 -132,0.0000,0.9980,0.0631 -133,0.0000,0.9980,0.0631 -134,0.0000,0.9980,0.0631 -135,-0.0000,0.9980,0.0631 -136,-0.0000,0.9980,0.0631 -137,-0.0000,0.9980,0.0631 -138,-0.0000,0.9980,0.0631 -139,-0.0000,0.9980,0.0631 -140,-0.0000,0.9980,0.0631 -141,-0.0000,0.9980,0.0631 -142,-0.0000,0.9980,0.0631 -143,-0.0000,0.9980,0.0631 -144,-0.0000,0.9980,0.0631 -145,0.0000,0.9980,0.0631 -146,0.0000,0.9980,0.0631 -147,0.0000,0.9980,0.0631 -148,0.0000,0.9980,0.0631 -149,0.0001,0.9980,0.0632 -150,0.0000,0.9980,0.0631 -151,0.0000,0.9980,0.0631 -152,-0.0000,0.9980,0.0631 -153,-0.0000,0.9980,0.0631 -154,-0.0000,0.9980,0.0631 -155,-0.0000,0.9980,0.0631 -156,-0.0000,0.9980,0.0631 -157,-0.0000,0.9980,0.0631 -158,-0.0000,0.9980,0.0631 -159,-0.0000,0.9980,0.0631 -160,-0.0000,0.9980,0.0631 -161,-0.0000,0.9980,0.0631 -162,-0.0000,0.9980,0.0631 -163,-0.0000,0.9980,0.0631 -164,-0.0000,0.9980,0.0631 -165,0.0000,0.9980,0.0631 -166,0.0000,0.9981,0.0631 -167,0.0001,0.9981,0.0631 -168,0.0001,0.9981,0.0631 -169,0.0000,0.9980,0.0631 -170,-0.0000,0.9980,0.0631 -171,-0.0000,0.9980,0.0631 -172,-0.0001,0.9979,0.0631 -173,-0.0001,0.9979,0.0631 -174,-0.0000,0.9980,0.0631 -175,-0.0000,0.9980,0.0631 -176,0.0000,0.9980,0.0631 -177,0.0000,0.9981,0.0631 -178,0.0000,0.9981,0.0631 -179,0.0000,0.9980,0.0631 -180,-0.0000,0.9980,0.0631 -181,0.0000,0.9980,0.0631 -182,0.0000,0.9980,0.0631 -183,0.0000,0.9980,0.0631 -184,-0.0000,0.9980,0.0631 -185,-0.0000,0.9980,0.0631 -186,-0.0000,0.9980,0.0631 -187,-0.0000,0.9980,0.0631 -188,-0.0000,0.9980,0.0631 -189,-0.0000,0.9980,0.0631 -190,0.0000,0.9980,0.0631 -191,-0.0000,0.9980,0.0631 -192,-0.0000,0.9980,0.0631 -193,0.0000,0.9980,0.0631 -194,0.0000,0.9980,0.0631 -195,0.0000,0.9980,0.0631 -196,-0.0000,0.9980,0.0631 -197,-0.0000,0.9980,0.0631 -198,0.0000,0.9980,0.0631 -199,0.0000,0.9980,0.0631 From ee854c0dd639c5cc80232f1c721e5581cb4f0a51 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Thu, 11 Jun 2020 16:43:10 -0400 Subject: [PATCH 02/13] v2.0.0 OOD Branch --- src/Embed.cc | 183 --------------------------------------------------- src/Embed.h | 36 ---------- 2 files changed, 219 deletions(-) delete mode 100644 src/Embed.cc delete mode 100644 src/Embed.h diff --git a/src/Embed.cc b/src/Embed.cc deleted file mode 100644 index c1fba55..0000000 --- a/src/Embed.cc +++ /dev/null @@ -1,183 +0,0 @@ - -#include "Embed.h" - -// NOTE: The returned data block does NOT have the time column - -//---------------------------------------------------------------- -// API Overload 1: Explicit data file path/name -// Embed DataFrame columns in E dimensions. -// Data is read from path/dataFile -// Implemented as a wrapper to API Overload 2: -// which is a wrapper for MakeBlock() -// -// NOTE: Truncates data by tau * (E-1) rows to remove -// nan values (partial data rows) -//---------------------------------------------------------------- -DataFrame< double > Embed( std::string path, - std::string dataFile, - int E, // embedding dimension - int tau, // time step delay - std::string columns, // column names or indices - bool verbose ) { - - DataFrame< double > dataFrame( path, dataFile ); - DataFrame< double > embedded = Embed(dataFrame, E, tau, columns, verbose); - return embedded; -} - -//---------------------------------------------------------------- -// API Overload 2: DataFrame provided -// Implemented as a wrapper for MakeBlock() -// Note: dataFrame must have the columnNameToIndex map -//---------------------------------------------------------------- -DataFrame< double > Embed( DataFrame< double > dataFrameIn, - int E, - int tau, - std::string columns, - bool verbose ) { - - // Parameter.Validate will convert columns into a vector of names - // or a vector of column indices - Parameters param = Parameters( Method::Embed, "", "", "", "", - "1 1", "1 1", E, 0, 0, tau, 0, 0, - columns, "", false, false, verbose ); - - if ( not param.columnIndex.size() and - dataFrameIn.ColumnNameToIndex().empty() ) { - throw std::runtime_error("Embed(DataFrame): columnNameIndex empty.\n"); - } - - // If columns provided, validate they are in dataFrameIn - for ( auto colName : param.columnNames ) { - auto ci = find( dataFrameIn.ColumnNames().begin(), - dataFrameIn.ColumnNames().end(), colName ); - - if ( ci == dataFrameIn.ColumnNames().end() ) { - std::stringstream errMsg; - errMsg << "Embed(DataFrame): Failed to find column " - << colName << " in dataFrame with columns: [ "; - for ( auto col : dataFrameIn.ColumnNames() ) { - errMsg << col << " "; - } errMsg << " ]\n"; - throw std::runtime_error( errMsg.str() ); - } - } - - // Get column names for MakeBlock - std::vector< std::string > colNames; - if ( param.columnNames.size() ) { - // column names are strings use as-is - colNames = param.columnNames; - } - else if ( param.columnIndex.size() ) { - // columns are indices : Create column names for MakeBlock - for ( size_t i = 0; i < param.columnIndex.size(); i++ ) { - std::stringstream ss; - ss << "V" << param.columnIndex[i]; - colNames.push_back( ss.str() ); - } - } - else { - throw std::runtime_error( "Embed(DataFrame): columnNames and " - " columnIndex are empty.\n" ); - } - - // Extract the specified columns (sub)DataFrame from dataFrameIn - DataFrame< double > dataFrame; - - if ( param.columnNames.size() ) { - dataFrame = dataFrameIn.DataFrameFromColumnNames( param.columnNames ); - } - else if ( param.columnIndex.size() ) { - // already have column indices - // Note there will be no column names transferred - dataFrame = dataFrameIn.DataFrameFromColumnIndex( param.columnIndex ); - } - - DataFrame< double > embedding = MakeBlock( dataFrame, E, tau, - colNames, verbose ); - - return embedding; -} - -//------------------------------------------------------------------------ -// MakeBlock from dataFrame -// Ignores the first (or last) tau * (E-1) dataFrame rows of partial data. -// Does not validate parameters or columns, use Embed() -//------------------------------------------------------------------------ -DataFrame< double > MakeBlock( DataFrame< double > dataFrame, - int E, - int tau, - std::vector columnNames, - bool verbose ) { - - if ( columnNames.size() != dataFrame.NColumns() ) { - std::stringstream errMsg; - errMsg << "MakeBlock: The number of columns in the dataFrame (" - << dataFrame.NColumns() << ") is not equal to the number " - << "of columns specified (" << columnNames.size() << ").\n";; - throw std::runtime_error( errMsg.str() ); - } - - if ( E < 1 ) { - std::stringstream errMsg; - errMsg << "MakeBlock(): E = " << E << " is invalid.\n" ; - throw std::runtime_error( errMsg.str() ); - } - - size_t NRows = dataFrame.NRows(); // number of input rows - size_t NColOut = dataFrame.NColumns() * E; // number of output columns - size_t NPartial = abs( tau ) * (E-1); // rows to shift & delete - - // Create embedded data frame column names X(t-0) X(t-1)... - std::vector< std::string > newColumnNames( NColOut ); - size_t newCol_i = 0; - for ( size_t col = 0; col < columnNames.size(); col ++ ) { - for ( size_t e = 0; e < E; e++ ) { - std::stringstream ss; - if ( tau < 0 ) { - ss << columnNames[ col ] << "(t-" << e << ")"; - } - else { - ss << columnNames[ col ] << "(t+" << e << ")"; - } - newColumnNames[ newCol_i ] = ss.str(); - newCol_i++; - } - } - - // Ouput data frame with tau * E-1 fewer rows - DataFrame< double > embedding( NRows - NPartial, NColOut, newColumnNames ); - - // To keep track of where to insert column in new data frame - size_t colCount = 0; - - // Slice to ignore rows with partial data - std::slice slice_i; - if ( tau < 0 ) { - slice_i = std::slice( NPartial, NRows - NPartial, 1 ); - } - else { - slice_i = std::slice( 0, NRows - NPartial, 1 ); - } - - // Shift column data and write to embedding data frame - for ( size_t col = 0; col < dataFrame.NColumns(); col++ ) { - // for each embedding dimension - for ( size_t e = 0; e < E; e++ ) { - - std::valarray< double > column = dataFrame.Column( col ); - - // Returns a copy of the valarray object with its elements - // shifted left n spaces (or right if n is negative). - std::valarray< double > tmp = column.shift( e * tau ); - - // Write shifted columns to the output embedding DataFrame - embedding.WriteColumn( colCount, tmp[ slice_i ] ); - - colCount++; - } - } - - return embedding; -} diff --git a/src/Embed.h b/src/Embed.h deleted file mode 100644 index 3bccba0..0000000 --- a/src/Embed.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef EMBED_H -#define EMBED_H - -#include "Common.h" -#include "Parameter.h" - -//---------------------------------------------------------------- -// API Overload 1: Explicit data file path/name -// Implemented as a wrapper to API Overload 2: -// which is a wrapper for MakeBlock() -//---------------------------------------------------------------- -DataFrame< double > Embed ( std::string path = "", - std::string dataFile = "", - int E = 0, - int tau = 0, - std::string columns = "", - bool verbose = false ); - -//---------------------------------------------------------------- -// API Overload 2: DataFrame provided -// Implemented as a wrapper for MakeBlock() -//---------------------------------------------------------------- -DataFrame< double > Embed ( DataFrame< double > dataFrame, - int E = 0, - int tau = 0, - std::string columns = "", - bool verbose = false ); - -//---------------------------------------------------------------- -//---------------------------------------------------------------- -DataFrame< double > MakeBlock ( DataFrame< double > dataFrame, - int E, - int tau, - std::vector columnNames, - bool verbose ); -#endif From 71463e3996278a8660c9ff9f314cff4b7fb188ec Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Sun, 14 Jun 2020 16:26:39 -0400 Subject: [PATCH 03/13] v2.0.0 UBSAN fix. --- src/DataFrame.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/DataFrame.h b/src/DataFrame.h index 210a4a1..88d449e 100644 --- a/src/DataFrame.h +++ b/src/DataFrame.h @@ -54,8 +54,9 @@ class DataFrame { //----------------------------------------------------------------- // Constructors //----------------------------------------------------------------- - DataFrame() : n_rows(0), n_columns(0), noTime( false ), - partialDataRowsDeleted( false ) {} + DataFrame(): + n_rows( 0 ), n_columns( 0 ), elements( 0 ), + maxRowPrint( 10 ), noTime( false ), partialDataRowsDeleted( false ) {} //----------------------------------------------------------------- // Load data from CSV file path/fileName, populate DataFrame @@ -72,7 +73,7 @@ class DataFrame { //----------------------------------------------------------------- DataFrame( size_t rows, size_t columns ): n_rows( rows ), n_columns( columns ), elements( columns * rows ), - maxRowPrint( 10 ), partialDataRowsDeleted( false ) {} + maxRowPrint( 10 ), noTime( false ), partialDataRowsDeleted( false ) {} //----------------------------------------------------------------- // Empty DataFrame of size (rows, columns) with column names in a @@ -81,7 +82,7 @@ class DataFrame { DataFrame( size_t rows, size_t columns, std::string colNames ): n_rows( rows ), n_columns( columns ), elements( columns * rows ), columnNames( std::vector(columns) ), maxRowPrint( 10 ), - partialDataRowsDeleted( false ) + noTime( false ), partialDataRowsDeleted( false ) { BuildColumnNameIndex( colNames ); } @@ -94,7 +95,7 @@ class DataFrame { std::vector< std::string > columnNames ): n_rows( rows ), n_columns( columns ), elements( columns * rows ), columnNames( columnNames ), maxRowPrint( 10 ), - partialDataRowsDeleted( false ) + noTime( false ), partialDataRowsDeleted( false ) { BuildColumnNameIndex(); } From a11f78d01a8a957bcf25ba944022465c1147b05a Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Fri, 19 Jun 2020 10:36:47 -0400 Subject: [PATCH 04/13] v1.5.0 Degenerate neighbor row check. --- src/EDM_Neighbors.cc | 73 +++++++++++++++++++++++++++----------------- src/Parameter.cc | 2 +- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/src/EDM_Neighbors.cc b/src/EDM_Neighbors.cc index 359bb8e..c32d590 100644 --- a/src/EDM_Neighbors.cc +++ b/src/EDM_Neighbors.cc @@ -22,8 +22,7 @@ namespace EDM_Neighbors_Lock { // dataBlock. The target vector is also reduced. // // NOTE: If rows are deleted, then the library and prediction -// vectors in Parameters are updated to reflect this. -// +// vectors in Parameters are updated to reflect this. //---------------------------------------------------------------- void EDM::PrepareEmbedding( bool checkDataRows ) { @@ -33,7 +32,7 @@ void EDM::PrepareEmbedding( bool checkDataRows ) { // Embed if ( parameters.embedded ) { - // dataIn is a multivariable block, no embedding needed + // data is a multivariable block, no embedding needed // Select the specified columns into embedding if ( parameters.columnNames.size() ) { embedding = data.DataFrameFromColumnNames( parameters.columnNames ); @@ -47,8 +46,8 @@ void EDM::PrepareEmbedding( bool checkDataRows ) { } } else { - // embedded = false: Create the embedding dataBlock via EmbedData() - // dataBlock will have tau * (E-1) fewer rows than dataIn + // embedded = false: Create embedding via EmbedData() + // embedding will have tau * (E-1) fewer rows than data EmbedData(); } @@ -65,7 +64,7 @@ void EDM::PrepareEmbedding( bool checkDataRows ) { } //------------------------------------------------------------ - // embedded = false: Embed() was called on dataIn + // embedded = false: Embed() was called on data // Remove target, data rows as needed // Adjust parameters.library and parameters.prediction indices //------------------------------------------------------------ @@ -73,15 +72,16 @@ void EDM::PrepareEmbedding( bool checkDataRows ) { if ( parameters.E < 1 ) { std::stringstream errMsg; - errMsg << "EmbedNN(): E = " << parameters.E << " is invalid.\n" ; + errMsg << "PreparEmbedding(): E = " << parameters.E + << " is invalid.\n" ; throw std::runtime_error( errMsg.str() ); } - + size_t shift = abs( parameters.tau ) * ( parameters.E - 1 ); // Copy targetIn excluding partial data into targetEmbed std::valarray< double > targetEmbed( data.NRows() - shift ); - + // Bogus cast to ( std::valarray ) for MSVC // as it doesn't export its own slice_array applied to [] if ( parameters.tau < 0 ) { @@ -92,10 +92,10 @@ void EDM::PrepareEmbedding( bool checkDataRows ) { targetEmbed = ( std::valarray< double > ) target[ std::slice( 0, target.size() - shift, 1 ) ]; } - + // Resize target to ignore partial data rows target.resize( targetEmbed.size() ); - + // Copy target without partial data into resized targetIn std::slice targetEmbed_i = std::slice( 0, targetEmbed.size(), 1 ); target[ targetEmbed_i ] = ( std::valarray< double > ) @@ -179,7 +179,7 @@ void EDM::FindNeighbors() { } } - // allLibrows are the lib row indices, 1 row x lib columns + // allLibRows are the library row indices, 1 row x lib columns std::valarray< size_t > rowLib = allLibRows.Row( 0 ); // Pair the distances and library row indices for sort on distance @@ -225,13 +225,18 @@ void EDM::FindNeighbors() { // For each prediction vector (row in prediction DataFrame) find the // list of library indices that are within k_NN points //------------------------------------------------------------------- - for ( size_t pred_row = 0; pred_row < predPairs.size(); pred_row++ ) { + for ( size_t predPair_i = 0; predPair_i < predPairs.size(); predPair_i++ ) { + + // The actual prediction row specified by user (zero offset) + size_t predictionRow = parameters.prediction[ predPair_i ]; // rowPair is a vector of pairs of length library rows // Get the rowPair for this prediction row - std::vector< std::pair > rowPair = predPairs[ pred_row ]; + std::vector< std::pair > rowPair = predPairs[predPair_i]; - // sort < distance, lib_row > pairs for this pred_row + int rowPairSize = (int) rowPair.size(); + + // sort < distance, lib_row > pairs for this predPair_i // distance must be .first std::sort( rowPair.begin(), rowPair.end(), DistanceCompare ); @@ -242,13 +247,25 @@ void EDM::FindNeighbors() { int lib_row_i = 0; int k = 0; while ( k < parameters.knn ) { + if ( lib_row_i >= rowPairSize ) { + std::stringstream errMsg; + errMsg << "FindNeighbors(): knn search failed. " + << k << " out of " << parameters.knn + << " neighbors were found in the library.\n" ; + throw std::runtime_error( errMsg.str() ); + } + double distance = rowPair[ lib_row_i ].first; - int lib_row = rowPair[ lib_row_i ].second; + size_t lib_row = rowPair[ lib_row_i ].second; + + if ( lib_row == predictionRow ) { + continue; // degenerate pred : lib, ignore + } if ( not parameters.noNeighborLimit ) { // Reach exceeding grasp : forecast point is outside library - if ( lib_row + parameters.Tp > max_lib_index or - lib_row + parameters.Tp < 0 ) { + if ( (int) lib_row + parameters.Tp > max_lib_index or + (int) lib_row + parameters.Tp < 0 ) { lib_row_i++; continue; // keep looking } @@ -256,7 +273,7 @@ void EDM::FindNeighbors() { // Exclusion radius: units are data rows, not time if ( parameters.exclusionRadius ) { - int xrad = (int) lib_row - (int) pred_row; + int xrad = (int) lib_row - (int) predPair_i; if ( std::abs( xrad ) <= parameters.exclusionRadius ) { lib_row_i++; continue; // skip this neighbor @@ -269,8 +286,8 @@ void EDM::FindNeighbors() { k++; } - knn_distances.WriteRow( pred_row, knnDistances ); - knn_neighbors.WriteRow( pred_row, knnLibRows ); + knn_distances.WriteRow( predPair_i, knnDistances ); + knn_neighbors.WriteRow( predPair_i, knnLibRows ); // Check for ties. 1.18e−38 is float 32-bit min if ( k < (int) rowPair.size() ) { @@ -280,9 +297,9 @@ void EDM::FindNeighbors() { while( k < (int) rowPair.size() and rowPair[ k ].first > 0 and rowPair[ k ].first <= rowPair[ k-1 ].first ) { - + // Set flag in ties and store tie pairs in tiePairs - ties[ pred_row ] = true; + ties[ predPair_i ] = true; rowTiePairs.push_back(std::make_pair( rowPair[ k ].first, rowPair[ k ].second )); @@ -291,11 +308,11 @@ void EDM::FindNeighbors() { if ( find( ties.begin(), ties.end(), true ) != ties.end() ) { anyTies = true; - tiePairs[ pred_row ] = rowTiePairs; + tiePairs[ predPair_i ] = rowTiePairs; } } } - } // for ( pred_row = 0; pred_row < predPairs.size(); pred_row++ ) + } // for ( predPair_i = 0; predPair_i < predPairs.size(); predPair_i++ ) #ifdef DEBUG_ALL for ( size_t i = 0; i < ties.size(); i++ ) { @@ -363,14 +380,14 @@ void EDM::Distances () { // Compute all prediction row : library row distances for ( size_t predRow = 0; predRow < Npred; predRow++ ) { - + size_t predictionRow = parameters.prediction[ predRow ]; - + // Get E-dimensional vector from this prediction row std::valarray< double > v1 = embedding.Row( predictionRow ); for ( size_t libRow = 0; libRow < Nlib; libRow++ ) { - + if ( predictionRow == parameters.library[ libRow ] ) { continue; // degenerate pred & lib } diff --git a/src/Parameter.cc b/src/Parameter.cc index df98a6a..c86ddd0 100644 --- a/src/Parameter.cc +++ b/src/Parameter.cc @@ -88,7 +88,7 @@ Parameters::Parameters( // Set validated flag and instantiate Version validated ( false ), - version ( 2, 0, 0, "2020-06-03" ) + version ( 1, 5, 0, "2020-06-19" ) { // Constructor code if ( method != Method::None ) { From 4dd03dddc5745d33ba6342855d3d52338da522ca Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Fri, 19 Jun 2020 14:52:12 -0400 Subject: [PATCH 05/13] v1.5.0 Degenerate neighbors. --- src/EDM_Neighbors.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/EDM_Neighbors.cc b/src/EDM_Neighbors.cc index c32d590..5c3e8e8 100644 --- a/src/EDM_Neighbors.cc +++ b/src/EDM_Neighbors.cc @@ -259,6 +259,7 @@ void EDM::FindNeighbors() { size_t lib_row = rowPair[ lib_row_i ].second; if ( lib_row == predictionRow ) { + lib_row_i++; continue; // degenerate pred : lib, ignore } From 0dcb1800cc92c436749fe79d576ac5bcf102dc29 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Sat, 20 Jun 2020 22:33:28 -0400 Subject: [PATCH 06/13] v1.5.0 makefile CXX instead of CC. --- src/CCM.cc | 12 ++++++++++-- src/makefile | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/CCM.cc b/src/CCM.cc index 3527c55..0114bd0 100644 --- a/src/CCM.cc +++ b/src/CCM.cc @@ -108,7 +108,7 @@ void CrossMap( SimplexClass & S ) { errMsg << "CrossMap(): E = " << S.parameters.E << " is invalid.\n" ; throw std::runtime_error( errMsg.str() ); } - + //----------------------------------------------------------------- // Set number of samples //----------------------------------------------------------------- @@ -135,7 +135,7 @@ void CrossMap( SimplexClass & S ) { } } std::default_random_engine DefaultRandomEngine( S.parameters.seed ); - + //---------------------------------------------------------- // Predictions //---------------------------------------------------------- @@ -361,6 +361,9 @@ void CrossMap( SimplexClass & S ) { //----------------------------------------------------------------- void CCMClass::SetupParameters() { + { // JP Synchronization protection needed? + std::lock_guard lck( EDM_CCM_Lock::mtx ); + // Each thread has it's own copy of input data & parameters colToTargetCCM.dataCCM = data; // Copy targetToColCCM.dataCCM = data; // Copy @@ -419,6 +422,7 @@ void CCMClass::SetupParameters() { colToTargetCCM.colToTarget.PredictStats = PredictionStats1; targetToColCCM.targetToCol.PredictStats = PredictionStats2; } + } // JP Synchronization protection needed? } //---------------------------------------------------------------- @@ -437,6 +441,9 @@ void CCMClass::CopyData () { // //---------------------------------------------------------------- void CCMClass::FormatOutput () { + { // JP Synchronization protection needed? + std::lock_guard lck( EDM_CCM_Lock::mtx ); + // Create unified column names of output DataFrame std::stringstream libRhoNames; libRhoNames << "LibSize " @@ -450,6 +457,7 @@ void CCMClass::FormatOutput () { allLibStats.WriteColumn(0, colToTargetCCM.colToTarget.LibStats.Column( 0 )); allLibStats.WriteColumn(1, colToTargetCCM.colToTarget.LibStats.Column( 1 )); allLibStats.WriteColumn(2, targetToColCCM.targetToCol.LibStats.Column( 1 )); + } // JP Synchronization protection needed? } //---------------------------------------------------------------- diff --git a/src/makefile b/src/makefile index 080c699..b2dc274 100644 --- a/src/makefile +++ b/src/makefile @@ -1,7 +1,7 @@ .PHONY: all clean distclean depend -CC = g++ +CXX = g++ HEADERS = API.h CCM.h Common.h DataFrame.h DateTime.h EDM.h EDM_Neighbors.h\ Multiview.h Parameter.h Simplex.h SMap.h Version.h @@ -31,7 +31,7 @@ $(LIB): $(OBJ) ar -rcs $(LIB) $(OBJ) %.o : %.cc - $(CC) $(CFLAGS) -c $< + $(CXX) $(CFLAGS) -c $< depend: @echo ${SRCS} From a4c21def19f765f76ed02f165a06e28fd0b7acd3 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Sun, 21 Jun 2020 11:12:03 -0400 Subject: [PATCH 07/13] v1.5.0 Add Thrips CCM test. --- data/Thrips.csv | 82 ++++++++++++++++++++++++++++ tests/CCMTest.cc | 76 ++++++++++++++++++++++++++ tests/TestCommon.h | 2 +- tests/data/Thrips_CMmatrix_valid.csv | 5 ++ tests/makefile | 3 +- 5 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 data/Thrips.csv create mode 100644 tests/data/Thrips_CMmatrix_valid.csv diff --git a/data/Thrips.csv b/data/Thrips.csv new file mode 100644 index 0000000..6c6bd50 --- /dev/null +++ b/data/Thrips.csv @@ -0,0 +1,82 @@ +Year,Month,Thrips_imaginis,maxT_degC,Rain_mm,Season +1932,4,4.5,19.2,140.1,-0.5 +1932,5,23.4,19.1,53.7,-0.866 +1932,6,17.8,14.3,134.1,-1 +1932,7,4.4,13.8,89.9,-0.866 +1932,8,3.3,14.6,92.2,-0.5 +1932,9,34,17.6,49.7,0 +1932,10,13.8,17.4,61.6,0.5 +1932,11,133.7,23.3,14.2,0.866 +1932,12,267.4,24.3,10.4,1 +1933,1,37.8,24.8,31.1,0.866 +1933,2,9.8,28.4,10.2,0.5 +1933,3,2.8,24.4,36.2,0 +1933,4,2.4,21.1,53.1,-0.5 +1933,5,3.9,17.1,174.2,-0.866 +1933,6,5.3,15.9,46,-1 +1933,7,5.5,14.1,55.5,-0.866 +1933,8,1.7,14,97.8,-0.5 +1933,9,2.8,16.3,94.2,0 +1933,10,10.1,21.8,17.5,0.5 +1933,11,78.3,23.4,6.4,0.866 +1933,12,71.3,25.4,10.8,1 +1934,1,15.5,29.8,14.5,0.866 +1934,2,5.1,28.5,3.4,0.5 +1934,3,7.3,29.2,18.3,0 +1934,4,6.1,19.8,37.8,-0.5 +1934,5,14.4,21,1.6,-0.866 +1934,6,26.9,16.2,29.2,-1 +1934,7,6.7,15.8,30.2,-0.866 +1934,8,3.7,14.9,124.4,-0.5 +1934,9,5.9,18.7,102.8,0 +1934,10,1.6,19.8,63.2,0.5 +1934,11,36,22.5,96.3,0.866 +1934,12,76.3,25.6,38.6,1 +1935,1,14.5,26.2,22,0.866 +1935,2,7.8,26.9,0.9,0.5 +1935,3,3.7,25.3,68.1,0 +1935,4,2.6,20.1,49,-0.5 +1935,5,6.2,16.4,74.1,-0.866 +1935,6,16.5,14.3,72.9,-1 +1935,7,0.8,14.2,81.9,-0.866 +1935,8,1.7,16.4,82.4,-0.5 +1935,9,5.5,18.1,72.8,0 +1935,10,21.9,20.7,66.6,0.5 +1935,11,133.5,23.7,23.5,0.866 +1935,12,136.6,26.2,16.2,1 +1936,1,15.9,27.2,35.7,0.866 +1936,2,5.7,26.6,18.8,0.5 +1936,3,6.3,26.1,2.1,0 +1936,4,3.3,20.4,39.4,-0.5 +1936,5,8.1,19.5,68.8,-0.866 +1936,6,12.9,14.2,71.4,-1 +1936,7,9.1,13.9,77.8,-0.866 +1936,8,1.8,15.6,52.3,-0.5 +1936,9,4.3,17.1,28.2,0 +1936,10,11.2,19.3,72.1,0.5 +1936,11,41.1,23.7,12.9,0.866 +1936,12,69,25.4,41.4,1 +1937,1,5.9,25.1,71.7,0.866 +1937,2,2.9,27.3,14.1,0.5 +1937,3,3.7,26.9,25.6,0 +1937,4,2,20.4,28.8,-0.5 +1937,5,3,17.9,119.3,-0.866 +1937,6,4.8,14.5,56.4,-1 +1937,7,2.5,13.8,44.3,-0.866 +1937,8,1.7,15.8,92.9,-0.5 +1937,9,4.6,17.7,91.4,0 +1937,10,24.4,21.1,20,0.5 +1937,11,310.4,25.4,51.5,0.866 +1937,12,127.4,24.3,42.1,1 +1938,1,15.6,26.7,19.7,0.866 +1938,2,6.2,25.7,56.2,0.5 +1938,3,4,25.8,3.8,0 +1938,4,3.5,22.3,150.9,-0.5 +1938,5,11.6,18.7,32.7,-0.866 +1938,6,5.6,13.8,67.6,-1 +1938,7,7.9,14.2,78.8,-0.866 +1938,8,5.4,14.7,68.5,-0.5 +1938,9,50.7,17.8,25.8,0 +1938,10,157.8,21.6,25.6,0.5 +1938,11,574.6,25.6,17.9,0.866 +1938,12,137.6,26.9,12.4,1 diff --git a/tests/CCMTest.cc b/tests/CCMTest.cc index 384b66a..9a7f319 100644 --- a/tests/CCMTest.cc +++ b/tests/CCMTest.cc @@ -41,4 +41,80 @@ int main () { // comparison MakeTest ( "CCM sardine_anchovy_sst test", cppOutput, output ); + + //------------------------------------------------------------------ + // Thrips + // Create the ccm matrix of the rEDM-tutorial.Rmd vignette + //------------------------------------------------------------------ + std::vector< std::string > columns; + std::vector< std::string > targets; + + columns.push_back( "Thrips_imaginis" ); + targets.push_back( "maxT_degC" ); + + columns.push_back( "Thrips_imaginis" ); + targets.push_back( "Rain_mm" ); + + columns.push_back( "Thrips_imaginis" ); + targets.push_back( "Season" ); + + columns.push_back( "maxT_degC" ); + targets.push_back( "Rain_mm" ); + + columns.push_back( "maxT_degC" ); + targets.push_back( "Season" ); + + columns.push_back( "Rain_mm" ); + targets.push_back( "Season" ); + + std::vector< size_t > rows1( { 1, 2, 3, 2, 3, 3 } ); + std::vector< size_t > cols1( { 0, 0, 0, 1, 1, 2 } ); + std::vector< size_t > rows2( { 0, 0, 0, 1, 1, 2 } ); + std::vector< size_t > cols2( { 1, 2, 3, 2, 3, 3 } ); + + DataFrame< double > ccmMatrix( 4, 4, + "Thrips_imaginis maxT_degC Rain_mm Season" ); + + for ( size_t i = 0; i < columns.size(); i++ ) { + CCMValues ccmThrips = CCM( "../data/", // pathIn + "Thrips.csv", // dataFile + "./data/", // pathOut + "", //predictFile + 8, // E + 0, // Tp + 0, // knn + -1, // tau + columns[ i ],// columns + targets[ i ],// target + "73 73 10", // libSizes_str + 200, // sample + true, // random + false, // replacement + 0, // seed + false, // includeData + false ); // verbose + + // Load cppEDM output + DataFrame< double > ccmThripsCSV = ccmThrips.AllLibStats; + + double rho1 = ccmThripsCSV( 0, 1 ); + double rho2 = ccmThripsCSV( 0, 2 ); + + ccmMatrix( rows1[ i ], cols1[ i ] ) = rho1; + ccmMatrix( rows2[ i ], cols2[ i ] ) = rho2; + } + + // ccmMatrix.WriteData( "./data", "Thrips_CMmatrix_valid.csv" ); + + //--------------------------------------------------------- + // Load cppEDM valid output + //--------------------------------------------------------- + DataFrame< double > thripsValid = + DataFrame < double > ( "./data/", + "Thrips_CMmatrix_valid.csv", + true ); // noTime = true + + // comparison + MakeTest ( "CCM Thrips test", thripsValid, ccmMatrix ); + // std::cout << ccmMatrix; } diff --git a/tests/TestCommon.h b/tests/TestCommon.h index 1257005..5ff816a 100644 --- a/tests/TestCommon.h +++ b/tests/TestCommon.h @@ -9,7 +9,7 @@ #define STR_LINE_SEP "-------------------------------------------------" #define TAB_CHAR '\t' -const float EPSILON = .2; +const float EPSILON = .01; //consts for different output color const std::string RED_TEXT ("\033[0;31m"); diff --git a/tests/data/Thrips_CMmatrix_valid.csv b/tests/data/Thrips_CMmatrix_valid.csv new file mode 100644 index 0000000..65b6a9d --- /dev/null +++ b/tests/data/Thrips_CMmatrix_valid.csv @@ -0,0 +1,5 @@ +Thrips_imaginis,maxT_degC,Rain_mm,Season +0.0000,0.617,0.45,0.57 +0.917,0.0000,0.816,0.962 +0.498,0.465,0.0000,0.389 +0.95,0.991,0.77,0.0000 diff --git a/tests/makefile b/tests/makefile index 7a3c247..b29d306 100644 --- a/tests/makefile +++ b/tests/makefile @@ -34,7 +34,8 @@ clean: rm -f TestCommon.o $(OBJ) $(EXE) distclean: - rm -f TestCommon.o $(OBJ) $(EXE) *~ *.bak *.csv ./data/*_cppEDM.csv + rm -f TestCommon.o $(OBJ) $(EXE) *~ *.bak + rm -f *.csv ./data/*_cppEDM.csv ./data/Smap_circle_coef.csv SRCS = `echo ${OBJ} | sed -e 's/.o /.cc /g'` depend: From 332d117d31a4416bdf5c5fd1520e3691beef15b3 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Tue, 23 Jun 2020 06:49:04 -0400 Subject: [PATCH 08/13] v1.5.0 CCM move Simplex object into CrossMap. --- src/API.cc | 10 +- src/CCM.cc | 206 ++++++++++++++++++------------------------ src/CCM.h | 11 ++- src/DataFrame.h | 12 +-- src/EDM.cc | 4 - src/EDM.h | 21 +++-- src/EDM_Formatting.cc | 48 ++++++++++ src/EDM_Neighbors.cc | 43 ++------- src/Parameter.cc | 2 - src/Parameter.h | 3 +- src/Simplex.cc | 5 - src/Simplex.h | 7 -- src/makefile | 2 +- tests/CCMTest.cc | 5 +- tests/makefile | 6 +- 15 files changed, 180 insertions(+), 205 deletions(-) diff --git a/src/API.cc b/src/API.cc index fc3e6b1..1b74405 100644 --- a/src/API.cc +++ b/src/API.cc @@ -10,12 +10,12 @@ #include "API.h" //---------------------------------------------------------------- -// Embed with file path/file input +// Embed from file path/file input //---------------------------------------------------------------- DataFrame< double > Embed( std::string path, std::string dataFile, int E, // embedding dimension - int tau, // time step delay + int tau, // time step offset std::string columns, // column names or indices bool verbose ) { @@ -26,7 +26,7 @@ DataFrame< double > Embed( std::string path, } //---------------------------------------------------------------- -// Embed with DataFrame input +// Embed from DataFrame input //---------------------------------------------------------------- DataFrame< double > Embed( DataFrame< double > & dataFrameIn, int E, @@ -453,8 +453,8 @@ CCMValues CCM( DataFrame< double > & DF, CCMValues values = CCMValues(); values.AllLibStats = CCMModel.allLibStats; - values.CrossMap1 = CCMModel.colToTarget; - values.CrossMap2 = CCMModel.targetToCol; + values.CrossMap1 = CCMModel.colToTargetValues; + values.CrossMap2 = CCMModel.targetToColValues; return values; } diff --git a/src/CCM.cc b/src/CCM.cc index 0114bd0..193444f 100644 --- a/src/CCM.cc +++ b/src/CCM.cc @@ -10,7 +10,9 @@ namespace EDM_CCM_Lock { //---------------------------------------------------------------- // forward declaration //---------------------------------------------------------------- -void CrossMap( SimplexClass & S ); +void CrossMap( DataFrame< double > & data, + Parameters & parameters, + CrossMapValues & values ); //---------------------------------------------------------------- // Constructor @@ -20,10 +22,13 @@ void CrossMap( SimplexClass & S ); CCMClass::CCMClass ( DataFrame< double > & data, Parameters & parameters ) : - SimplexClass ( data, parameters ), // base class initialise - colToTargetCCM( data, parameters ), // forward mapping object - targetToColCCM( data, parameters ) // reverse mapping object -{} + SimplexClass( data, parameters ) // base class initialise +{ + // Copy input parameters for forward and reverse mapping + // Reverse mapping of column : target in SetupParameters() + colToTargetParameters = parameters; + targetToColParameters = parameters; +} //---------------------------------------------------------------- // Project : Polymorphic implementation @@ -54,11 +59,15 @@ void CCMClass::CCM () { if ( parameters.columnNames.size() > 1 ) { std::cout << "WARNING: CCM() Only the first column will be mapped.\n"; } - + #ifdef CCM_THREADED - std::thread CrossMapColTarget( CrossMap, std::ref( colToTargetCCM ) ); - std::thread CrossMapTargetCol( CrossMap, std::ref( targetToColCCM ) ); + std::thread CrossMapColTarget( CrossMap, std::ref( data ), + std::ref( colToTargetParameters ), + std::ref( colToTargetValues ) ); + std::thread CrossMapTargetCol( CrossMap, std::ref( data ), + std::ref( targetToColParameters ), + std::ref( targetToColValues )); CrossMapColTarget.join(); CrossMapTargetCol.join(); @@ -76,8 +85,13 @@ void CCMClass::CCM () { std::rethrow_exception( exceptionPtr ); } #else - CrossMap( std::ref( colToTargetCCM ) ); - CrossMap( std::ref( targetToColCCM ) ); + CrossMap( std::ref( data ), + std::ref( colToTargetParameters ), + std::ref( colToTargetValues ) ); + + CrossMap( std::ref( data ), + std::ref( targetToColParameters ), + std::ref( targetToColValues ) ); #endif } @@ -85,27 +99,29 @@ void CCMClass::CCM () { // CrossMap() // Thread worker function for CCM. //---------------------------------------------------------------- -void CrossMap( SimplexClass & S ) { - - if ( S.parameters.verbose ) { +void CrossMap( DataFrame< double > & data, + Parameters & parameters, + CrossMapValues & values ) +{ + if ( parameters.verbose ) { std::lock_guard lck( EDM_CCM_Lock::mtx ); std::stringstream msg; msg << "CrossMap(): Simplex cross mapping from " - << S.parameters.columnNames[0] - << " to " << S.parameters.targetName << " E=" << S.parameters.E - << " knn=" << S.parameters.knn << " Library range: [" - << S.parameters.libSizes_str << "] "; - for ( size_t i = 0; i < S.parameters.librarySizes.size(); i++ ) { - msg << S.parameters.librarySizes[ i ] << " "; + << parameters.columnNames[0] + << " to " << parameters.targetName << " E=" << parameters.E + << " knn=" << parameters.knn << " Library range: [" + << parameters.libSizes_str << "] "; + for ( size_t i = 0; i < parameters.librarySizes.size(); i++ ) { + msg << parameters.librarySizes[ i ] << " "; } msg << std::endl << std::endl; std::cout << msg.str(); } try { - if ( S.parameters.E < 1 ) { + if ( parameters.E < 1 ) { std::lock_guard lck( EDM_CCM_Lock::mtx ); std::stringstream errMsg; - errMsg << "CrossMap(): E = " << S.parameters.E << " is invalid.\n" ; + errMsg << "CrossMap(): E = " << parameters.E << " is invalid.\n" ; throw std::runtime_error( errMsg.str() ); } @@ -113,9 +129,9 @@ void CrossMap( SimplexClass & S ) { // Set number of samples //----------------------------------------------------------------- size_t maxSamples; - if ( S.parameters.randomLib ) { + if ( parameters.randomLib ) { // Random samples from library - maxSamples = S.parameters.subSamples; + maxSamples = parameters.subSamples; } else { // Contiguous samples up to the size of the library @@ -125,30 +141,30 @@ void CrossMap( SimplexClass & S ) { //----------------------------------------------------------------- // Create random number generator: DefaultRandomEngine //----------------------------------------------------------------- - if ( S.parameters.randomLib ) { - if ( S.parameters.seed == 0 ) { + if ( parameters.randomLib ) { + if ( parameters.seed == 0 ) { // Select a random seed typedef std::chrono::high_resolution_clock CCMclock; CCMclock::time_point beginning = CCMclock::now(); CCMclock::duration duration = CCMclock::now() - beginning; - S.parameters.seed = duration.count(); + parameters.seed = duration.count(); } } - std::default_random_engine DefaultRandomEngine( S.parameters.seed ); + std::default_random_engine DefaultRandomEngine( parameters.seed ); //---------------------------------------------------------- // Predictions //---------------------------------------------------------- size_t predictionCount = 0; - size_t N_row = S.embedding.NRows(); + size_t N_row = data.NRows(); //---------------------------------------------------------- // Loop for library sizes //---------------------------------------------------------- for ( size_t libSize_i = 0; - libSize_i < S.parameters.librarySizes.size(); libSize_i++ ) { + libSize_i < parameters.librarySizes.size(); libSize_i++ ) { - size_t libSize = S.parameters.librarySizes[ libSize_i ]; + size_t libSize = parameters.librarySizes[ libSize_i ]; // Create random RNG sampler for this libSize out of N_row std::uniform_int_distribution< size_t > distribution( 0, N_row - 1 ); @@ -175,10 +191,10 @@ void CrossMap( SimplexClass & S ) { //------------------------------------------------------ std::vector< size_t > lib_i( libSize ); - if ( S.parameters.randomLib ) { + if ( parameters.randomLib ) { // Uniform random sample of rows - if ( S.parameters.replacement ) { + if ( parameters.replacement ) { // With replacement for ( size_t i = 0; i < libSize; i++ ) { lib_i[ i ] = distribution( DefaultRandomEngine ); @@ -209,6 +225,7 @@ void CrossMap( SimplexClass & S ) { lib_i = result; // Copy result to lib_i } + // std::sort( lib_i.begin(), lib_i.end() ); // JP why?? } else { // Not random samples, contiguous samples increasing size @@ -218,7 +235,7 @@ void CrossMap( SimplexClass & S ) { std::iota( lib_i.begin(), lib_i.end(), 0 ); libSize = N_row; - if ( S.parameters.verbose ) { + if ( parameters.verbose ) { std::stringstream msg; msg << "CCM(): Sequential library samples," << " max libSize is " << N_row @@ -263,19 +280,21 @@ void CrossMap( SimplexClass & S ) { //---------------------------------------------------------- // Set library and predict indices to lib_i //---------------------------------------------------------- - S.parameters.library.resize( lib_i.size() ); - std::iota( S.parameters.library.begin(), - S.parameters.library.end(), 0 ); - S.parameters.prediction.resize( lib_i.size() ); - std::iota( S.parameters.prediction.begin(), - S.parameters.prediction.end(), 0 ); - - S.CopyData(); // Reset to input data for subsetting - + parameters.library.resize( lib_i.size() ); + std::iota( parameters.library.begin(), + parameters.library.end(), 0 ); + parameters.prediction.resize( lib_i.size() ); + std::iota( parameters.prediction.begin(), + parameters.prediction.end(), 0 ); + // Subset data to lib_i rows - S.data = S.dataCCM.DataFrameFromRowIndex( lib_i ); + DataFrame< double > dataLib_i = + data.DataFrameFromRowIndex( lib_i ); + SimplexClass S = SimplexClass( std::ref( dataLib_i ), parameters ); + S.PrepareEmbedding( false ); // checkDataRows = false + //S.PrepareEmbedding(); S.Distances(); // Write EDM: allDistances, allLibRows @@ -285,6 +304,14 @@ void CrossMap( SimplexClass & S ) { S.FormatOutput(); +#ifdef JP_REMOVE + std::cout << "============== " + << S.parameters.columnNames[0] << " : " + << S.parameters.targetName + << " ============== " << std::endl; + std::cout << S.projection; // JP REMOVE +#endif + VectorError ve = ComputeError( S.projection.VectorColumnName( "Observations" ), S.projection.VectorColumnName( "Predictions" ) ); @@ -316,23 +343,12 @@ void CrossMap( SimplexClass & S ) { predOutVec[ 6 ] = ve.RMSE; // RMSE predOutVec[ 7 ] = ve.MAE; // MAE - if ( S.parameters.colToTargetFlag ) { - // S object is SimplexClass colToTargetCCM - // Write to EDM object CrossMapValues colToTarget - S.colToTarget.PredictStats.WriteRow( predictionCount, - predOutVec ); - // Save predictions - S.colToTarget.Predictions.push_front( S.projection ); - } - else { - // S object is SimplexClass targetToColCCM - // Write to EDM object CrossMapValues targetToCol - S.targetToCol.PredictStats.WriteRow( predictionCount, - predOutVec ); - S.targetToCol.Predictions.push_front( S.projection ); - } + values.PredictStats.WriteRow( predictionCount, + predOutVec ); + // Save predictions + values.Predictions.push_front( S.projection ); } - predictionCount++; + predictionCount++; } // for ( n = 0; n < maxSamples; n++ ) std::valarray< double > statVec( 4 ); @@ -341,12 +357,7 @@ void CrossMap( SimplexClass & S ) { statVec[ 2 ] = RMSE.sum() / maxSamples; statVec[ 3 ] = MAE.sum() / maxSamples; - if ( S.parameters.colToTargetFlag ) { - S.colToTarget.LibStats.WriteRow( libSize_i, statVec ); - } - else { - S.targetToCol.LibStats.WriteRow( libSize_i, statVec ); - } + values.LibStats.WriteRow( libSize_i, statVec ); } // for ( libSize_i < parameters.librarySizes ) } // try catch(...) { @@ -361,32 +372,10 @@ void CrossMap( SimplexClass & S ) { //----------------------------------------------------------------- void CCMClass::SetupParameters() { - { // JP Synchronization protection needed? - std::lock_guard lck( EDM_CCM_Lock::mtx ); - - // Each thread has it's own copy of input data & parameters - colToTargetCCM.dataCCM = data; // Copy - targetToColCCM.dataCCM = data; // Copy - - colToTargetCCM.parameters = parameters; // Copy - targetToColCCM.parameters = parameters; // Copy - - // Swap column : target in targetToColCCM - targetToColCCM.parameters.columnNames = + // Swap column : target in targetToColParameters + targetToColParameters.columnNames = std::vector< std::string >( 1, parameters.targetName ); - targetToColCCM.parameters.targetName = parameters.columnNames[0]; - - // Set flags to track direction of mapping for output data routing - colToTargetCCM.parameters.colToTargetFlag = true; - targetToColCCM.parameters.colToTargetFlag = false; - - // JP: No need for embedding, lib/pred adjust. Just to get target. - colToTargetCCM.PrepareEmbedding( false ); // embedding, target, lib, pred - targetToColCCM.PrepareEmbedding( false ); // embedding, target, lib, pred - - // Each thread has it's own copy of input target - colToTargetCCM.targetCCM = colToTargetCCM.target; // Copy - targetToColCCM.targetCCM = targetToColCCM.target; // Copy + targetToColParameters.targetName = parameters.columnNames[0]; //------------------------------------------------------------------ // DataFrames for output CrossMapValues structs in EDM object @@ -411,39 +400,23 @@ void CCMClass::SetupParameters() { DataFrame< double > LibStats2( parameters.librarySizes.size(), 4, "LibSize rho RMSE MAE" ); - // Instantiate EDM CrossMapValues output structs and insert DataFrames - colToTargetCCM.colToTarget = CrossMapValues(); - targetToColCCM.targetToCol = CrossMapValues(); + // Instantiate Simplex CrossMapValues output structs and insert DataFrames + colToTargetValues = CrossMapValues(); + targetToColValues = CrossMapValues(); - colToTargetCCM.colToTarget.LibStats = LibStats1; - targetToColCCM.targetToCol.LibStats = LibStats2; + colToTargetValues.LibStats = LibStats1; + targetToColValues.LibStats = LibStats2; if ( parameters.includeData ) { - colToTargetCCM.colToTarget.PredictStats = PredictionStats1; - targetToColCCM.targetToCol.PredictStats = PredictionStats2; + colToTargetValues.PredictStats = PredictionStats1; + targetToColValues.PredictStats = PredictionStats2; } - } // JP Synchronization protection needed? -} - -//---------------------------------------------------------------- -// Copy full library input data to EDM::Simplex objects for threads -//---------------------------------------------------------------- -void CCMClass::CopyData () { - - colToTargetCCM.data = colToTargetCCM.dataCCM; - targetToColCCM.data = targetToColCCM.dataCCM; - - colToTargetCCM.target = colToTargetCCM.targetCCM; - targetToColCCM.target = targetToColCCM.targetCCM; } //---------------------------------------------------------------- // //---------------------------------------------------------------- void CCMClass::FormatOutput () { - { // JP Synchronization protection needed? - std::lock_guard lck( EDM_CCM_Lock::mtx ); - // Create unified column names of output DataFrame std::stringstream libRhoNames; libRhoNames << "LibSize " @@ -454,10 +427,9 @@ void CCMClass::FormatOutput () { allLibStats = DataFrame< double >( parameters.librarySizes.size(), 3, libRhoNames.str() ); - allLibStats.WriteColumn(0, colToTargetCCM.colToTarget.LibStats.Column( 0 )); - allLibStats.WriteColumn(1, colToTargetCCM.colToTarget.LibStats.Column( 1 )); - allLibStats.WriteColumn(2, targetToColCCM.targetToCol.LibStats.Column( 1 )); - } // JP Synchronization protection needed? + allLibStats.WriteColumn(0, colToTargetValues.LibStats.Column( 0 )); + allLibStats.WriteColumn(1, colToTargetValues.LibStats.Column( 1 )); + allLibStats.WriteColumn(2, targetToColValues.LibStats.Column( 1 )); } //---------------------------------------------------------------- diff --git a/src/CCM.h b/src/CCM.h index eff4ca7..f6b4309 100644 --- a/src/CCM.h +++ b/src/CCM.h @@ -18,9 +18,15 @@ //---------------------------------------------------------------- class CCMClass : public SimplexClass { public: - SimplexClass colToTargetCCM; // object for column to target mapping - SimplexClass targetToColCCM; // object for target to column mapping + // CCM implements two Simplex objects for cross mapping + // Cross mapping results are stored here + DataFrame< double > allLibStats; // CCM unified libsize, rho, RMSE, MAE + CrossMapValues colToTargetValues; // CCM CrossMap() thread results + CrossMapValues targetToColValues; // CCM CrossMap() thread results + Parameters colToTargetParameters; + Parameters targetToColParameters; + // Constructor CCMClass ( DataFrame< double > & data, Parameters & parameters ); @@ -28,7 +34,6 @@ class CCMClass : public SimplexClass { // Method declarations void Project(); void SetupParameters(); - void CopyData(); void CCM(); void FormatOutput(); void WriteOutput(); diff --git a/src/DataFrame.h b/src/DataFrame.h index 88d449e..02a3f75 100644 --- a/src/DataFrame.h +++ b/src/DataFrame.h @@ -385,16 +385,8 @@ class DataFrame { //----------------------------------------------------------------- void DeletePartialDataRows( size_t nrows, int tau ) { - // NOTE : Not thread safe - - if ( partialDataRowsDeleted ) { - std::cout << "DeletePartialDataRows(): Partial data rows have " - "already been deleted." << std::endl; - return; - } - - partialDataRowsDeleted = true; - + // NOTE : Not thread safe : Call needs mutex wrap + if ( nrows > n_rows ) { std::stringstream errMsg; errMsg << "DataFrame::DeletePartialDataRows() " diff --git a/src/EDM.cc b/src/EDM.cc index 84d773c..ce539f7 100644 --- a/src/EDM.cc +++ b/src/EDM.cc @@ -12,10 +12,6 @@ EDM::EDM ( DataFrame< double > & data, Parameters & parameters ) : data( data ), anyTies( false ), parameters( parameters ) {} -//---------------------------------------------------------------- -// FindNeighbors : See EDM_Neighbors.cc -//---------------------------------------------------------------- - //---------------------------------------------------------------- // Project : Implemented in sub-class //---------------------------------------------------------------- diff --git a/src/EDM.h b/src/EDM.h index 4f14acb..6a55a9b 100644 --- a/src/EDM.h +++ b/src/EDM.h @@ -12,7 +12,7 @@ // Specific algorithm projection methods defined in sub-classes. // // NOTE JP: Tony recommends to explicitly define special members: -// http://www.cplusplus.com/doc/tutorial/classes2/ +// http://www.cplusplus.com/doc/tutorial/classes2/ //--------------------------------------------------------------------- class EDM { @@ -29,10 +29,6 @@ class EDM { DataFrame< double > projection; // Simplex & SMap Output DataFrame< double > coefficients; // SMap Output - DataFrame< double > allLibStats; // CCM unified libsize, rho, RMSE, MAE - CrossMapValues colToTarget; // CCM CrossMap() thread results - CrossMapValues targetToCol; // CCM CrossMap() thread results - // Project() vectors to populate projection DataFrame in FormatData() // JP Can we do away with these and write directly to projection (+Tp)? std::valarray< double > predictions; @@ -52,16 +48,21 @@ class EDM { EDM ( DataFrame< double > & data, Parameters & parameters ); // Method declarations - void CheckDataRows( std::string call ); + // EDM.cc + void EmbedData(); + void Project(); // Simplex.cc : SMap.cc : CCM.cc : Multiview.cc + + // EDM_Neighbors.cc void PrepareEmbedding( bool checkDataRows = true ); void Distances(); - void EmbedData(); void FindNeighbors(); - void Project(); + + // EDM_Formatting.cc + void CheckDataRows( std::string call ); + void RemovePartialData(); void FormatOutput(); void FillTimes( std::vector< std::string > & timeOut ); - void PrintLibPred(); // ifdef DEBUG_ALL - void PrintNeighbors(); // ifdef DEBUG_ALL + void PrintNeighbors(); // EDM_Neighbors.cc #ifdef DEBUG_ALL }; #endif diff --git a/src/EDM_Formatting.cc b/src/EDM_Formatting.cc index b2404de..32771ab 100644 --- a/src/EDM_Formatting.cc +++ b/src/EDM_Formatting.cc @@ -2,6 +2,54 @@ #include "EDM.h" #include "DateTime.h" +//---------------------------------------------------------- +// Clip data & target rows to match the embedding +//---------------------------------------------------------- +void EDM::RemovePartialData() +{ + // NOTE : Not thread safe : Call needs mutex wrap + + if ( data.PartialDataRowsDeleted() ) { + std::cout << "RemovePartialData(): Partial data rows have " + "already been deleted." << std::endl; + return; + } + + data.PartialDataRowsDeleted() = true; + + size_t shift = abs( parameters.tau ) * ( parameters.E - 1 ); + + // Resize target : copy target excluding partial data into targetEmbed + std::valarray< double > targetEmbed( data.NRows() - shift ); + + // Bogus cast to ( std::valarray ) for MSVC + // as it doesn't export its own slice_array applied to [] + if ( parameters.tau < 0 ) { + targetEmbed = ( std::valarray< double > ) + target[ std::slice( shift, target.size() - shift, 1 ) ]; + } + else { + targetEmbed = ( std::valarray< double > ) + target[ std::slice( 0, target.size() - shift, 1 ) ]; + } + + // Resize target to ignore partial data rows + target.resize( targetEmbed.size() ); + + // Copy target without partial data into resized targetIn + std::slice targetEmbed_i = std::slice( 0, targetEmbed.size(), 1 ); + target[ targetEmbed_i ] = + ( std::valarray< double > ) targetEmbed[ targetEmbed_i ]; + + // Delete data rows corresponding to embedding partial data rows + data.DeletePartialDataRows( shift, parameters.tau ); + + // Adjust parameters.library and parameters.prediction vectors of indices + if ( shift > 0 ) { + parameters.DeleteLibPred(); + } +} + //---------------------------------------------------------- // Validate dataFrameIn rows against lib and pred indices //---------------------------------------------------------- diff --git a/src/EDM_Neighbors.cc b/src/EDM_Neighbors.cc index 5c3e8e8..001fa16 100644 --- a/src/EDM_Neighbors.cc +++ b/src/EDM_Neighbors.cc @@ -10,7 +10,7 @@ namespace EDM_Neighbors_Lock { // 0) CheckDataRows() // 1) Extract or Embed() data into embedding // 2) Get target (library) vector -// 3) DeletePartialDataRows() +// 3) RemovePartialData() // 4) Adjust parameters.library and parameters.prediction indices // // NOTE: time column is not returned in the embedding dataBlock. @@ -65,53 +65,22 @@ void EDM::PrepareEmbedding( bool checkDataRows ) { //------------------------------------------------------------ // embedded = false: Embed() was called on data - // Remove target, data rows as needed + // Remove data & target rows as needed to match embedding // Adjust parameters.library and parameters.prediction indices //------------------------------------------------------------ if ( not parameters.embedded ) { if ( parameters.E < 1 ) { std::stringstream errMsg; - errMsg << "PreparEmbedding(): E = " << parameters.E - << " is invalid.\n" ; + errMsg << "PrepareEmbedding(): E = " << parameters.E + << " is invalid with embedded = true.\n" ; throw std::runtime_error( errMsg.str() ); } - size_t shift = abs( parameters.tau ) * ( parameters.E - 1 ); - - // Copy targetIn excluding partial data into targetEmbed - std::valarray< double > targetEmbed( data.NRows() - shift ); - - // Bogus cast to ( std::valarray ) for MSVC - // as it doesn't export its own slice_array applied to [] - if ( parameters.tau < 0 ) { - targetEmbed = ( std::valarray< double > ) - target[ std::slice( shift, target.size() - shift, 1 ) ]; - } - else { - targetEmbed = ( std::valarray< double > ) - target[ std::slice( 0, target.size() - shift, 1 ) ]; - } - - // Resize target to ignore partial data rows - target.resize( targetEmbed.size() ); - - // Copy target without partial data into resized targetIn - std::slice targetEmbed_i = std::slice( 0, targetEmbed.size(), 1 ); - target[ targetEmbed_i ] = ( std::valarray< double > ) - targetEmbed[ targetEmbed_i ]; - - // Delete dataIn top or bottom rows of partial data + // Delete data & target top or bottom rows of partial embedding data if ( not data.PartialDataRowsDeleted() ) { - // Not thread safe std::lock_guard lck( EDM_Neighbors_Lock::mtx ); - - data.DeletePartialDataRows( shift, parameters.tau ); - } - - // Adjust parameters.library and parameters.prediction vectors of indices - if ( shift > 0 ) { - parameters.DeleteLibPred(); + RemovePartialData(); } // Check boundaries again since rows were removed diff --git a/src/Parameter.cc b/src/Parameter.cc index c86ddd0..b02314d 100644 --- a/src/Parameter.cc +++ b/src/Parameter.cc @@ -42,7 +42,6 @@ Parameters::Parameters( bool replacement, unsigned seed, bool includeData, - bool colToTargetFlag, bool noNeighborLimit ) : // Variable initialization from Parameters arguments @@ -83,7 +82,6 @@ Parameters::Parameters( replacement ( replacement ), seed ( seed ), includeData ( includeData ), - colToTargetFlag ( colToTargetFlag ), noNeighborLimit ( noNeighborLimit ), // Set validated flag and instantiate Version diff --git a/src/Parameter.h b/src/Parameter.h index 071dad6..b400e6d 100644 --- a/src/Parameter.h +++ b/src/Parameter.h @@ -59,7 +59,7 @@ class Parameters { bool replacement; // CCM random select with replacement if true unsigned seed; // CCM random selection RNG seed bool includeData; // CCM include all simplex projection results - bool colToTargetFlag; // CCM thread flag to select output objects + //bool colToTargetFlag; // CCM thread flag to select output objects bool noNeighborLimit; // Strictly forbid neighbors outside library bool validated; @@ -106,7 +106,6 @@ class Parameters { bool replacement = false, unsigned seed = 0, // 0: Generate random seed in CCM bool includeData = false, - bool colToTargetFlag = true, bool noNeighborLimit = false ); diff --git a/src/Simplex.cc b/src/Simplex.cc index 71d155b..ce996b2 100644 --- a/src/Simplex.cc +++ b/src/Simplex.cc @@ -234,8 +234,3 @@ void SimplexClass::WriteOutput () { parameters.predictOutputFile ); } } - -//---------------------------------------------------------------- -// Implemented in CCMClass -//---------------------------------------------------------------- -void SimplexClass::CopyData () {} diff --git a/src/Simplex.h b/src/Simplex.h index 72ae490..5c1a6e3 100644 --- a/src/Simplex.h +++ b/src/Simplex.h @@ -10,12 +10,6 @@ //---------------------------------------------------------------- class SimplexClass : public EDM { public: - // CCMClass includes two instances of SimplexClass. One for - // forward mapping, one for reverse. These objects hold the - // original input data subsetted for each library size. - DataFrame < double > dataCCM; // Original, full data - std::valarray< double > targetCCM; // Original, full target - // Constructor SimplexClass ( DataFrame & data, Parameters & parameters ); @@ -23,7 +17,6 @@ class SimplexClass : public EDM { // Method declarations void Project(); void Simplex(); - void CopyData(); // CCMClass void WriteOutput(); }; #endif diff --git a/src/makefile b/src/makefile index b2dc274..3560e76 100644 --- a/src/makefile +++ b/src/makefile @@ -15,7 +15,7 @@ LIB = libEDM.a CFLAGS += -std=c++11 -Wpedantic -Wall -Wextra -Wreorder -O3 CFLAGS += -DCCM_THREADED -# CFLAGS += -g # -DDEBUG_ALL +CFLAGS += -g # -DDEBUG_ALL # LFLAGS = -L./ -lstdc++ -lEDM -lpthread -llapacke -llapack -lblas all: $(LIB) diff --git a/tests/CCMTest.cc b/tests/CCMTest.cc index 9a7f319..460af29 100644 --- a/tests/CCMTest.cc +++ b/tests/CCMTest.cc @@ -4,6 +4,8 @@ int main () { +#ifdef JP_REMOVE +#endif // Declare DataFrame to hold the valid output DataFrame< double > cppOutput; DataFrame< double > output; @@ -41,7 +43,7 @@ int main () { // comparison MakeTest ( "CCM sardine_anchovy_sst test", cppOutput, output ); - + //------------------------------------------------------------------ // Thrips // Create the ccm matrix of the rEDM-tutorial.Rmd vignette @@ -104,6 +106,7 @@ int main () { ccmMatrix( rows2[ i ], cols2[ i ] ) = rho2; } + std::cout << ccmMatrix; // ccmMatrix.WriteData( "./data", "Thrips_CMmatrix_valid.csv" ); //--------------------------------------------------------- diff --git a/tests/makefile b/tests/makefile index b29d306..42edf73 100644 --- a/tests/makefile +++ b/tests/makefile @@ -5,6 +5,7 @@ EXE = SimplexTest TestCommonTest SMapTest CCMTest MultiviewTest DateTimeTest OBJ = $(EXE:=.o) TestCommon.o CFLAGS = -std=c++11 -D PRINT_DIFFERENCE_IN_RESULTS +CFLAGS += -g LFLAGS = -lstdc++ -L../lib/ -I../src/ -lEDM -lpthread -llapack all: $(EXE) @@ -14,20 +15,23 @@ SimplexTest: SimplexTest.cc $(CC) TestCommon.cc -c $(CFLAGS) $(LFLAGS) $(CC) $@.cc -o $@ $(CFLAGS) $(LFLAGS) TestCommon.o -DateTimeTest: DateTimeTest.cc ../src/DateTimeUtil.cc +DateTimeTest: DateTimeTest.cc ../src/DateTime.cc $(CC) DateTimeTest.cc -c $(CFLAGS) $(LFLAGS) $(CC) $@.cc -o $@ $(CFLAGS) $(LFLAGS) TestCommon.o CCMTest: CCMTest.cc + $(CC) TestCommon.cc -c $(CFLAGS) $(LFLAGS) $(CC) $@.cc -o $@ $(CFLAGS) $(LFLAGS) TestCommon.o TestCommonTest: TestCommonTest.cc $(CC) $@.cc -o $@ $(CFLAGS) $(LFLAGS) TestCommon.o SMapTest: SMapTest.cc + $(CC) TestCommon.cc -c $(CFLAGS) $(LFLAGS) $(CC) $@.cc -o $@ $(CFLAGS) $(LFLAGS) TestCommon.o MultiviewTest: MultiviewTest.cc + $(CC) TestCommon.cc -c $(CFLAGS) $(LFLAGS) $(CC) $@.cc -o $@ $(CFLAGS) $(LFLAGS) TestCommon.o clean: From bed07527da21c04e6acc50adb3ac906e01fe9d43 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Tue, 23 Jun 2020 07:22:30 -0400 Subject: [PATCH 09/13] v1.5.0 CCM move Simplex object into CrossMap. --- src/CCM.cc | 7 +++---- src/DataFrame.h | 12 +++++++----- tests/CCMTest.cc | 2 -- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/CCM.cc b/src/CCM.cc index 193444f..f80a5fe 100644 --- a/src/CCM.cc +++ b/src/CCM.cc @@ -286,13 +286,12 @@ void CrossMap( DataFrame< double > & data, parameters.prediction.resize( lib_i.size() ); std::iota( parameters.prediction.begin(), parameters.prediction.end(), 0 ); - + // Subset data to lib_i rows - DataFrame< double > dataLib_i = - data.DataFrameFromRowIndex( lib_i ); + DataFrame< double > dataLib_i = data.DataFrameFromRowIndex( lib_i ); SimplexClass S = SimplexClass( std::ref( dataLib_i ), parameters ); - + S.PrepareEmbedding( false ); // checkDataRows = false //S.PrepareEmbedding(); diff --git a/src/DataFrame.h b/src/DataFrame.h index 02a3f75..cd4da67 100644 --- a/src/DataFrame.h +++ b/src/DataFrame.h @@ -399,11 +399,13 @@ class DataFrame { n_rows = n_rows - nrows; // Update time - if ( tau < 0 ) { - time.erase( time.begin(), time.begin() + nrows ); - } - else { - time.erase( time.end() - nrows, time.end() ); + if ( time.size() ) { + if ( tau < 0 ) { + time.erase( time.begin(), time.begin() + nrows ); + } + else { + time.erase( time.end() - nrows, time.end() ); + } } // Copy elements into data diff --git a/tests/CCMTest.cc b/tests/CCMTest.cc index 460af29..af245df 100644 --- a/tests/CCMTest.cc +++ b/tests/CCMTest.cc @@ -4,8 +4,6 @@ int main () { -#ifdef JP_REMOVE -#endif // Declare DataFrame to hold the valid output DataFrame< double > cppOutput; DataFrame< double > output; From d2988da6f68f0049bac85327f53fa8b1473128c0 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Sun, 28 Jun 2020 12:13:45 -0400 Subject: [PATCH 10/13] v1.5.0 update. --- data/paramecium_didinium.csv | 72 ++++++++ src/API.cc | 2 +- src/CCM.cc | 216 ++++++++++++----------- src/CCM.h | 6 +- src/DataFrame.h | 28 +-- src/EDM.cc | 16 ++ src/EDM.h | 4 +- src/EDM_Formatting.cc | 31 +--- src/EDM_Neighbors.cc | 32 ++-- tests/CCMTest.cc | 12 +- tests/MultiviewTest.cc | 4 +- tests/SMapTest.cc | 4 +- tests/SimplexTest.cc | 8 +- tests/data/CCM_anch_sst_cppEDM_valid.csv | 28 +-- tests/data/Thrips_CMmatrix_valid.csv | 8 +- tests/run | 2 +- 16 files changed, 275 insertions(+), 198 deletions(-) create mode 100644 data/paramecium_didinium.csv diff --git a/data/paramecium_didinium.csv b/data/paramecium_didinium.csv new file mode 100644 index 0000000..ff22cb1 --- /dev/null +++ b/data/paramecium_didinium.csv @@ -0,0 +1,72 @@ +time,paramecium,didinium +0,15.65,5.76 +0.52,53.57,9.05 +1.01,73.34,17.26 +1.54,93.93,41.97 +2.04,115.4,55.97 +2.51,76.57,74.91 +3,32.83,62.52 +3.46,23.74,27.04 +3.97,56.7,18.77 +4.5,86.37,31.11 +4.95,121,58.31 +5.47,71.48,73.13 +5.99,55.78,63.21 +6.46,31.84,52.46 +6.98,26.87,40.07 +7.45,53.24,27.67 +7.95,65.59,26 +8.5,81.23,24.32 +8.96,143.9,21 +9.49,237.9,33.35 +10,276.6,64.67 +10.47,222.2,94.34 +10.96,137.2,103.4 +11.47,46.45,82.74 +11.96,27.46,65.4 +12.46,41.46,51.35 +12.95,44.73,28.24 +13.46,88.42,23.27 +13.98,105.7,38.09 +14.49,155.2,14.97 +15.02,205.5,24.84 +15.51,312.7,49.56 +16.02,213.7,75.93 +15.99,163.4,104 +17,85.78,106.4 +17.52,48.64,100.6 +18.05,44.49,84.08 +18.51,63.44,45.3 +19.06,71.66,35.37 +19.51,127.7,35.35 +20.05,206.9,41.1 +20.54,309.9,52.62 +21.1,156.5,120.2 +21.56,63.3,112.8 +22.06,77.29,92.14 +22.52,45.11,65.72 +23.02,57.45,33.54 +23.54,69.8,21.14 +24.04,121.7,17.82 +24.58,185.2,26.04 +25.08,175.3,65.61 +25.6,139,76.3 +26.04,77.11,96.07 +26.51,57.29,68.84 +27.05,54.79,54.79 +27.55,75.38,35.8 +28.05,87.73,32.48 +28.55,136.4,24.21 +29.04,290.6,35.73 +29.55,345.8,55.5 +30.04,271.6,93.41 +30.59,156.1,117.3 +30.99,71.1,95.02 +31.48,43.86,85.92 +32.03,30.64,82.6 +32.5,35.56,66.08 +33.03,52.03,63.58 +33.49,37.99,37.99 +34.02,62.71,25.6 +34.49,103.9,23.1 +35.08,187.2,37.09 diff --git a/src/API.cc b/src/API.cc index 1b74405..023f0dc 100644 --- a/src/API.cc +++ b/src/API.cc @@ -412,7 +412,7 @@ CCMValues CCM( DataFrame< double > & DF, bool includeData, bool verbose ) { - // Set library and prediction indices to entire library + // Set library and prediction indices to entire library (embedded) std::stringstream ss; ss << "1 " << DF.NRows(); diff --git a/src/CCM.cc b/src/CCM.cc index f80a5fe..ed130ab 100644 --- a/src/CCM.cc +++ b/src/CCM.cc @@ -10,24 +10,22 @@ namespace EDM_CCM_Lock { //---------------------------------------------------------------- // forward declaration //---------------------------------------------------------------- -void CrossMap( DataFrame< double > & data, - Parameters & parameters, - CrossMapValues & values ); +void CrossMap( SimplexClass & S, + CrossMapValues & values ); //---------------------------------------------------------------- // Constructor -// data & parameters initialise EDM::SimplexClass parent, and, -// both mapping objects to the same initial parameters. +// Initialise EDM::SimplexClass parent, and, +// both mapping objects to the same initial data & parameters. //---------------------------------------------------------------- CCMClass::CCMClass ( DataFrame< double > & data, Parameters & parameters ) : - SimplexClass( data, parameters ) // base class initialise + SimplexClass( data, parameters ), // base class initialise + colToTarget ( data, parameters ), + targetToCol ( data, parameters ) { - // Copy input parameters for forward and reverse mapping - // Reverse mapping of column : target in SetupParameters() - colToTargetParameters = parameters; - targetToColParameters = parameters; + // Set targetToCol reverse mapping in SetupParameters() } //---------------------------------------------------------------- @@ -37,37 +35,45 @@ void CCMClass::Project () { SetupParameters(); // Forward and reverse mapping objects - CCM(); // PrepareEmbedding, Distances, FindNeighbors, Simplex + // Compute all distances in SimplexClass objects + // CrossMap() will take subsets of these for each library size + // with calls to FindNeighbors(), Simplex() in CrossMap() + colToTarget.PrepareEmbedding(); // embedding, target, RemovePartialData() + targetToCol.PrepareEmbedding(); // embedding, target, RemovePartialData() - FormatOutput(); + colToTarget.Distances(); // allDistances, allLibRows + targetToCol.Distances(); // allDistances, allLibRows + + CCM(); // FindNeighbors(), Simplex() + FormatOutput(); WriteOutput(); } //---------------------------------------------------------------- // CCM // To accomodate two threads running forward & inverse mapping -// CrossMap() is called with separate EDM::Simplex objects: -// SimplexClass colToTargetCCM; column to target mapping -// SimplexClass targetToColCCM; target to column mapping +// CrossMap() is called with separate Simplex and CrossMapValues: +// SimpexClass colToTarget; column to target mapping +// SimpexClass targetToCol; target to column mapping // These fill the respective EDM object CrossMapValues structs: -// CrossMapValues colToTarget; -// CrossMapValues targetToCol; +// CrossMapValues colToTargetValues; +// CrossMapValues targetToColValues; //---------------------------------------------------------------- void CCMClass::CCM () { - + if ( parameters.columnNames.size() > 1 ) { std::cout << "WARNING: CCM() Only the first column will be mapped.\n"; } #ifdef CCM_THREADED - std::thread CrossMapColTarget( CrossMap, std::ref( data ), - std::ref( colToTargetParameters ), + std::thread CrossMapColTarget( CrossMap, + std::ref( colToTarget ), std::ref( colToTargetValues ) ); - - std::thread CrossMapTargetCol( CrossMap, std::ref( data ), - std::ref( targetToColParameters ), - std::ref( targetToColValues )); + + std::thread CrossMapTargetCol( CrossMap, + std::ref( targetToCol ), + std::ref( targetToColValues ) ); CrossMapColTarget.join(); CrossMapTargetCol.join(); @@ -85,13 +91,9 @@ void CCMClass::CCM () { std::rethrow_exception( exceptionPtr ); } #else - CrossMap( std::ref( data ), - std::ref( colToTargetParameters ), - std::ref( colToTargetValues ) ); + CrossMap( std::ref( colToTarget ), std::ref( colToTargetValues ) ); - CrossMap( std::ref( data ), - std::ref( targetToColParameters ), - std::ref( targetToColValues ) ); + CrossMap( std::ref( targetToCol ), std::ref( targetToColValues ) ); #endif } @@ -99,39 +101,50 @@ void CCMClass::CCM () { // CrossMap() // Thread worker function for CCM. //---------------------------------------------------------------- -void CrossMap( DataFrame< double > & data, - Parameters & parameters, - CrossMapValues & values ) +void CrossMap( SimplexClass & S, + CrossMapValues & values ) { - if ( parameters.verbose ) { + if ( S.parameters.verbose ) { std::lock_guard lck( EDM_CCM_Lock::mtx ); std::stringstream msg; msg << "CrossMap(): Simplex cross mapping from " - << parameters.columnNames[0] - << " to " << parameters.targetName << " E=" << parameters.E - << " knn=" << parameters.knn << " Library range: [" - << parameters.libSizes_str << "] "; - for ( size_t i = 0; i < parameters.librarySizes.size(); i++ ) { - msg << parameters.librarySizes[ i ] << " "; + << S.parameters.columnNames[0] + << " to " << S.parameters.targetName << " E=" << S.parameters.E + << " knn=" << S.parameters.knn << " Library range: [" + << S.parameters.libSizes_str << "] "; + for ( size_t i = 0; i < S.parameters.librarySizes.size(); i++ ) { + msg << S.parameters.librarySizes[ i ] << " "; } msg << std::endl << std::endl; std::cout << msg.str(); } try { - if ( parameters.E < 1 ) { + if ( S.parameters.E < 1 ) { + std::lock_guard lck( EDM_CCM_Lock::mtx ); + std::stringstream errMsg; + errMsg << "CrossMap(): E = " << S.parameters.E << " is invalid.\n"; + throw std::runtime_error( errMsg.str() ); + } + + int shift = abs( S.parameters.tau ) * ( S.parameters.E - 1 ); + + if ( shift >= (int) S.data.NRows() ) { std::lock_guard lck( EDM_CCM_Lock::mtx ); std::stringstream errMsg; - errMsg << "CrossMap(): E = " << parameters.E << " is invalid.\n" ; + errMsg << "CrossMap(): Number of data rows " << S.data.NRows() + << " is not sufficient for tau*(E-1) = " << shift << ".\n"; throw std::runtime_error( errMsg.str() ); } + size_t N_row = S.embedding.NRows(); + //----------------------------------------------------------------- // Set number of samples //----------------------------------------------------------------- size_t maxSamples; - if ( parameters.randomLib ) { + if ( S.parameters.randomLib ) { // Random samples from library - maxSamples = parameters.subSamples; + maxSamples = S.parameters.subSamples; } else { // Contiguous samples up to the size of the library @@ -141,30 +154,29 @@ void CrossMap( DataFrame< double > & data, //----------------------------------------------------------------- // Create random number generator: DefaultRandomEngine //----------------------------------------------------------------- - if ( parameters.randomLib ) { - if ( parameters.seed == 0 ) { + if ( S.parameters.randomLib ) { + if ( S.parameters.seed == 0 ) { // Select a random seed typedef std::chrono::high_resolution_clock CCMclock; CCMclock::time_point beginning = CCMclock::now(); CCMclock::duration duration = CCMclock::now() - beginning; - parameters.seed = duration.count(); + S.parameters.seed = duration.count(); } } - std::default_random_engine DefaultRandomEngine( parameters.seed ); + std::default_random_engine DefaultRandomEngine( S.parameters.seed ); //---------------------------------------------------------- // Predictions //---------------------------------------------------------- size_t predictionCount = 0; - size_t N_row = data.NRows(); //---------------------------------------------------------- // Loop for library sizes //---------------------------------------------------------- for ( size_t libSize_i = 0; - libSize_i < parameters.librarySizes.size(); libSize_i++ ) { + libSize_i < S.parameters.librarySizes.size(); libSize_i++ ) { - size_t libSize = parameters.librarySizes[ libSize_i ]; + size_t libSize = S.parameters.librarySizes[ libSize_i ]; // Create random RNG sampler for this libSize out of N_row std::uniform_int_distribution< size_t > distribution( 0, N_row - 1 ); @@ -172,8 +184,9 @@ void CrossMap( DataFrame< double > & data, #ifdef DEBUG_ALL { std::lock_guard lck( EDM_CCM_Lock::mtx ); - std::cout << "libSize: " << libSize - << " ------------------------------------------\n"; + std::cout << "N_row: " << N_row << " maxSamples: " << maxSamples + << " libSize: " << libSize + << " ------------------------------------------\n"; } #endif // Output statistics vectors @@ -191,10 +204,9 @@ void CrossMap( DataFrame< double > & data, //------------------------------------------------------ std::vector< size_t > lib_i( libSize ); - if ( parameters.randomLib ) { + if ( S.parameters.randomLib ) { // Uniform random sample of rows - - if ( parameters.replacement ) { + if ( S.parameters.replacement ) { // With replacement for ( size_t i = 0; i < libSize; i++ ) { lib_i[ i ] = distribution( DefaultRandomEngine ); @@ -222,10 +234,9 @@ void CrossMap( DataFrame< double > & data, // Copy samples into result std::vector result( samples.begin(), samples.end() ); - + lib_i = result; // Copy result to lib_i } - // std::sort( lib_i.begin(), lib_i.end() ); // JP why?? } else { // Not random samples, contiguous samples increasing size @@ -234,8 +245,8 @@ void CrossMap( DataFrame< double > & data, lib_i.resize( N_row ); std::iota( lib_i.begin(), lib_i.end(), 0 ); libSize = N_row; - - if ( parameters.verbose ) { + + if ( S.parameters.verbose ) { std::stringstream msg; msg << "CCM(): Sequential library samples," << " max libSize is " << N_row @@ -252,7 +263,7 @@ void CrossMap( DataFrame< double > & data, // n + libSize > N_row, wrap around to data origin std::vector< size_t > lib_start( N_row - n ); std::iota( lib_start.begin(), lib_start.end(), n ); - + size_t max_i = std::min( libSize-(N_row - n), N_row ); std::vector< size_t > lib_wrap( max_i ); std::iota( lib_wrap.begin(), lib_wrap.end(), 0 ); @@ -276,52 +287,42 @@ void CrossMap( DataFrame< double > & data, } std::cout << std::endl; } #endif - //---------------------------------------------------------- - // Set library and predict indices to lib_i + // Local SimplexClass object for mapping + // Uses subset of CCMClass SimplexClass object //---------------------------------------------------------- - parameters.library.resize( lib_i.size() ); - std::iota( parameters.library.begin(), - parameters.library.end(), 0 ); - parameters.prediction.resize( lib_i.size() ); - std::iota( parameters.prediction.begin(), - parameters.prediction.end(), 0 ); - - // Subset data to lib_i rows - DataFrame< double > dataLib_i = data.DataFrameFromRowIndex( lib_i ); - - SimplexClass S = SimplexClass( std::ref( dataLib_i ), parameters ); - - S.PrepareEmbedding( false ); // checkDataRows = false - //S.PrepareEmbedding(); - - S.Distances(); // Write EDM: allDistances, allLibRows + SimplexClass Simplex_( S.data, S.parameters ); - S.FindNeighbors(); // On allDistances allLibRows - - S.Simplex(); + //---------------------------------------------------------- + // Subset Distances and lib row indices to lib_i + //---------------------------------------------------------- + Simplex_.allLibRows = + S.allLibRows.DataFrameFromColumnIndex( lib_i ); - S.FormatOutput(); - -#ifdef JP_REMOVE - std::cout << "============== " - << S.parameters.columnNames[0] << " : " - << S.parameters.targetName - << " ============== " << std::endl; - std::cout << S.projection; // JP REMOVE -#endif + Simplex_.allDistances = + S.allDistances.DataFrameFromColumnIndex( lib_i ); + + //---------------------------------------------------------- + // Cross mapping + //---------------------------------------------------------- + Simplex_.GetTarget(); + Simplex_.FindNeighbors(); + Simplex_.Simplex(); + Simplex_.FormatOutput(); VectorError ve = ComputeError( - S.projection.VectorColumnName( "Observations" ), - S.projection.VectorColumnName( "Predictions" ) ); + Simplex_.projection.VectorColumnName( "Observations" ), + Simplex_.projection.VectorColumnName( "Predictions" ) ); #ifdef DEBUG_ALL { std::lock_guard lck( EDM_CCM_Lock::mtx ); - std::cout << "CCM Simplex ---------------------------------\n"; - S.projection.MaxRowPrint() = S.projection.NRows(); - std::cout << S.projection; - std::cout << "rho " << ve.rho << " RMSE " << ve.RMSE + std::cout << "CCM Simplex -------- Column: "; + std::cout << Simplex_.parameters.columnNames[0] << " : Target: " + << Simplex_.parameters.targetName << " --------\n"; + // Simplex_.projection.MaxRowPrint() = Simplex_.projection.NRows(); + // std::cout << Simplex_.projection; + std::cout << " rho " << ve.rho << " RMSE " << ve.RMSE << " MAE " << ve.MAE << std::endl; } #endif @@ -347,6 +348,7 @@ void CrossMap( DataFrame< double > & data, // Save predictions values.Predictions.push_front( S.projection ); } + predictionCount++; } // for ( n = 0; n < maxSamples; n++ ) @@ -360,9 +362,13 @@ void CrossMap( DataFrame< double > & data, } // for ( libSize_i < parameters.librarySizes ) } // try catch(...) { +#ifdef CCM_THREADED // push exception pointer onto queue for main thread to catch std::lock_guard lck( EDM_CCM_Lock::q_mtx ); EDM_CCM_Lock::exceptionQ.push( std::current_exception() ); +#else + throw std::rethrow_exception( std::current_exception() ); +#endif } } @@ -371,10 +377,10 @@ void CrossMap( DataFrame< double > & data, //----------------------------------------------------------------- void CCMClass::SetupParameters() { - // Swap column : target in targetToColParameters - targetToColParameters.columnNames = - std::vector< std::string >( 1, parameters.targetName ); - targetToColParameters.targetName = parameters.columnNames[0]; + // Swap column : target in targetToCol.parameters + targetToCol.parameters.columns_str = parameters.targetName; + targetToCol.parameters.target_str = parameters.columnNames[0]; + targetToCol.parameters.Validate(); //------------------------------------------------------------------ // DataFrames for output CrossMapValues structs in EDM object @@ -421,14 +427,14 @@ void CCMClass::FormatOutput () { libRhoNames << "LibSize " << parameters.columnNames[0] <<":"<< parameters.targetName << " " << parameters.targetName <<":"<< parameters.columnNames[0]; - + // Allocate unified LibStats output DataFrame in EDM object allLibStats = DataFrame< double >( parameters.librarySizes.size(), 3, libRhoNames.str() ); - allLibStats.WriteColumn(0, colToTargetValues.LibStats.Column( 0 )); - allLibStats.WriteColumn(1, colToTargetValues.LibStats.Column( 1 )); - allLibStats.WriteColumn(2, targetToColValues.LibStats.Column( 1 )); + allLibStats.WriteColumn( 0, colToTargetValues.LibStats.Column( 0 ) ); + allLibStats.WriteColumn( 1, colToTargetValues.LibStats.Column( 1 ) ); + allLibStats.WriteColumn( 2, targetToColValues.LibStats.Column( 1 ) ); } //---------------------------------------------------------------- diff --git a/src/CCM.h b/src/CCM.h index f6b4309..16dfec6 100644 --- a/src/CCM.h +++ b/src/CCM.h @@ -19,14 +19,14 @@ class CCMClass : public SimplexClass { public: // CCM implements two Simplex objects for cross mapping + SimplexClass colToTarget; + SimplexClass targetToCol; + // Cross mapping results are stored here DataFrame< double > allLibStats; // CCM unified libsize, rho, RMSE, MAE CrossMapValues colToTargetValues; // CCM CrossMap() thread results CrossMapValues targetToColValues; // CCM CrossMap() thread results - Parameters colToTargetParameters; - Parameters targetToColParameters; - // Constructor CCMClass ( DataFrame< double > & data, Parameters & parameters ); diff --git a/src/DataFrame.h b/src/DataFrame.h index cd4da67..971346c 100644 --- a/src/DataFrame.h +++ b/src/DataFrame.h @@ -188,9 +188,9 @@ class DataFrame { //----------------------------------------------------------------- // Return (sub)DataFrame of specified column indices //----------------------------------------------------------------- - DataFrame DataFrameFromColumnIndex( std::vector column_i ) { + DataFrame< T > DataFrameFromColumnIndex( std::vector column_i ) { - DataFrame M = DataFrame( n_rows, column_i.size() ); + DataFrame< T > M = DataFrame( n_rows, column_i.size() ); // Can't use slice since column_i are not structured size_t col_j = 0; @@ -205,7 +205,7 @@ class DataFrame { throw std::runtime_error( errMsg.str() ); } - std::valarray< double > column_vec_i = Column( col_i ); + std::valarray< T > column_vec_i = Column( col_i ); M.WriteColumn( col_j, column_vec_i ); col_j++; @@ -233,7 +233,7 @@ class DataFrame { // Return (sub)DataFrame selected by columnNames // columnNames converted to column indices for DataFrameFromColumnIndex() //------------------------------------------------------------------ - DataFrame< double > DataFrameFromColumnNames( + DataFrame< T > DataFrameFromColumnNames( std::vector colNames ) { // vector of column indices for DataFrameFromColumnIndex() @@ -264,7 +264,7 @@ class DataFrame { throw std::runtime_error( errMsg.str() ); } - DataFrame< double > M_col = DataFrameFromColumnIndex( col_i_vec ); + DataFrame< T > M_col = DataFrameFromColumnIndex( col_i_vec ); // Insert columnNames if not already present if ( not M_col.ColumnNames().size() ) { @@ -278,9 +278,9 @@ class DataFrame { //----------------------------------------------------------------- // Return (sub)DataFrame of specified row indices //----------------------------------------------------------------- - DataFrame DataFrameFromRowIndex( std::vector row_index ) { + DataFrame< T > DataFrameFromRowIndex( std::vector row_index ) { - DataFrame< double > M = DataFrame( row_index.size(), n_columns ); + DataFrame< T > M = DataFrame( row_index.size(), n_columns ); // Can't use slice since row_index_i are not structured size_t row_j = 0; @@ -293,7 +293,7 @@ class DataFrame { throw std::runtime_error( errMsg.str() ); } - std::valarray row_vec_i = Row( row_i ); + std::valarray< T > row_vec_i = Row( row_i ); M.WriteRow( row_j, row_vec_i ); row_j++; @@ -320,9 +320,9 @@ class DataFrame { //----------------------------------------------------------------- // Return Elements in Column Major order (Fortran) //----------------------------------------------------------------- - std::valarray ColumnMajorData() const { + std::valarray< T > ColumnMajorData() const { - std::valarray colMajorElements( elements.size() ); + std::valarray< T > colMajorElements( elements.size() ); for ( size_t col = 0; col < n_columns; col++ ) { // slice( size_t start, size_t length, size_t stride ) @@ -336,7 +336,7 @@ class DataFrame { //----------------------------------------------------------------- // Write array to row //----------------------------------------------------------------- - void WriteRow( size_t row, std::valarray array ) { + void WriteRow( size_t row, std::valarray< T > array ) { size_t N = array.size(); if ( N != n_columns ) { @@ -359,7 +359,7 @@ class DataFrame { //----------------------------------------------------------------- // Write array to col //----------------------------------------------------------------- - void WriteColumn( size_t col, std::valarray array ) { + void WriteColumn( size_t col, std::valarray< T > array ) { size_t N = array.size(); if ( N != n_rows ) { @@ -409,7 +409,7 @@ class DataFrame { } // Copy elements into data - std::valarray< double > data( elements ); + std::valarray< T > data( elements ); // Resize elements size_t n_elements = elements.size() - nrows * n_columns; @@ -426,7 +426,7 @@ class DataFrame { // Bogus cast for MSVC elements[ std::slice( 0, n_elements, 1 ) ] = - ( std::valarray< double > ) data[ elements_i ]; + ( std::valarray< T > ) data[ elements_i ]; } //----------------------------------------------------------------- diff --git a/src/EDM.cc b/src/EDM.cc index ce539f7..e52b5be 100644 --- a/src/EDM.cc +++ b/src/EDM.cc @@ -17,6 +17,22 @@ EDM::EDM ( DataFrame< double > & data, //---------------------------------------------------------------- void EDM::Project () {} +//---------------------------------------------------------------- +// Set target (library) vector +//---------------------------------------------------------------- +void EDM::GetTarget() { + if ( parameters.targetIndex ) { + target = data.Column( parameters.targetIndex ); + } + else if ( parameters.targetName.size() ) { + target = data.VectorColumnName( parameters.targetName ); + } + else { + // Default to first column + target = data.Column( 0 ); + } +} + //---------------------------------------------------------------- // Implemented as a wrapper for API MakeBlock() // Note: dataFrame must have the columnNameToIndex map diff --git a/src/EDM.h b/src/EDM.h index 6a55a9b..5191b2b 100644 --- a/src/EDM.h +++ b/src/EDM.h @@ -49,6 +49,7 @@ class EDM { // Method declarations // EDM.cc + void GetTarget(); void EmbedData(); void Project(); // Simplex.cc : SMap.cc : CCM.cc : Multiview.cc @@ -63,6 +64,7 @@ class EDM { void FormatOutput(); void FillTimes( std::vector< std::string > & timeOut ); - void PrintNeighbors(); // EDM_Neighbors.cc #ifdef DEBUG_ALL + void PrintDataFrameIn(); // EDM_Neighbors.cc #ifdef DEBUG_ALL + void PrintNeighbors(); // EDM_Neighbors.cc #ifdef DEBUG_ALL }; #endif diff --git a/src/EDM_Formatting.cc b/src/EDM_Formatting.cc index 32771ab..905f9a1 100644 --- a/src/EDM_Formatting.cc +++ b/src/EDM_Formatting.cc @@ -17,33 +17,13 @@ void EDM::RemovePartialData() data.PartialDataRowsDeleted() = true; - size_t shift = abs( parameters.tau ) * ( parameters.E - 1 ); - - // Resize target : copy target excluding partial data into targetEmbed - std::valarray< double > targetEmbed( data.NRows() - shift ); - - // Bogus cast to ( std::valarray ) for MSVC - // as it doesn't export its own slice_array applied to [] - if ( parameters.tau < 0 ) { - targetEmbed = ( std::valarray< double > ) - target[ std::slice( shift, target.size() - shift, 1 ) ]; - } - else { - targetEmbed = ( std::valarray< double > ) - target[ std::slice( 0, target.size() - shift, 1 ) ]; - } - - // Resize target to ignore partial data rows - target.resize( targetEmbed.size() ); - - // Copy target without partial data into resized targetIn - std::slice targetEmbed_i = std::slice( 0, targetEmbed.size(), 1 ); - target[ targetEmbed_i ] = - ( std::valarray< double > ) targetEmbed[ targetEmbed_i ]; - + int shift = abs( parameters.tau ) * ( parameters.E - 1 ); + // Delete data rows corresponding to embedding partial data rows data.DeletePartialDataRows( shift, parameters.tau ); + GetTarget(); + // Adjust parameters.library and parameters.prediction vectors of indices if ( shift > 0 ) { parameters.DeleteLibPred(); @@ -88,6 +68,9 @@ void EDM::CheckDataRows( std::string call ) throw std::runtime_error( errMsg.str() ); } + // Tweak for CCM that sets lib = pred = [1, NRow] + if ( parameters.method == Method::CCM ) { shift = 0; } + if ( data.NRows() <= library_max_i + shift ) { std::stringstream errMsg; errMsg << "CheckDataRows(): " << call diff --git a/src/EDM_Neighbors.cc b/src/EDM_Neighbors.cc index 001fa16..798318b 100644 --- a/src/EDM_Neighbors.cc +++ b/src/EDM_Neighbors.cc @@ -51,17 +51,7 @@ void EDM::PrepareEmbedding( bool checkDataRows ) { EmbedData(); } - // Get target (library) vector - if ( parameters.targetIndex ) { - target = data.Column( parameters.targetIndex ); - } - else if ( parameters.targetName.size() ) { - target = data.VectorColumnName( parameters.targetName ); - } - else { - // Default to first column - target = data.Column( 0 ); - } + GetTarget(); //------------------------------------------------------------ // embedded = false: Embed() was called on data @@ -158,6 +148,7 @@ void EDM::FindNeighbors() { predPairs( N_prediction_rows ); for ( size_t pred_row = 0; pred_row < N_prediction_rows; pred_row++ ) { + std::valarray< double > rowDist = allDistances.Row( pred_row ); std::vector< std::pair< double, size_t > > rowPairs( rowDist.size() ); @@ -209,19 +200,28 @@ void EDM::FindNeighbors() { // distance must be .first std::sort( rowPair.begin(), rowPair.end(), DistanceCompare ); + //---------------------------------------------------------------- // Insert knn distance / library row index into knn vectors - std::valarray< double > knnDistances( parameters.knn ); - std::valarray< size_t > knnLibRows ( parameters.knn ); + //---------------------------------------------------------------- + // JP: This is sneaky: knnDistances & knnLibRows are initialised + // to nan, which translate to "quiet nan". Following PEP 20, + // generate WARNING if parameters.knn neighbors are not found. + std::valarray< double > knnDistances( nanf("knn"), parameters.knn ); + std::valarray< size_t > knnLibRows ( nanl("knn"), parameters.knn ); int lib_row_i = 0; int k = 0; while ( k < parameters.knn ) { if ( lib_row_i >= rowPairSize ) { std::stringstream errMsg; - errMsg << "FindNeighbors(): knn search failed. " + errMsg << "WARNING: FindNeighbors(): knn search failed " + << "at prediction row " << predictionRow << ". " << k << " out of " << parameters.knn - << " neighbors were found in the library.\n" ; - throw std::runtime_error( errMsg.str() ); + << " neighbors were found in the library.\n"; + std::cout << errMsg.str(); + + k = (int) rowPair.size(); // Avoid tie check below + break; // Continue to next row } double distance = rowPair[ lib_row_i ].first; diff --git a/tests/CCMTest.cc b/tests/CCMTest.cc index af245df..079a986 100644 --- a/tests/CCMTest.cc +++ b/tests/CCMTest.cc @@ -19,7 +19,7 @@ int main () { CCMValues ccmOut = CCM( "../data/", // pathIn "sardine_anchovy_sst.csv", // dataFile "./data/", // pathOut - "CCM_anch_sst_cppEDM.csv", //predictFile + "CCM_anch_sst_cppEDM.csv", // predictFile 3, // E 0, // Tp 0, // knn @@ -38,9 +38,8 @@ int main () { output = DataFrame < double > ( "./data/", "CCM_anch_sst_cppEDM.csv", true ); // noTime = true - // comparison - MakeTest ( "CCM sardine_anchovy_sst test", cppOutput, output ); + MakeTest ( "CCM: sardine_anchovy_sst test", cppOutput, output ); //------------------------------------------------------------------ // Thrips @@ -104,8 +103,7 @@ int main () { ccmMatrix( rows2[ i ], cols2[ i ] ) = rho2; } - std::cout << ccmMatrix; - // ccmMatrix.WriteData( "./data", "Thrips_CMmatrix_valid.csv" ); + // ccmMatrix.WriteData( "./data/", "Thrips_CMmatrix_valid.csv" ); //--------------------------------------------------------- // Load cppEDM valid output @@ -114,8 +112,6 @@ int main () { DataFrame < double > ( "./data/", "Thrips_CMmatrix_valid.csv", true ); // noTime = true - // comparison - MakeTest ( "CCM Thrips test", thripsValid, ccmMatrix ); - // std::cout << ccmMatrix; + MakeTest ( "CCM: Thrips test", thripsValid, ccmMatrix ); } diff --git a/tests/MultiviewTest.cc b/tests/MultiviewTest.cc index e422021..a821562 100644 --- a/tests/MultiviewTest.cc +++ b/tests/MultiviewTest.cc @@ -47,8 +47,8 @@ int main( int argc, char *argv[] ) { // << " MAE " << ve.MAE << " RMSE " << ve.RMSE << std::endl; // comparison - MakeTest ("Multiview combos test", validCppCombos, combos ); - MakeTest ("Multiview prediction test", validCppPredict, output ); + MakeTest ("Multiview: combos test", validCppCombos, combos ); + MakeTest ("Multiview: prediction test", validCppPredict, output ); return 0; } diff --git a/tests/SMapTest.cc b/tests/SMapTest.cc index 079fcb1..c4d0154 100644 --- a/tests/SMapTest.cc +++ b/tests/SMapTest.cc @@ -27,7 +27,7 @@ int main () { DataFrame < double > cppCoef = smapVals.coefficients; // Comparison - MakeTest ( "circle.csv test", coefOutput, cppCoef ); + MakeTest ( "SMap: circle test", coefOutput, cppCoef ); //--------------------------------------------------------- @@ -46,5 +46,5 @@ int main () { cppOutput = smapVals.predictions; // Comparison - MakeTest ( "block_3sp test", pyOutput, cppOutput ); + MakeTest ( "SMap: block_3sp test", pyOutput, cppOutput ); } diff --git a/tests/SimplexTest.cc b/tests/SimplexTest.cc index 6687d8d..71adebd 100644 --- a/tests/SimplexTest.cc +++ b/tests/SimplexTest.cc @@ -22,7 +22,8 @@ int main () { "1 99","100 198", 3, 1, 0, -1, 0, "x_t y_t z_t", "x_t", true, false, false ); // Comparison - MakeTest ( "block_3sp.csv embedded data test", pyOutput, cppOutput ); + MakeTest ( "Simplex: block_3sp.csv embedded data test", + pyOutput, cppOutput ); //---------------------------------------------------------- // Simplex prediction with dynamically embedded data @@ -38,7 +39,8 @@ int main () { "1 100", "101 195", 3, 1, 0, -1, 0, "x_t", "x_t", false, false, false ); // Comparison - MakeTest ( "block_3sp.csv dynamic embedding test", pyOutput, cppOutput ); + MakeTest ( "Simplex: block_3sp.csv dynamic embedding test", + pyOutput, cppOutput ); //------------------------------------------------------------------- @@ -60,6 +62,6 @@ int main () { // std::cout << cppOutput; // ISO datetime in Time column - MakeTest ( "S12CD-S333 ISO datetime", pyOutput, cppOutput ); + MakeTest ( "Simplex: S12CD-S333 ISO datetime", pyOutput, cppOutput ); } diff --git a/tests/data/CCM_anch_sst_cppEDM_valid.csv b/tests/data/CCM_anch_sst_cppEDM_valid.csv index 6200902..73f5b92 100644 --- a/tests/data/CCM_anch_sst_cppEDM_valid.csv +++ b/tests/data/CCM_anch_sst_cppEDM_valid.csv @@ -1,15 +1,15 @@ LibSize,anchovy:np_sst,np_sst:anchovy -10.0000,-0.6530,-0.8953 -15.0000,-0.7293,0.2875 -20.0000,-0.7014,-0.4845 -25.0000,-0.3149,0.6335 -30.0000,0.1033,-0.2095 -35.0000,-0.0756,-0.1894 -40.0000,-0.0699,-0.0061 -45.0000,0.0126,0.0165 -50.0000,0.0958,-0.1779 -55.0000,0.0233,-0.0728 -60.0000,0.1149,-0.0749 -65.0000,0.1374,-0.0755 -70.0000,0.0726,-0.0744 -75.0000,0.2008,-0.0681 +10.0000,-0.0919,0.0370 +15.0000,-0.2160,0.0337 +20.0000,-0.1988,-0.1671 +25.0000,0.0368,0.0226 +30.0000,0.1145,0.0894 +35.0000,0.1601,0.1012 +40.0000,0.1389,0.0064 +45.0000,0.1345,-0.1908 +50.0000,0.1290,-0.1149 +55.0000,0.1111,-0.0650 +60.0000,0.1246,-0.0656 +65.0000,0.0820,-0.0662 +70.0000,0.1686,-0.0615 +75.0000,0.2139,-0.0912 diff --git a/tests/data/Thrips_CMmatrix_valid.csv b/tests/data/Thrips_CMmatrix_valid.csv index 65b6a9d..2955f32 100644 --- a/tests/data/Thrips_CMmatrix_valid.csv +++ b/tests/data/Thrips_CMmatrix_valid.csv @@ -1,5 +1,5 @@ Thrips_imaginis,maxT_degC,Rain_mm,Season -0.0000,0.617,0.45,0.57 -0.917,0.0000,0.816,0.962 -0.498,0.465,0.0000,0.389 -0.95,0.991,0.77,0.0000 +0.0000,0.6045,0.4291,0.5609 +0.9222,0.0000,0.8212,0.9625 +0.5119,0.4639,0.0000,0.3931 +0.9544,0.9918,0.7771,0.0000 diff --git a/tests/run b/tests/run index 0fc3abb..108c6ca 100755 --- a/tests/run +++ b/tests/run @@ -1,7 +1,7 @@ #!/bin/bash make ./SimplexTest -./MultiviewTest ./SMapTest +./MultiviewTest ./CCMTest make distclean From 9cf20623826a67f7d89893ab88203c50a75cb707 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Tue, 30 Jun 2020 15:04:00 -0400 Subject: [PATCH 11/13] v1.5.0 update. --- src/CCM.cc | 10 +++++----- tests/TestCommon.cc | 16 +++++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/CCM.cc b/src/CCM.cc index ed130ab..e5e6889 100644 --- a/src/CCM.cc +++ b/src/CCM.cc @@ -10,8 +10,8 @@ namespace EDM_CCM_Lock { //---------------------------------------------------------------- // forward declaration //---------------------------------------------------------------- -void CrossMap( SimplexClass & S, - CrossMapValues & values ); +void CrossMap( SimplexClass & S, // input + CrossMapValues & values ); // output //---------------------------------------------------------------- // Constructor @@ -43,7 +43,7 @@ void CCMClass::Project () { colToTarget.Distances(); // allDistances, allLibRows targetToCol.Distances(); // allDistances, allLibRows - + CCM(); // FindNeighbors(), Simplex() FormatOutput(); @@ -92,7 +92,6 @@ void CCMClass::CCM () { } #else CrossMap( std::ref( colToTarget ), std::ref( colToTargetValues ) ); - CrossMap( std::ref( targetToCol ), std::ref( targetToColValues ) ); #endif } @@ -237,6 +236,7 @@ void CrossMap( SimplexClass & S, lib_i = result; // Copy result to lib_i } + // std::sort( lib_i.begin(), lib_i.end() ); // JP Why? } else { // Not random samples, contiguous samples increasing size @@ -300,7 +300,7 @@ void CrossMap( SimplexClass & S, S.allLibRows.DataFrameFromColumnIndex( lib_i ); Simplex_.allDistances = - S.allDistances.DataFrameFromColumnIndex( lib_i ); + S.allDistances.DataFrameFromColumnIndex( lib_i ); //---------------------------------------------------------- // Cross mapping diff --git a/tests/TestCommon.cc b/tests/TestCommon.cc index dd560c3..db16f13 100644 --- a/tests/TestCommon.cc +++ b/tests/TestCommon.cc @@ -69,23 +69,17 @@ void MakeTest (std::string testName, DataFrame< double > data1, if ( badRows.empty() ) { std::cout << GREEN_TEXT; - std::cout << TAB_CHAR << "TEST PASSED. All rows same."; + std::cout << TAB_CHAR << "PASSED. EPSILON: " << EPSILON; } else { int numBadRows = std::count_if( badRows.begin(), badRows.end(), [](int i){ return i != 0; } ); - if ( numBadRows < 5 ) { + if ( numBadRows ) { std::cout << YELLOW_TEXT; - std::cout << TAB_CHAR << "TEST MARGINALLY FAILED. " ; + std::cout << TAB_CHAR << "FAILED. EPSILON: " << EPSILON; } - else { - std::cout << RED_TEXT; - std::cout << TAB_CHAR << "TEST FAILED. " ; - } - std::cout << numBadRows << " rows different "; - std::cout << std::endl; #ifdef PRINT_DIFFERENCE_IN_RESULTS std::cout << TAB_CHAR << TAB_CHAR << "Block 1 column names: "; @@ -101,8 +95,8 @@ void MakeTest (std::string testName, DataFrame< double > data1, std::cout << TAB_CHAR << "first 10 different rows:" << std::endl; - for (auto iterate = badRows.begin(); - iterate - badRows.begin() < 10 && iterate != badRows.end(); + for (auto iterate = badRows.begin(); + iterate != badRows.end(); ++iterate) { std::valarray< double > badRow1 = data1.Row( *iterate ); From e5eb54bad0fd91444062e6b9234f6c2b2611a32d Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Tue, 30 Jun 2020 17:26:51 -0400 Subject: [PATCH 12/13] v1.5.0 update. --- src/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/makefile b/src/makefile index 3560e76..b2dc274 100644 --- a/src/makefile +++ b/src/makefile @@ -15,7 +15,7 @@ LIB = libEDM.a CFLAGS += -std=c++11 -Wpedantic -Wall -Wextra -Wreorder -O3 CFLAGS += -DCCM_THREADED -CFLAGS += -g # -DDEBUG_ALL +# CFLAGS += -g # -DDEBUG_ALL # LFLAGS = -L./ -lstdc++ -lEDM -lpthread -llapacke -llapack -lblas all: $(LIB) From 2c1615c5ac82bdfa4f7018a36dfb3d3d9544d510 Mon Sep 17 00:00:00 2001 From: SoftwareLiteracy Date: Tue, 30 Jun 2020 17:40:39 -0400 Subject: [PATCH 13/13] v1.5.0 update. --- src/Parameter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parameter.cc b/src/Parameter.cc index b02314d..940acb6 100644 --- a/src/Parameter.cc +++ b/src/Parameter.cc @@ -86,7 +86,7 @@ Parameters::Parameters( // Set validated flag and instantiate Version validated ( false ), - version ( 1, 5, 0, "2020-06-19" ) + version ( 1, 5, 0, "2020-07-01" ) { // Constructor code if ( method != Method::None ) {