#include <iostream>
#include <sstream>
#include <fstream>
#include <functional>

#include "../shared/DatabaseWithRetry.h"
#include "../shared/GlobalConfigFile.h"
#include "../shared/DatabaseForThread.h"
#include "DataFormat.h"
#include "Parse.h"
#include "FieldLists.h"
#include "Execution.h"
#include "Semantics.h"
#include "IAlertsDaily.h"
#include "UserFilters.h"
#include "Strategy.h"

#include "ParserTest.h"

using namespace Parse;

namespace ParserTest
{

  class FakeAlertsDaily : public IAlertsDaily
  {
  private:
    Record::Ref _record;
  public:
    virtual Record::Ref find(std::string const &symbol, time_t time) const
    { return _record; }
    FakeAlertsDaily()
    {
      RecordBuilder rb;
      rb.append(DailyFields::advol, 1000000);
      rb.append(DailyFields::list_exch, "NASD");
      rb.append(DailyFields::high_p, 100);
      rb.append(DailyFields::low_p, 50);
      _record = Record::create(rb.exportAsString());
    }
  };

  static void testDataFormat()
  {
    FieldId id[] = {10, 0xffff, 20, 30, 64, 66, 68, 70, 72, 71, 69, 67, 65, 'z',
                    'y', 'x', 'w', 'v', 'a', 'b', 'c', 'd', 'e', 'f', '`',
                    'g','h', 'i', 'j', 'k', 'l','m',
                    300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311,
                    312, 313 };
    RecordBuilder rb;
    rb.append(id[0], "The quick brown fox jumps over the lazy dog!");
    rb.append(id[1], "[" + std::string(253, 'P') + "]"); // 255 chars
    rb.append(id[2], "[" + std::string(254, 'S') + "]"); // 256 chars
    rb.append(id[3], 0);
    rb.append(id[4], 1);
    rb.append(id[5], -1);
    rb.append(id[6], 255);
    rb.append(id[7], 256);
    rb.append(id[8], 0x7fff);
    rb.append(id[9], (int)0x7fff);
    rb.append(id[10], 0x8000);
    rb.append(id[11], (int)0x8000);
    rb.append(id[12], -0x8000);
    rb.append(id[13], -0x8001);
    rb.append(id[14], 0x7fffffff);
    rb.append(id[15], 0x80000000);
    rb.append(id[16], -2147483648);  // -0x80000000
    rb.append(id[17], -2147483649);  // -0x80000001
    rb.append(id[18], 0.0);
    rb.append(id[19], 0.0f);
    rb.append(id[20], 1.23);
    rb.append(id[21], 1.23f);
    rb.append(id[22], 0.0/0.0);
    rb.reserveInt(id[23], 80);
    // Skip 24
    rb.append(id[25], 838.86);
    rb.append(id[26], 838.86f);
    rb.append(id[27], 838.8599);
    rb.append(id[28], 838.8599f);
    rb.append(id[29], -838.86);
    rb.append(id[30], -838.8599);
    rb.append(id[31], 1.23456);
    rb.append(id[32], 3.2767);
    rb.append(id[33], 3.276699);
    rb.append(id[34], -3.2767);
    rb.append(id[35], -3.276699);
    rb.append(id[36], 10.005);
    rb.append(id[37], 100.005);
    rb.append(id[38], 1000.005);
    rb.append(id[39], 3.2767f);
    rb.append(id[40], 3.276699f);
    rb.append(id[41], -3.2767f);
    rb.append(id[42], -3.276699f);
    rb.append(id[43], 10.005f);
    rb.append(id[44], 100.005f);
    rb.append(id[45], 1000.005f);
    std::string const rbAsString = rb.exportAsString();
    std::cout<<(std::string)(TclList()<<"rbAsString"<<rbAsString)<<std::endl;

    auto record = Record::create(rbAsString);
    if (!record)
    {
      std::cout<<"Unable to read record!"<<std::endl;
    }
    else
    {
      std::cout<<record->debugDump()<<std::endl;
      for (int i = 0; i < 45; i++)
      {
        TclList line;
        ValueBox value = record->lookUpValue(id[i]);
        line<<i<<id[i]<<value.shortDebug();
        std::cout<<(std::string)line<<std::endl;
      }
    }
    if (record->update(id[21],9999))
      std::cout<<"That shouldn't have worked!  21"<<std::endl;
    if (record->update(id[24],9999))
      std::cout<<"That shouldn't have worked!  24"<<std::endl;
    if (!record->update(id[23], 9999))
      std::cout<<"Unexpected error!  23"<<std::endl;
    Record::Ref thisIsThreadSafe = record;
    for (int i = 0; i < 25; i++)
    {
      TclList line;
      ValueBox value = thisIsThreadSafe->lookUpValue(id[i]);
      line<<i<<id[i]<<value.shortDebug();
      std::cout<<(std::string)line<<std::endl;
    }
    std::cout<<"====================="<<std::endl;
  }

  static void reportException(Exception const &ex)
  {
    std::cout<<"Error:  "<<ex.getMessage()<<std::endl
             <<ex.simpleLocationMessage()<<std::endl;
  }

  static Record::Ref getFakeAlert()
  { // Just enough to test the basic execution abilities.  We don't need to test
    // every field, just the ability to read fields.
    RecordBuilder rb;
    rb.append(MainFields::alert_type, "NHP");
    rb.append(MainFields::description, "New High!");
    rb.append(MainFields::timestamp, time(NULL));
    rb.append(MainFields::price, 4.20);
    rb.append(MainFields::last, 4.19);
    rb.append(MainFields::bid_size, 400);
    rb.append(MainFields::ask_size, 1400);
    rb.append(MainFields::relvol, 12.345);
    rb.append(MainFields::num, 57);
    rb.append(MainFields::tvol, 1234567);
    rb.append(MainFields::t_high, 5.0);
    rb.append(MainFields::t_low, 4.0);
    rb.append(MainFields::symbol, "IBM");
    rb.append(MainFields::timestamp, time(NULL));
    rb.append(MainFields::alt_description,
              "e=Pink%20Sheets&t=above&p=10.47&mdy=4/24/2014&u=0.01");
    return Record::create(rb.exportAsString());
  }

  static const std::string s_GOOD = "GOOD";

  static bool isGood(ValueBox const &result)
  {
    bool valid;
    std::string value;
    result.getString(valid, value);
    return valid && (value == s_GOOD);
  }

  static std::string goodItems[] =
    {
      "if(3+4*5==23,\"GOOD\")",
      "if(3+4*5==21,null,\"GOOD\")",
      "if(0.4+1==1.4,\"GOOD\")",
      //"if(0.4*3==1.2,\"GOOD\")",
      "if(abs(0.4*3-1.2)<0.00000000001,\"GOOD\")",
      "if(4.0*3.0==12,\"GOOD\")",
      "if(4.0==4,\"GOOD\")",
      "if(4==4.0,\"GOOD\")",
      "if(4==4,\"GOOD\")",
      "if(4.0==4.0,\"GOOD\")",
      "if(4.0<5,\"GOOD\")",
      "if(4<5.0,\"GOOD\")",
      "if(4<5,\"GOOD\")",
      "if(4.0<5.0,\"GOOD\")",
      "if(4.0>5,null,\"GOOD\")",
      "if(4>5.0,null,\"GOOD\")",
      "if(4>5,null,\"GOOD\")",
      "if(4.0>5.0,null,\"GOOD\")",
      "null?\?(1/0)?\?(1/0.0)??\"GOOD\"",  // Beware of trigraphs!
      "if(1==\"1\",null,\"GOOD\")",
      "if(\"1\"==1,null,\"GOOD\")",
      "if(1.0==\"1\",null,\"GOOD\")",
      "if(\"1\"==1.0,null,\"GOOD\")",
      "(1==\"1\")??\"GOOD\"",
      "(\"1\"==1)??\"GOOD\"",
      "(1.0==\"1\")??\"GOOD\"",
      "(\"1\"==1.0)??\"GOOD\"",
      "if(sin(0) == 0,\"GOOD\")",
      "if(sin(atan(1)*2) == 1,\"GOOD\")",
      "if(abs(sin(atan(1)*4)) < 0.00000000000001,\"GOOD\")",
      "if(sin(atan(1)*6) == -1,\"GOOD\")",
      "if(cos(0) == 1,\"GOOD\")",
      "if(abs(cos(atan(1)*2)) < 0.00000000000001,\"GOOD\")",
      "if(cos(atan(1)*4) == -1,\"GOOD\")",
      "if(abs(cos(atan(1)*6)) < 0.00000000000001,\"GOOD\")",
      "if(tan(atan(3))==3,\"GOOD\")",
      "if(cos(acos(0.25))==0.25,\"GOOD\")",
      "if(sin(asin(0.5))==0.5,\"GOOD\")",
      "asin(3)??\"GOOD\"",
      "acos(4)??\"GOOD\"",
      "if((ceil(4.5)==5)&&(floor(4.5)==4),\"GOOD\")",
      "if((ceil(-4.5)==-4)&&(floor(-4.5)==-5),\"GOOD\")",
      "if((ceil(3)==3)&&(floor(3)==3),\"GOOD\")",
      "if(log(4,32)==2.5,\"GOOD\")",
      "log(4,-4)??\"GOOD\"",
      "log(4,0)??\"GOOD\"",
      "log(0,4)??\"GOOD\"",
      "if(exp(ln(3)+ln(4))==12,\"GOOD\")",
      "if([Price]==4.20,\"GOOD\")",
      // The test data is currently broken.
      //"if([Vol]==1000000,\"GOOD\")",
      //"if([RPD]==-91.6,\"GOOD\")",
      "if([Price]&&1==4.20,\"BAD\",\"GOOD\")",
      "if([Price]&&1==1,\"GOOD\")",
      //"if([Price]&&[Vol]&&[Price]&&[Vol],\"GOOD\")",
      "if(![Price],\"BAD\",\"GOOD\")",
      "if([Price]&&0,\"BAD\",\"GOOD\")",
      "if(([Price]&&[Price])==[Price],\"BAD\",\"GOOD\")",
      "if(([Price]&&1)==[Price],\"BAD\",\"GOOD\")",
      "if(([Price]&&1)==1,\"GOOD\")",
      "if(([Price]&&[Price])==1,\"GOOD\")",
      "if(([Price]||1)==1,\"GOOD\")",
      "if(([Price]||[Price])==1,\"GOOD\")",
      "[=x]??\"GOOD\"",  // No such field.
      "[=e]??\"GOOD\"", // Not a valid number.
      "[=mdy]??\"GOOD\"", // Not a valid number, but starts with a valid number.
      "if([=u]==0.01,\"GOOD\")",
      "if([=u]+[=p]==10.48,\"GOOD\",[=u]+[=p])",  // Caching the alt_description field.
      "if(abs(5.25)==5.25,\"GOOD\")",
      "if(abs(-5.25)==5.25,\"GOOD\")",
      "if(abs(5)==5,\"GOOD\")",
      "if(abs(-5)==5,\"GOOD\")",
      ""
    };

  static Tree standardParse(std::string const &input)
  {
    Parser parser(input);
    Tree tree = parser.parse();
    tree = replace(tree, *CommonRules::instance().publicResources());
    tree = replace(tree, AlertRules());
    tree = AppropriatePrice::useCurrentPrice(tree);
    tree->assertDone();
    return tree;
  }

  static Tree constructAll(Tree original)
  {
    TreeNode::Args const &originalArgs = original->getChildren();
    TreeNode::Args args;
    args.reserve(originalArgs.size());
    for (TreeNode::Args::const_iterator it = originalArgs.begin();
         it != originalArgs.end();
         it++)
      args.push_back(constructAll(*it));
    Tree result = original->construct(args);
    if (!result)
    {
      if (args.empty())
        // This would never be called in real life, so skip it.
        result = original;
      else
        throw original->getToken().makeException("construct() returned null.");
    }
    return result;
  }

  // testRecord points into save.  save keeps the smart pointer alive.
  // RecordInfo only contains a normal pointer, not a smart pointer.  If these
  // two variables get out of sync, testRecord will point to random memory.
  // (Most of the time we are already keeping a copy of the record, so you don't
  // have to jump through these hoops.  This test program is different.)
  static Record::Ref save;
  static Execution::RecordInfo testRecord;

  static ValueBox execute(Tree tree)
  {

    ValueBox result;
    Execution::Executable *executable = NULL;
    try
    {
      executable = Execution::Executable::create(tree);
      result = executable->execute(testRecord);
    }
    catch (...)
    {
      delete executable;
      throw;
    }
    delete executable;
    return result;
  }

  static void optimizationTest(Tree tree)
  {
    try
    {
      const ValueBox original = execute(tree);
      Tree optimizedTree = TreeNode::optimize(tree);
      const ValueBox optimized = execute(optimizedTree);
      Tree forBooleanTree =
        TreeNode::optimize(tree, Parse::OptimizationContext::FOR_BOOLEAN);
      const ValueBox forBoolean = execute(forBooleanTree);
      if (original != optimized)
      {
        std::cerr<<"Problem with optimize."<<std::endl
                 <<original.shortDebug()<<"  <--  "
                 <<tree->shortDebug()<<std::endl
                 <<optimized.shortDebug()<<"  <--  "
                 <<optimizedTree->shortDebug()<<std::endl;
      }
      if (original.getBoolean() != forBoolean.getBoolean())
      {
        std::cerr<<"Problem with optimize for boolean."<<std::endl
                 <<original.getBoolean()<<"  <--  "
                 <<tree->shortDebug()<<std::endl
                 <<forBoolean.getBoolean()<<"  <--  "
                 <<forBooleanTree->shortDebug()<<std::endl;
      }
    }
    catch (Exception &ex)
    {
      std::cerr<<"Unexpected error in optimizationTest("<<tree->shortDebug()
               <<")"<<std::endl
               <<ex.getMessage()<<std::endl
               <<ex.simpleLocationMessage()<<std::endl;
    }
    for (TreeNode::Args::const_iterator it = tree->getChildren().begin();
         it != tree->getChildren().end();
         it++)
    {
      optimizationTest(*it);
    }
  }

  // Use real formulae that people have created.  This is similar to what we
  // do with our own test cases, with a few small exceptions.  E.g. Our test
  // cases were designed so no two of them would generate the same parse tree,
  // but we don't have the level of control here.
  static void parseUserFormulae()
  { // These were never real formulas.  Sometimes I wrote the SQL code by hand,
    // and I put notes into the source field.
    std::set< std::string > ignore =
      { "", "---", "based on time, not the smile curve",
        "based on the smile curve", "Pivot + (R1 - S1)", "2 * Pivot - Low",
        "( High + Close + Low )/3", "2 * Pivot - High", "Pivot - (R1 - S1)",
        "1 is max", "1 is normal, 1.5 is max", "1 is normal, infinity is max",
        "1 is normal, very large number is max", "Magic [DUp30]/[YSD]"
      };
    DatabaseWithRetry database(true, "ParserTest.C");
    const std::string sql =
      "SELECT DISTINCT source FROM user_filters WHERE sql_code <> 'null'";
    for (MysqlResultRef testCase = database.tryQueryUntilSuccess(sql);
         testCase->rowIsValid();
         testCase->nextRow())
    {
      const std::string source = testCase->getStringField(0);
      if (ignore.count(source))
        continue;
      Execution::Executable *executable = NULL;
      try
      {
        Tree tree = standardParse(source);
        if (tree->compare(*standardParse(source + " ")))
        {
          std::cerr<<"“"<<source<<"” is not equal to itself!"<<std::endl;
        }
        executable = Execution::Executable::create(tree);
        if (!executable)
        { // This is usually an assertion, not an exception!
          std::cerr<<"“"<<source<<"” compiles to NULL!"<<std::endl;
        }
        Tree reconstructed = constructAll(tree);
        if (tree->compare(*reconstructed))
          std::cerr<<"Unable to reconstruct “"<<source<<"”!"<<std::endl;
        optimizationTest(tree);
      }
      catch (Exception &ex)
      {
        std::cerr<<"Problem compiling “"<<source<<"”!"<<std::endl
                 <<ex.getMessage()<<std::endl
                 <<ex.simpleLocationMessage()<<std::endl;
      }
      delete executable;
    }
  }

  static void initializeTestRecord()
  {
    save = getFakeAlert();
    testRecord.setRecord(save);
  }

  static void timingTest(std::vector< std::string > const &formulae,
                         int repeatCount = 1)
  {
    initializeTestRecord();
    StopWatch stopWatch;
    int64_t parseTime = 0;
    int64_t compareSelfTime = 0;
    int64_t compileTime = 0;
    int64_t reconstructionTime = 0;
    int64_t reconstructionTestTime = 0;
    int64_t optimizationTestTime = 0;
    for (int i = 0; i < repeatCount; i++)
      for (std::string const &source : formulae)
      {
        Execution::Executable *executable = NULL;
        try
        {
          Tree tree = standardParse(source);
          parseTime += stopWatch.getMicroSeconds();
          if (tree->compare(*standardParse(source + " ")))
          {
            std::cerr<<"“"<<source<<"” is not equal to itself!"<<std::endl;
          }
          compareSelfTime += stopWatch.getMicroSeconds();
          executable = Execution::Executable::create(tree);
          if (!executable)
          { // This is usually an assertion, not an exception!
            std::cerr<<"“"<<source<<"” compiles to NULL!"<<std::endl;
          }
          compileTime += stopWatch.getMicroSeconds();
          Tree reconstructed = constructAll(tree);
          reconstructionTime += stopWatch.getMicroSeconds();
          if (tree->compare(*reconstructed))
            std::cerr<<"Unable to reconstruct “"<<source<<"”!"<<std::endl;
          reconstructionTestTime += stopWatch.getMicroSeconds();
          optimizationTest(tree);
          optimizationTestTime += stopWatch.getMicroSeconds();
        }
        catch (Exception &ex)
        {
          std::cerr<<"Problem compiling “"<<source<<"”!"<<std::endl
                   <<ex.getMessage()<<std::endl
                   <<ex.simpleLocationMessage()<<std::endl;
        }
        delete executable;
      }
    std::cout<<"========= Parser timing test ========="<<std::endl
             <<repeatCount<<" iterations of "<<formulae.size()
             <<" formulae."<<std::endl
             <<"parseTime:  "<<(parseTime / 1.0e+6)<<std::endl
             <<"compareSelfTime:  "<<(compareSelfTime / 1.0e+6)<<std::endl
             <<"compileTime:  "<<(compileTime / 1.0e+6)<<std::endl
             <<"reconstructionTime:  "
             <<(reconstructionTime / 1.0e+6)<<std::endl
             <<"reconstructionTestTime:  "
             <<(reconstructionTestTime / 1.0e+6)<<std::endl
             <<"optimizationTestTime:  "
             <<(optimizationTestTime / 1.0e+6)<<std::endl;
    // Presumably we will be running the profiler.  That only works if you
    // exit properly.  Most of the time (like if you don't specify any master
    // for alerts or top lists) we exit by calling _exit().  We explicitly want
    // to skip C++'s normal cleanup because there's no orderly way to shut down
    // all of the threads.  But we need to call the normal exit() to do a
    // normal shutdown for the profiler won't work.  That means you need to
    // call this function pretty early in the startup process, before we
    // create any objects we don't know how to destroy.
    exit(1);
  }

  class TreeCompareAndTest
  {
  private:
    // This has to be static because operator() has to be const.
    static bool _errorFound;
  public:
    int operator()(Tree const &a, Tree const &b) const
    {
      int result = a->compare(*b);
      int altResult = b->compare(*a);
      bool success = ((result == 0) && (altResult == 0))
        || ((result < 0) && (altResult > 0))
        || ((result > 0) && (altResult < 0));
      if (!success)
      {
        std::cerr<<"Inconsistent results!!"<<std::endl
                 <<"a = "<<a->shortDebug()<<std::endl
                 <<"b = "<<b->shortDebug()<<std::endl
                 <<"a->compare(*b) = "<<result<<std::endl
                 <<"b->compare(*a) = "<<altResult<<std::endl;
        std::cerr.flush();
	_errorFound = true;
      }
      return result < 0;  // True if and only if a < b.
    }

    // Originally we aborted as soon as we found a problem.  But that doesn't
    // work well.  That would give me a big item that failed, like an entire
    // where condition.  I want to keep diving in until I find the smallest
    // thing that failed. 
    //
    // If this code was running in production, aborting immediately would
    // probably make sense.
    static void assertAllGood() { if (_errorFound) abort(); }
  };

  bool TreeCompareAndTest::_errorFound = false;

  typedef std::set< Tree, TreeCompareAndTest > SafeTreeSet;

  static void addAll(SafeTreeSet &destination, int &totalCount,
                     Tree const &toAdd)
  {
    destination.insert(toAdd);
    totalCount++;
    for (Tree const &child : toAdd->getChildren())
      addAll(destination, totalCount, child);
  }

  static void addAll(SafeTreeSet &destination, int &totalCount,
		     StrategyTrees strategyTrees)
  { // The alerts will automatically call useCurrentPrice() and half optimize
    // the trees before it gives the trees to us.  The following would cause
    // an assertion to fail.
    // strategyTrees.useCurrentPrice();
    // Add all parts of the strategy.  We're looking for a bug in the
    // optimizer.  In particular, the step that combines the various elements
    // of a strategy, and possibly multiple strategies, and looks for
    // duplicates.
    addAll(destination, totalCount, strategyTrees.getWhere());
  }

  // See the window name in each collaborate string to see what makes it
  // interesting or different.
  //
  // Note that Strategy.[Ch] uses Tree types that are not available in the
  // expression language.  (They could be, but we haven't seen the need yet.)
  // In particular the symbol lists and the exchanges in a strategy cannot
  // be simulated in the expression language.
  //
  // These were the key to finding some mysterious core dumps.  Objects of
  // type SymbolListsWhere in Strategy.C didn't always compare correctly.
  // That file was trying to use > on smart pointers.  <, == and != were
  // correctly defined on smart pointers, but not >, <=. or >=.  If you tried
  // an operation from the second group there was no compiler warning, but
  // you didn't get the expected result.  This was fixed in revision 1.21 of
  // ParserTest.C.
  static std::vector< std::string > alertCollaborateStrings =
    {
      "O=400C000000000CC00030_419_0&QNHPF=30&QNLPF=30&QGDTOP=4&QGDBOT=4&QSV=2&MaxPrice=1000&MinPrice=5&MinRV=1.5&MinVol3M=500000&WN=Market+Watcher&SL=X1o5&show0=D_Symbol&show1=D_Type&show2=D_Time&show3=D_Desc&show4=Price&show5=RV&show6=FCP&show7=RD&col_ver=1",
      "form=1&Sh_NHPF=on&QNHPF=30&Sh_NLPF=on&QNLPF=30&Sh_SV=on&QSV=2&Sh_CA200=on&Sh_CB200=on&Sh_CA20=on&Sh_CB20=on&Sh_GDBOT=on&QGDBOT=4&Sh_GDTOP=on&QGDTOP=4&MinPrice=5&MinRV=1.5&MinVol3M=500000&MaxPrice=1000&X_NYSE=on&X_ARCA=on&X_AMEX=on&XN=on&EntireUniverse=2&SL_0_5=on&SL_0_27=on&SL_1_5=on&WN=Market+Watcher+--+exclude+3+lists&show0=D_Time&show1=D_Type&show2=D_Symbol&show3=D_Desc&show4=Price&show5=FCP&show6=RV&show7=RD&show8=&col_ver=1",
      "form=1&Sh_NHPF=on&QNHPF=30&Sh_NLPF=on&QNLPF=30&Sh_SV=on&QSV=2&Sh_CA200=on&Sh_CB200=on&Sh_CA20=on&Sh_CB20=on&Sh_GDBOT=on&QGDBOT=4&Sh_GDTOP=on&QGDTOP=4&MinPrice=5&MinRV=1.5&MinVol3M=500000&MaxPrice=1000&X_NYSE=on&X_ARCA=on&X_AMEX=on&XN=on&SL_0_5=on&SL_0_27=on&SL_1_5=on&WN=Market+Watcher+--+only+3+lists&show0=D_Time&show1=D_Type&show2=D_Symbol&show3=D_Desc&show4=Price&show5=FCP&show6=RV&show7=RD&show8=&col_ver=1",
      "form=1&Sh_NHPF=on&QNHPF=30&Sh_NLPF=on&QNLPF=30&Sh_SV=on&QSV=2&Sh_CA200=on&Sh_CB200=on&Sh_CA20=on&Sh_CB20=on&Sh_GDBOT=on&QGDBOT=4&Sh_GDTOP=on&QGDTOP=4&MinPrice=5&MinRV=1.5&MinVol3M=500000&MaxPrice=1000&X_NYSE=on&X_ARCA=on&X_AMEX=on&XN=on&WN=Market+Watcher+--+all+symbols&show0=D_Time&show1=D_Type&show2=D_Symbol&show3=D_Desc&show4=Price&show5=FCP&show6=RV&show7=RD&show8=&col_ver=1",
      "form=1&Sh_NHPF=on&QNHPF=30&Sh_NLPF=on&QNLPF=30&Sh_SV=on&QSV=2&Sh_CA200=on&Sh_CB200=on&Sh_CA20=on&Sh_CB20=on&Sh_GDBOT=on&QGDBOT=4&Sh_GDTOP=on&QGDTOP=4&MinPrice=5&MinRV=1.5&MinVol3M=500000&MaxPrice=1000&X_NYSE=on&X_ARCA=on&X_AMEX=on&XN=on&EntireUniverse=3&SingleSymbol=MSFT&WN=Market+Watcher+--+Only+MSFT&show0=D_Time&show1=D_Type&show2=D_Symbol&show3=D_Desc&show4=Price&show5=FCP&show6=RV&show7=RD&show8=&col_ver=1",
      "form=1&Sh_NHPF=on&QNHPF=30&Sh_NLPF=on&QNLPF=30&Sh_SV=on&QSV=2&Sh_CA200=on&Sh_CB200=on&Sh_CA20=on&Sh_CB20=on&Sh_GDBOT=on&QGDBOT=4&Sh_GDTOP=on&QGDTOP=4&MinPrice=5&MinRV=1.5&MinVol3M=500000&MaxPrice=1000&X_NYSE=on&X_ARCA=on&X_AMEX=on&XN=on&X_OTC=on&X_PINK=on&X_CAT=on&X_CAV=on&XX=on&EntireUniverse=2&SL_1_5=on&WN=Market+Watcher+--+all+exchanges&show0=D_Time&show1=D_Type&show2=D_Symbol&show3=D_Desc&show4=Price&show5=FCP&show6=RV&show7=RD&show8=&col_ver=1",
      "form=1&Sh_NHPF=on&QNHPF=30&Sh_NLPF=on&QNLPF=30&Sh_SV=on&QSV=2&Sh_CA200=on&Sh_CB200=on&Sh_CA20=on&Sh_CB20=on&Sh_GDBOT=on&QGDBOT=4&Sh_GDTOP=on&QGDTOP=4&MinPrice=5&MinRV=1.5&MinVol3M=500000&MaxPrice=1000&X_NYSE=on&EntireUniverse=2&SL_1_5=on&WN=Market+Watcher+--+only+NYSE&show0=D_Time&show1=D_Type&show2=D_Symbol&show3=D_Desc&show4=Price&show5=FCP&show6=RV&show7=RD&show8=&col_ver=1",
    };

  static void autoTestDetailedCompare()
  {
    SafeTreeSet allPieces;
    int totalCount = 0;
    int topLevelCount = 0;
    int optimizedCount = 0;
    for (std::string const &source : goodItems)
      if (!source.empty())
      {
        topLevelCount++;
        Tree unoptimized = standardParse(source);
        addAll(allPieces, totalCount, unoptimized);
        if (Tree optimized = unoptimized->optimize())
        {
          optimizedCount++;
          addAll(allPieces, totalCount, optimized);
        }
      }
    DatabaseWithRetry &database = *DatabaseForThread(DatabaseWithRetry::SLAVE);
    if (!database.probablyWorks())
      std::cout<<"Skipping alert collaborate strings.  No database."
	       <<std::endl;
    else
    {
      for (std::string const &source : alertCollaborateStrings)
      {
	AlertStrategy alertStrategy;
	const UserId userId = 4;  // The philip account.
	// Compile now implies full optimize now.  But the whole point of this
	// exercise is to simulate the code that's occasionally dumping core.
	// That code takes a lot of collaborate strings at once and compiles
	// and optimizes the entire set together.
	const bool compileNow = false;  
	alertStrategy.load(source, userId, database, compileNow);
	addAll(allPieces, totalCount, alertStrategy.debugGetStrategyTrees());
      }
      std::cout<<"Parsed "<<alertCollaborateStrings.size()
	       <<" alert collaborate strings."<<std::endl;
    }
    std::cout<<"autoTestDetailedCompare(), "
             <<totalCount<<" expressions from "
             <<topLevelCount<<" strings, "<<optimizedCount
             <<" of which could be optimized"<<std::endl;
    for (Tree const &leftTree : allPieces)
      for (Tree const &rightTree : allPieces)
      {
        if (leftTree == rightTree)
        {
          const int result = leftTree->compare(*rightTree);
          if (result)
            std::cerr<<"“"<<leftTree->shortDebug()<<"” is not equal to itself!"
                     <<std::endl;
        }
        else
        {
          const auto h1 = leftTree->hash_code();
          const auto h2 = rightTree->hash_code();
          if (h1 == h2)
            // This is not an error.  But it should be rare.
            std::cerr<<"“"<<leftTree->shortDebug()<<"” and “"
                     <<rightTree->shortDebug()
                     <<"” both have the same hash code:  "<<h1
                     <<std::endl;
          const int result1 = leftTree->compare(*rightTree);
          const int result2 = rightTree->compare(*leftTree);
          if ((result1 == 0) || (result2 == 0))
            std::cerr<<"“"<<leftTree->shortDebug()<<"” is equal to “"
                     <<rightTree->shortDebug()<<"”!"
                     <<std::endl;
          else if ((result1 < 0) == (result2 < 0))
            std::cerr<<"Inconsistent results comparing “"<<leftTree->shortDebug()
                     <<"” to “"<<rightTree->shortDebug()<<"”!"<<std::endl;
        }
      }

    enum Count { count0, count1, countMore };
    typedef std::map< Tree, Count, TreeCompareAndTest > Counts;
    typedef std::map< Tree, Tree, TreeCompareAndTest > Replacements;
    class Internal
    {
    public:
      static void countDuplicates(Tree tree, Counts &counts)
      {
        if (tree->easyToCompute())
          return;
        Count &count = counts[tree];
        if (count == count0)
        {
          count = count1;
          for (TreeNode::Args::const_iterator it = tree->getChildren().begin();
               it != tree->getChildren().end();
               it++)
            countDuplicates(*it, counts);
        }
        else
          count = countMore;
      }
      static Tree replaceTopDown(Tree original,
                                 Replacements const &replacements)
      {
        Replacements::const_iterator it = replacements.find(original);
        if (it != replacements.end())
          // Found an exact match for the entire tree.
          return it->second;
        bool somethingChanged = false;
        TreeNode::Args children = original->getChildren();
        for (TreeNode::Args::iterator it = children.begin();
             it != children.end();
             it++)
        {
          Tree newValue = replaceTopDown(*it, replacements);
          if (newValue->compare(**it))
          {
            *it = newValue;
            somethingChanged = true;
          }
        }
        if (somethingChanged)
          return original->construct(children);
        else
          return original;
      }
    };

    Counts counts;
    for (Tree const &tree : allPieces)
      Internal::countDuplicates(tree, counts);
    typedef std::set< std::pair< int, Tree > > SortedByHeight;
    SortedByHeight sortedByHeight;
    for (auto kvp : counts)
    {
      const Count count = kvp.second;
      assert(count != count0);
      if (count == countMore)
        sortedByHeight.insert(std::pair< int, Tree >(kvp.first->height(),
                                                     kvp.first));
    }
    Replacements replacements;
    for (auto kvp : sortedByHeight)
    {
      const Tree value = kvp.second;
      const Tree replacement =
        new CachedValue(replacements.size(),
                        Internal::replaceTopDown(value, replacements));
      replacements[value] = replacement;
    }

    int changed = 0;
    int unchanged = 0;
    for (Tree tree : allPieces)
    {
      Tree withReplacements = Internal::replaceTopDown(tree, replacements);
      if (tree->compare(*withReplacements))
        changed++;
      else
        unchanged++;
    }
    std::cout<<"Finished Strategy.[Ch] style optimization.  "
             <<changed<<" trees modified, "<<unchanged
             <<" left as is."<<std::endl;

    std::cout<<"autoTestDetailedCompare() is done"<<std::endl;

    TreeCompareAndTest::assertAllGood();
  }

  // The parts that don't require the command line.
  static void autoTestParser()
  {
    autoTestDetailedCompare();
    initializeTestRecord();
    StopWatch stopWatch;
    if (0)
      // This can be slow sometimes.  Maybe we need a command line setting
      // to turn this on or off.
      parseUserFormulae();
    std::cout<<"parseUserFormulae() finished after "
             <<(stopWatch.getMicroSeconds() / 1.0e+6)<<" seconds."<<std::endl;
    for (std::string const *rightPtr = goodItems; !rightPtr->empty(); rightPtr++)
      for (std::string const *leftPtr = goodItems; !leftPtr->empty(); leftPtr++)
        try
        {
          Tree leftTree = standardParse(*leftPtr);
          Tree rightTree = standardParse(" " + *rightPtr);
          if (leftPtr == rightPtr)
          {
            const int result = leftTree->compare(*rightTree);
            if (result)
              std::cerr<<"“"<<*leftPtr<<"” is not equal to itself!"<<std::endl;
            const auto h1 = leftTree->hash_code();
            const auto h2 = rightTree->hash_code();
            if (h1 != h2)
              std::cerr<<"Hash functions don't match for “"<<*leftPtr<<"”."
                       <<std::endl;
          }
          else
          {
            const int result1 = leftTree->compare(*rightTree);
            const int result2 = rightTree->compare(*leftTree);
            if ((result1 == 0) || (result2 == 0))
              std::cerr<<"“"<<*leftPtr<<"” is equal to “"<<*rightPtr<<"”!"
                       <<std::endl;
            else if ((result1 < 0) == (result2 < 0))
              std::cerr<<"Inconsistent results comparing “"<<*leftPtr<<"” to “"
                       <<*rightPtr<<"”!"<<std::endl;
          }
        }
        catch (Exception &ex)
        {
          std::cerr<<"Problem comparing “"<<*leftPtr<<"” to “"<<*rightPtr<<"”."
                   <<std::endl
                   <<ex.getMessage()<<std::endl
                   <<ex.simpleLocationMessage()<<std::endl;
        }
    for (std::string const *current = goodItems; !current->empty(); current++)
    {
      try
      {
        Tree tree = standardParse(*current);
        ValueBox result = execute(tree);
        if (!isGood(result))
          std::cerr<<"Invalid result from “"<<*current<<"”:  "
                   <<result.shortDebug()<<std::endl;
        Tree reconstructed = constructAll(tree);
        if (tree->compare(*reconstructed))
          std::cerr<<"Unable to reconstruct “"<<*current<<"”!"<<std::endl;
        optimizationTest(tree);
      }
      catch (Exception &ex)
      {
        std::cerr<<"Problem compiling “"<<*current<<"”!"<<std::endl
                 <<ex.getMessage()<<std::endl
                 <<ex.simpleLocationMessage()<<std::endl;
      }
    }

  }

  static void testParser()
  {
    IAlertsDaily::set(new FakeAlertsDaily());
    std::cout<<"Starting auto test"<<std::endl;
    autoTestParser();
    Record::Ref fakeAlert = getFakeAlert();
    Execution::RecordInfo recordInfo;
    recordInfo.setRecord(fakeAlert);
    std::cout<<"Starting interactive test"<<std::endl;
    UserId userId = 4;
    UserFiltersRule *userFilters = new UserFiltersRule(userId);
    while (true)
    {
      std::string line;
      std::getline(std::cin, line);
      if (!std::cin)
        break;
      auto pieces = explode("!userId=", line);
      if ((pieces.size() == 2) && (pieces[0].empty()))
      {
        std::cout<<"Changing userId from "<<userId<<" to ";
        userId = strtolDefault(pieces[1], 0);
        std::cout<<userId<<std::endl;
        delete userFilters;
        userFilters = new UserFiltersRule(userId);
      }
      else
      {
        Parser parser(line);
        try
        {
          std::cout<<parser.getTokenListDebug()<<std::endl;
          Tree parseTree = parser.parse();
          std::cout<<"parser.parse():"<<std::endl;
          std::cout<<parseTree->shortDebug()<<std::endl;
          parseTree =
            replace(parseTree, *CommonRules::instance().publicResources());
          std::cout<<"replace(parseTree, ...publicResources()):"<<std::endl;
          std::cout<<parseTree->shortDebug()<<std::endl;
          parseTree = replace(parseTree, *userFilters);
          std::cout<<"replace(parseTree, UserFilters("<<ntoa(userId)<<"))"
                   <<std::endl
                   <<parseTree->shortDebug()<<std::endl;
          parseTree = replace(parseTree, AlertRules());
          std::cout<<"replace(parseTree, AlertRules()):"<<std::endl;
          std::cout<<parseTree->shortDebug()<<std::endl;
          parseTree = AppropriatePrice::useCurrentPrice(parseTree);
          std::cout<<"AppropriatePrice::useCurrentPrice(parseTree):"<<std::endl;
          std::cout<<parseTree->shortDebug()<<std::endl;
          parseTree = TreeNode::optimize(parseTree);
          std::cout<<"TreeNode::optimize(parseTree):"<<std::endl;
          std::cout<<parseTree->shortDebug()<<std::endl;
          parseTree->assertDone();
          Execution::Executable *executable =
            Execution::Executable::create(parseTree);
          std::cout<<"Successful compile."<<std::endl;
          ValueBox result = executable->execute(recordInfo);
          std::cout<<"Result:  "<<result.shortDebug()<<std::endl;
          delete executable;
        }
        catch (Parse::Exception const &ex)
        {
          reportException(ex);
        }
      }
    }
    delete userFilters;
  }

}

void optionallyTestParser()
{
  if (getConfigItem("test_data_format") == "1")
    ParserTest::testDataFormat();
  if (getConfigItem("test_parser") == "1")
    ParserTest::testParser();
  int count = strtolDefault(getConfigItem("parser_timing_count"), 0);
  if (count > 0)
  { // How to read a whole file into a string:
    // http://stackoverflow.com/questions/2602013/read-whole-ascii-file-into-c-stdstring#2602258
    std::ifstream file("parser_timing_test.txt");
    std::stringstream buffer;
    buffer << file.rdbuf();
    std::string entireFile = buffer.str();
    std::vector< std::string > formulae = explode("@@@@@@@@@@", entireFile);
    ParserTest::timingTest(formulae, count);
  }
}