using System;
using System.Collections.Generic;
using System.Text;
using System.Windows.Forms;
using TradeIdeas.Extensions;
using TradeIdeas.MiscSupport;
using TradeIdeas.ServerConnection;
using TradeIdeas.TIProData;
using TradeIdeas.TIProData.Interfaces;
using TradeIdeas.TIProGUI;
using TradeIdeas.XML;
namespace TIProDevExtension
{
///
/// This is a special test rig I built to compare the recent changes to the chart server
/// to the older chart server that's been in production for a while.
///
/// I'm pointing to a micro_proxy with two special rules. candles_command_test maps to
/// flex_command on my test server. candles_command_production maps to flex_command on
/// the same group of servers that handle candles_command for production. I don't use
/// candles_command directly because I don't know (or care) if that is pointing to test
/// or production.
///
/// I'm sending a list of requests to both servers and comparing the results. Ideally
/// I'd hit a lot of special cases. Like a 15 minute candle with half of the data in
/// the local cache and the other half in the database. But that's hard to do precisely,
/// so I'm mostly relying on brute force. I'm asking for a lot of different variations.
///
public partial class CompareChartData : Form
{
private IConnectionMaster _connectionMaster;
private enum ServerType { Test, Production }
private static String GetCommand(ServerType serverType)
{
if (serverType == ServerType.Test)
return "candles_command_test";
else
return "candles_command_production";
}
private static String GetUserFriendly(ServerType serverType)
{
if (serverType == ServerType.Test)
return "Test";
else
return "Production";
}
///
/// This is basically an interface required to use OneRequest.
///
/// Originally OneRequest was only used by OneTest. They were closely intertwined. Since then
/// OneTest has been renamed to OldNewTest. And We've added the new WeekMonthTest class which
/// also uses OneRequest. Both OldNewTest and WeekMonthTest inherit from TestBase.
///
private abstract class TestBase
{
public readonly CompareChartData Form;
public TestBase(CompareChartData form)
{
Form = form;
}
public string Symbol { get; protected set; }
private int _totalRetries = 0;
private ulong _maxRetry = 0;
public void ReportRetry(ulong epoch, ServerType serverType)
{
_totalRetries++;
_maxRetry = Math.Max(_maxRetry, epoch);
Form.retriesLabel.Text = "Total retries: " + _totalRetries + ", Max retry: " + _maxRetry
+ ", Last: " + GetUserFriendly(serverType);
}
public abstract void ReportResult(string data, ServerType serverType);
protected void ShowText(string text)
{
Form.ShowText(text);
}
abstract public void ReportDebug(string message, ServerType serverType);
abstract public void ReportFatalError(string message, ServerType serverType);
}
private class OneRequest
{
private readonly TestBase _owner;
private readonly ServerType _serverType;
private readonly string _prototype;
private UInt64 _epoch = 0;
private TalkWithServer.CancelToken _cancelToken;
public OneRequest(TestBase owner, ServerType serverType, string prototype)
{
_owner = owner;
_serverType = serverType;
_prototype = prototype;
SendRequestNow();
}
private void CancelServerRequest()
{
_epoch++;
if (null != _cancelToken)
{
_cancelToken.Cancel();
_cancelToken = null;
}
}
private void SendRequestNow()
{
CancelServerRequest();
object[] message = new object[]
{
"command", GetCommand(_serverType),
"subcommand", "make_and_show",
"prototype", _prototype,
"symbols", _owner.Symbol
};
_cancelToken = new TalkWithServer.CancelToken();
_owner.Form._connectionMaster.SendManager.SendMessage
(TalkWithServer.CreateMessage(message), OnResponse, streaming: true, clientId: _epoch,
cancelToken: _cancelToken);
}
public void OnResponse(byte[] body, object clientId)
{
_owner.Form.BeginInvokeIfRequired(delegate ()
{
UInt64 originalEpoch = (UInt64)clientId;
if (originalEpoch != _epoch)
// A response to a previous request.
return;
if (body == null)
{
_owner.ReportRetry(_epoch, _serverType);
SendRequestNow();
}
else
{
var document = XmlHelper.Get(body);
if (document == null)
ReportFatalError("Couldn't parse XML");
else
{
var top = document.DocumentElement;
string type = top.Property("type");
if (type == "done")
ReportFatalError("No data!");
else if (type == "debug")
ReportDebug(top.Text());
else if (type == "error")
ReportFatalError("Error message from server: " + top.Text());
else if (type == "result")
{
string symbol = top.Property("symbol");
if (symbol != _owner.Symbol)
ReportFatalError("Wrong symbol. Expecting: " + _owner.Symbol + ". Found: " + symbol);
else
{
ReportSuccess(top.Text());
}
}
else
{ // This is explicitly not an error.
ReportDebug("Unexpected type: " + type);
}
}
}
});
}
private void ReportSuccess(string data)
{
CancelServerRequest();
_owner.ReportResult(data, _serverType);
}
private void ReportDebug(string message)
{
_owner.ReportDebug(message, _serverType);
}
private void ReportFatalError(string message)
{
CancelServerRequest();
_owner.ReportFatalError(message, _serverType);
}
}
static private char[] COMMA = new char[] { ',' };
static private char[] UNIX_END_OF_LINE = new char[] { '\n' };
///
/// Ask for the same data on the old and new servers and print any differences.
///
private class OldNewTest : TestBase
{
override public void ReportFatalError(string message, ServerType serverType)
{
ShowText("Error from " + GetUserFriendly(serverType) + " " + _description + ": " + message);
}
override public void ReportDebug(string message, ServerType serverType)
{
ShowText("Debug from " + GetUserFriendly(serverType) + " " + _description + ": " + message);
}
class Row
{
static private bool CloseEnough(string field1, string field2)
{
if (field1 == field2)
// Exact match.
return true;
double d1, d2;
if (!(double.TryParse(field1, out d1) && double.TryParse(field2, out d2)))
// At least one of these wasn't a number. We only allow exact match if it's not a number.
return false;
if ((d1 <= 0) || (d2 <= 0))
return false;
if ((d2 == Math.Round(d2, 4)) && (d1 == Math.Round(d2, 4)))
// The older server software would often round off to 4 digits after the decimal
// when it shouldn't have. If that's the case, and we can round the newer one
// to get the same answer, this is close enough.
// Example: AAMMF, the close of the candle starting 2018-12-28 12:37:00 is 0.24119.
// The older software would incorrectly round this to 0.2412. Many more
// more examples of the same.
return true;
double CUTOFF = 0.000005;
return Math.Abs(d2 - d1) / d1 < CUTOFF;
}
public bool CloseEnough(Row other)
{
string[] a1 = Original.Split(COMMA);
string[] a2 = other.Original.Split(COMMA);
if (a1.Length != a2.Length)
return false;
for (int i = 0; i < a1.Length; i++)
if (!CloseEnough(a1[i], a2[i]))
return false;
return true;
}
public readonly string Key;
public readonly string Original;
public readonly string Source;
public Row(string original, string source)
{ // We're using field 3 as the key. The first two fields are always the start and end time,
// in a human readable format. For a real chart request the next two fields are the start
// and end time in time_t format. We use the start time in time_t format because that should
// be unique and it is easy to sort by. This includes some assumptions. If we change the
// prototype we should keep the first column as the time or at least something unique. (The
// first column of the request will be the thrid column of the response.)
Key = original.Split(COMMA, 4)[2];
Original = original;
Source = source;
}
public static IList MakeList(IList strings, string source)
{
List rows = new List(strings.Count);
foreach (string s in strings)
if (s != "")
rows.Add(new Row(s, source));
return rows;
}
public static Dictionary MakeDictionary(IList rows)
{
Dictionary result = new Dictionary();
foreach (Row row in rows)
result.Add(row.Key, row);
return result;
}
public static void FindDifferences(IList source, Dictionary comparisons,
IList differences)
{
foreach (Row row in source)
{
Row otherRow = null;
comparisons.TryGetValue(row.Key, out otherRow);
if ((otherRow == null) || !row.CloseEnough(otherRow))
differences.Add(row);
}
}
public static int Compare(Row a, Row b)
{
int result = a.Key.CompareTo(b.Key);
if (result != 0)
return result;
return a.Source.CompareTo(b.Source);
}
}
private IList Compare(string s1, string s2)
{
IList list1 = Row.MakeList(s1.Split(UNIX_END_OF_LINE), "<<<<<< ");
IList list2 = Row.MakeList(s2.Split(UNIX_END_OF_LINE), ">>>>>> ");
var dictionary1 = Row.MakeDictionary(list1);
var dictionary2 = Row.MakeDictionary(list2);
List unmatchedRows = new List();
Row.FindDifferences(list1, dictionary2, unmatchedRows);
Row.FindDifferences(list2, dictionary1, unmatchedRows);
if (unmatchedRows.Count == 0)
return null;
unmatchedRows.Sort(Row.Compare);
List result = new List(unmatchedRows.Count);
foreach (Row row in unmatchedRows)
result.Add(row.Source + row.Original);
return result;
}
private readonly Dictionary _results = new Dictionary();
private string Summarize(ServerType serverType)
{
string responseFromServer;
_results.TryGetValue(serverType, out responseFromServer);
if (null == responseFromServer)
return "No data (null).";
string[] rows = responseFromServer.Split(UNIX_END_OF_LINE);
if (rows.Length == 2)
// One header row on top, one blank row on the bottom.
return "No data (header only).";
if (rows.Length < 3)
return "No data (" + rows.Length + " rows).";
string startTime = rows[1].Split(COMMA)[0];
string endTime = rows[rows.Length - 2].Split(COMMA)[1];
return startTime + " - " + endTime + ", " + (rows.Length - 2) + " rows.";
}
override public void ReportResult(string data, ServerType serverType)
{
_results.Add(serverType, data);
if (_results.Count == 2)
{
IList differences = Compare(_results[ServerType.Test], _results[ServerType.Production]);
if (null == differences)
ShowText("Success: " + _description + "; " + Summarize(ServerType.Test));
else
{
ShowText("Mismatch: " + _description);
//ShowText("Symbol: " + _symbol);
//ShowText("Prototype: " + _prototype);
ShowText("<<<<<< Test >>>>>> Production");
foreach (string line in differences)
ShowText(line);
}
}
}
private readonly String _description;
private readonly OneRequest _testRequest;
private readonly OneRequest _productionRequest;
public OldNewTest(String prototype, String symbol, String description, CompareChartData form) :
base(form)
{
Symbol = symbol;
_description = description;
_testRequest = new OneRequest(this, ServerType.Test, prototype);
_productionRequest = new OneRequest(this, ServerType.Production, prototype);
}
}
private class WeekMonthTest : TestBase
{
private string MakePrototype(int rowCount, string candleSize, long atTime)
{
StringBuilder prototype = new StringBuilder();
prototype.LAppend("daily");
prototype.LAppend(CHART_COLUMNS);
StringBuilder options = new StringBuilder();
options.LAppend("row_count").LAppend(rowCount.ToString());
options.LAppend("pack").LAppend("1");
options.LAppend("candle_size").LAppend(candleSize);
if (atTime > 0)
options.LAppend("at_time").LAppend(atTime.ToString());
prototype.LAppend(options.ToString());
return prototype.ToString();
}
public WeekMonthTest(CompareChartData form, string symbol) : base(form)
{
Symbol = symbol;
bool weekly = _random.Next(2) == 0; // !weekly <--> monthly
int rowCount = _random.Next(1, weekly ? 52 * 2 : 12 * 2); // 2 years.
long atTime = 0;
if (_random.Next(4) == 0)
atTime = ServerFormats.ToTimeT(ServerFormats.Now.AddDays(_random.NextDouble() * -5));
string prototype = MakePrototype(rowCount, weekly ? "week" : "month", atTime);
new OneRequest(this, ServerType.Test, prototype);
}
public override void ReportDebug(string message, ServerType serverType)
{
ShowText("Debug from WeekMonthTest(" + Symbol + "): " + message);
}
public override void ReportFatalError(string message, ServerType serverType)
{
System.Diagnostics.Debug.Assert(serverType == ServerType.Test);
ReportError(message);
}
private void ReportError(string message)
{
ShowText("Error from WeekMonthTest(" + Symbol + "): " + message);
}
private class Row
{
public readonly long StartTime;
public readonly long EndTime;
public readonly double Open;
public readonly double High;
public readonly double Low;
public readonly double Close;
public readonly long Volume;
Row(string asString)
{
string[] pieces = asString.Split(COMMA);
StartTime = (long)double.Parse(pieces[2]);
EndTime = (long)double.Parse(pieces[3]);
Open = double.Parse(pieces[4]);
High = double.Parse(pieces[5]);
Low = double.Parse(pieces[6]);
Close = double.Parse(pieces[7]);
Volume = (long)double.Parse(pieces[8]);
}
public static List MakeAll(string csv)
{
List result = new List();
foreach (string row in csv.Split(UNIX_END_OF_LINE))
try
{
result.Add(new Row(row));
}
catch
{
// Presumably this was the row at the top with all of the column
// headings, or the row at the bottom which is empty.
}
return result;
}
public int CompareTo(Row other)
{
return StartTime.CompareTo(other.StartTime);
}
public string TimesString()
{
return ServerFormats.FromTimeT(StartTime).ToString() + " - "
+ ServerFormats.FromTimeT(EndTime).ToString();
}
}
private class RowAmalgamation
{
public readonly Row BigRow;
public readonly List DailyRows = new List();
public RowAmalgamation(Row bigRow)
{
BigRow = bigRow;
}
public int CompareTo(RowAmalgamation other)
{
return BigRow.CompareTo(other.BigRow);
}
public class Comparer : IComparer
{
int IComparer.Compare(RowAmalgamation x, RowAmalgamation y)
{
return x.CompareTo(y);
}
}
}
private List _allRows;
public override void ReportResult(string data, ServerType serverType)
{
System.Diagnostics.Debug.Assert(serverType == ServerType.Test);
List rows = Row.MakeAll(data);
if (null == _allRows)
InitialResponse(rows);
else
DailyResponse(rows);
}
private void InitialResponse(List rows)
{
if (rows.Count == 0)
{
ShowText("Success from WeekMonthTest(" + Symbol + "): No data.");
return;
}
_allRows = new List(rows.Count);
foreach (Row row in rows)
{
_allRows.Add(new RowAmalgamation(row));
}
_allRows.Sort((a, b) => a.CompareTo(b));
long atTime = _allRows[0].BigRow.StartTime;
string prototype = MakePrototype(1, "day", atTime);
new OneRequest(this, ServerType.Test, prototype);
}
private RowAmalgamation FindRowAmalgamation(Row row)
{
int index = _allRows.BinarySearch(new RowAmalgamation(row), new RowAmalgamation.Comparer());
if (index >= 0)
return _allRows[index];
index = ~index;
// index now points to the first row that starts after the time we are looking for.
System.Diagnostics.Debug.Assert(index > 0);
// We know that the time couldn't be before the first candle. DailyResponse checked for that
// before calling FindRowAmalgamation.
index--;
// index now points to the last row that starts before the time we are looking for.
RowAmalgamation possible = _allRows[index];
if (row.EndTime <= possible.BigRow.EndTime)
return possible;
// We can't find a good place for this row. Either it comes between two rows
// in our list, it or comes after the last row in our list.
return null;
}
private void DailyResponse(List rows)
{
long startTime = _allRows[0].BigRow.StartTime;
int skipped = 0;
foreach (Row row in rows)
{
if (row.StartTime < startTime)
skipped++;
else
{
RowAmalgamation addTo = FindRowAmalgamation(row);
if (null == addTo)
{
ReportError("Can't find a place for daily candle "
+ row.TimesString() + ".");
return;
}
addTo.DailyRows.Add(row);
}
}
foreach (RowAmalgamation current in _allRows)
{
List dailyRows = current.DailyRows;
if (dailyRows.Count == 0)
{
ReportError("No daily candles in range " + current.BigRow.TimesString() + ".");
return;
}
dailyRows.Sort((a, b) => a.CompareTo(b));
double open = dailyRows[0].Open;
double close = dailyRows[dailyRows.Count - 1].Close;
double low = double.MaxValue;
double high = double.MinValue;
long volume = 0;
foreach (Row row in dailyRows)
{
low = Math.Min(low, row.Low);
high = Math.Max(high, row.High);
volume += row.Volume;
}
if ((open != current.BigRow.Open) || (high != current.BigRow.High) || (low != current.BigRow.Low) || (close != current.BigRow.Close) || (volume != current.BigRow.Volume))
{
ReportError("Mismatch " + current.BigRow.TimesString() + "Remote=("
+ current.BigRow.Open + ", " + current.BigRow.High + ", " + current.BigRow.Low
+ ", " + current.BigRow.Close + ", " + current.BigRow.Volume + ") Local=("
+ open + ", " + high + ", " + low + ", " + close + ", " + volume + ").");
return;
}
}
ShowText("Success from WeekMonthTest(" + Symbol + "). Oldest big candle "
+ _allRows[0].BigRow.TimesString() + ". " + skipped + " daily row(s) before that.");
}
}
public CompareChartData(IConnectionMaster connectionMaster)
{
_connectionMaster = connectionMaster;
InitializeComponent();
}
private static readonly Random _random = new Random();
private static readonly string CHART_COLUMNS =
"{START {{current(\"start\")}}} {END {{current(\"end\")}}} {OPEN {{}}} {HIGH {{}}} {LOW {{}}} {CLOSE {{}}} {VOLUME {{}}} {SWING_EXIT {{}}} {AVG_VOLUME {{average(30, 1, \"VOLUME\", 1)}}}";
private static string RandomPrototype(StringBuilder description)
{
int rowCount = 0;
switch (_random.Next(3))
{
case 0:
rowCount = 350;
break;
case 1:
rowCount = 60;
break;
case 2:
rowCount = _random.Next(1, 350);
break;
}
System.Diagnostics.Debug.Assert(rowCount > 0);
description.Append(" row_count=");
description.Append(rowCount);
bool intraDay = _random.Next(2) > 0;
StringBuilder result = new StringBuilder();
// TODO We should really be using LAppend() instead of Append(). See the examples below.
if (intraDay)
result.Append("intraday {");
else
result.Append("daily {");
result.Append(CHART_COLUMNS);
result.Append("} {row_count ");
result.Append(rowCount);
result.Append(" pack 1");
if (intraDay)
{
int[] options = new int[] { 1, 2, 3, 5, 10, 15, 60 };
result.Append(" minutes_per_candle ");
int minutesPerCandle = options[_random.Next(0, options.Length)];
result.Append(minutesPerCandle);
description.Append(" minutes_per_candle=");
description.Append(minutesPerCandle);
if (_random.Next(2) == 0)
{
result.Append(" start_offset 60 end_offset 60");
description.Append(" include_pre_and_post");
}
}
else
description.Append(" daily");
if (_random.Next(5) == 0)
{
DateTime time = ServerFormats.Now;
if (intraDay && (_random.Next(3) != 0))
time = time.AddDays(_random.NextDouble() * -3);
else
time = time.AddDays(_random.NextDouble() * -700);
result.Append(" at_time ");
result.Append(ServerFormats.ToTimeT(time).ToString());
description.Append(" at_time=");
description.Append(time);
}
result.Append('}');
return result.ToString();
}
// These initial symbols were chosen because they've each had a split recently.
private string[] _symbols = new string[] { "CATB", "IRET", "CEI", "SRC", "ROL", "AINV", "CASS", "GNBT",
"LARK", "NULTF", "CBSH", "BMRC", "ITUB", "TNXP", "TRI" };
private int _symbolIndex = -1;
private string NextSymbol()
{
_symbolIndex++;
if (_symbolIndex >= _symbols.Length)
_symbolIndex = 0;
return _symbols[_symbolIndex];
}
private void ShowText(string text)
{
textBox1.AppendText(text);
textBox1.AppendText("\r\n");
}
private static HashSet _symbolBlackList = new HashSet
{
};
private IList NextSymbols()
{
List result = new List();
int count;
if ((!int.TryParse(countTextBox.Text, out count)) || (count < 1))
{
ShowText("\"" + countTextBox.Text + "\" is not a valid count");
}
else
{
result.Capacity = count;
for (int i = 0; i < count; i++)
{
string symbol = NextSymbol();
if (_symbolBlackList.Contains(symbol))
// It's tempting to try grabbing another symbol. But that might lead to an
// infinite loop.
ShowText("Skipping: " + symbol);
else
result.Add(symbol);
}
}
return result;
}
private void oldNewTestButton_Click(object sender, EventArgs e)
{
foreach (string symbol in NextSymbols())
{
StringBuilder description = new StringBuilder("Random Prototype");
String prototype = RandomPrototype(description);
description.Append(" Symbol=");
description.Append(symbol);
new OldNewTest(prototype, symbol, description.ToString(), this);
}
}
public void SymbolListResponse(byte[] body, object clientId)
{
this.BeginInvokeIfRequired(delegate ()
{
if (null == body)
RequestSymbolList();
else
{
_symbols = TalkWithServer.BytesToStr(body).Split(new char[] { ' ' });
ShowText("Loaded " + _symbols.Length + " symbols from server.");
if (_symbols.Length == 0)
_symbols = new string[] { "A", "T", "INTC" };
}
});
}
private void RequestSymbolList()
{
object[] message = new object[]
{
"command", GetCommand(ServerType.Test),
"subcommand", "what_symbols_are_we_following"
};
_connectionMaster.SendManager.SendMessage(TalkWithServer.CreateMessage(message), SymbolListResponse);
}
private void getSymbolListButton_Click(object sender, EventArgs e)
{
RequestSymbolList();
}
private void restartSymbolsButton_Click(object sender, EventArgs e)
{
_symbolIndex = -1;
}
private void weekMonthTestButton_Click(object sender, EventArgs e)
{
foreach (string symbol in NextSymbols())
{
new WeekMonthTest(this, symbol);
}
}
}
///
/// The standard way to add an extension to the menu.
///
class CompareChartDataExtension : IMainInit
{
///
/// Normal init for extensions.
///
///
///
public static IList